diff --git a/.github/workflows/check-non-re2-regex.yml b/.github/workflows/check-non-re2-regex.yml new file mode 100644 index 000000000..fef403fc5 --- /dev/null +++ b/.github/workflows/check-non-re2-regex.yml @@ -0,0 +1,53 @@ +name: Detect Non-RE2 Regex +on: + pull_request: + types: [opened, synchronize] + +jobs: + check-non-re2-regex: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get diff of backend/* + run: | + git diff --unified=0 "origin/${{ github.base_ref }}"...HEAD -- backend/ > diff.txt + + - name: Scan backend diff for non-RE2 regex + run: | + # Extract only added lines (excluding file headers) + grep '^+' diff.txt | grep -v '^+++' | sed 's/^\+//' > added_lines.txt + + if [ ! -s added_lines.txt ]; then + echo "✅ No added lines in backend/ to check for regex usage." + exit 0 + fi + + regex_usage_pattern='(^|[^A-Za-z0-9_"'"'"'`\.\/\\])(\/(?:\\.|[^\/\n\\])+\/[gimsuyv]*(?=\s*[\.\(;,)\]}:]|$)|new RegExp\()' + + # Find all added lines that contain regex patterns + if grep -E "$regex_usage_pattern" added_lines.txt > potential_violations.txt 2>/dev/null; then + # Filter out lines that contain 'new RE2' (allowing for whitespace variations) + if grep -v -E 'new\s+RE2\s*\(' potential_violations.txt > actual_violations.txt 2>/dev/null && [ -s actual_violations.txt ]; then + echo "🚨 ERROR: Found forbidden regex pattern in added/modified backend code." + echo "" + echo "The following lines use raw regex literals (/.../) or new RegExp(...):" + echo "Please replace with 'new RE2(...)' for RE2 compatibility." + echo "" + echo "Offending lines:" + cat actual_violations.txt + exit 1 + else + echo "✅ All identified regex usages are correctly using 'new RE2(...)'." + fi + else + echo "✅ No regex patterns found in added/modified backend lines." + fi + + - name: Cleanup temporary files + if: always() + run: | + rm -f diff.txt added_lines.txt potential_violations.txt actual_violations.txt diff --git a/.infisicalignore b/.infisicalignore index 02cdd4f0e..7c203945f 100644 --- a/.infisicalignore +++ b/.infisicalignore @@ -40,3 +40,4 @@ cli/detect/config/gitleaks.toml:gcp-api-key:578 cli/detect/config/gitleaks.toml:gcp-api-key:579 cli/detect/config/gitleaks.toml:gcp-api-key:581 cli/detect/config/gitleaks.toml:gcp-api-key:582 +backend/src/services/smtp/smtp-service.ts:generic-api-key:79 diff --git a/backend/scripts/generate-schema-types.ts b/backend/scripts/generate-schema-types.ts index fc398c2ac..c0e18a763 100644 --- a/backend/scripts/generate-schema-types.ts +++ b/backend/scripts/generate-schema-types.ts @@ -84,6 +84,11 @@ const getZodDefaultValue = (type: unknown, value: string | number | boolean | Ob } }; +const bigIntegerColumns: Record = { + "folder_commits": ["commitId"] +}; + + const main = async () => { const tables = ( await db("information_schema.tables") @@ -108,6 +113,9 @@ const main = async () => { const columnName = columnNames[colNum]; const colInfo = columns[columnName]; let ztype = getZodPrimitiveType(colInfo.type); + if (bigIntegerColumns[tableName]?.includes(columnName)) { + ztype = "z.coerce.bigint()"; + } if (["zodBuffer"].includes(ztype)) { zodImportSet.add(ztype); } diff --git a/backend/src/@types/fastify.d.ts b/backend/src/@types/fastify.d.ts index 27f14fdb2..9f825b6b6 100644 --- a/backend/src/@types/fastify.d.ts +++ b/backend/src/@types/fastify.d.ts @@ -26,6 +26,7 @@ import { TLdapConfigServiceFactory } from "@app/ee/services/ldap-config/ldap-con import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { TOidcConfigServiceFactory } from "@app/ee/services/oidc/oidc-config-service"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { TPitServiceFactory } from "@app/ee/services/pit/pit-service"; import { TProjectTemplateServiceFactory } from "@app/ee/services/project-template/project-template-service"; import { TProjectUserAdditionalPrivilegeServiceFactory } from "@app/ee/services/project-user-additional-privilege/project-user-additional-privilege-service"; import { TRateLimitServiceFactory } from "@app/ee/services/rate-limit/rate-limit-service"; @@ -59,6 +60,7 @@ import { TCertificateTemplateServiceFactory } from "@app/services/certificate-te import { TCmekServiceFactory } from "@app/services/cmek/cmek-service"; import { TExternalGroupOrgRoleMappingServiceFactory } from "@app/services/external-group-org-role-mapping/external-group-org-role-mapping-service"; import { TExternalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TGroupProjectServiceFactory } from "@app/services/group-project/group-project-service"; import { THsmServiceFactory } from "@app/services/hsm/hsm-service"; import { TIdentityServiceFactory } from "@app/services/identity/identity-service"; @@ -119,6 +121,10 @@ declare module "@fastify/request-context" { oidc?: { claims: Record; }; + kubernetes?: { + namespace: string; + name: string; + }; }; identityPermissionMetadata?: Record; // filled by permission service assumedPrivilegeDetails?: { requesterId: string; actorId: string; actorType: ActorType; projectId: string }; @@ -272,6 +278,8 @@ declare module "fastify" { microsoftTeams: TMicrosoftTeamsServiceFactory; assumePrivileges: TAssumePrivilegeServiceFactory; githubOrgSync: TGithubOrgSyncServiceFactory; + folderCommit: TFolderCommitServiceFactory; + pit: TPitServiceFactory; secretScanningV2: TSecretScanningV2ServiceFactory; internalCertificateAuthority: TInternalCertificateAuthorityServiceFactory; pkiTemplate: TPkiTemplatesServiceFactory; diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index 44cd6bc79..8fd073a49 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -80,6 +80,24 @@ import { TExternalKms, TExternalKmsInsert, TExternalKmsUpdate, + TFolderCheckpointResources, + TFolderCheckpointResourcesInsert, + TFolderCheckpointResourcesUpdate, + TFolderCheckpoints, + TFolderCheckpointsInsert, + TFolderCheckpointsUpdate, + TFolderCommitChanges, + TFolderCommitChangesInsert, + TFolderCommitChangesUpdate, + TFolderCommits, + TFolderCommitsInsert, + TFolderCommitsUpdate, + TFolderTreeCheckpointResources, + TFolderTreeCheckpointResourcesInsert, + TFolderTreeCheckpointResourcesUpdate, + TFolderTreeCheckpoints, + TFolderTreeCheckpointsInsert, + TFolderTreeCheckpointsUpdate, TGateways, TGatewaysInsert, TGatewaysUpdate, @@ -1122,6 +1140,36 @@ declare module "knex/types/tables" { TGithubOrgSyncConfigsInsert, TGithubOrgSyncConfigsUpdate >; + [TableName.FolderCommit]: KnexOriginal.CompositeTableType< + TFolderCommits, + TFolderCommitsInsert, + TFolderCommitsUpdate + >; + [TableName.FolderCommitChanges]: KnexOriginal.CompositeTableType< + TFolderCommitChanges, + TFolderCommitChangesInsert, + TFolderCommitChangesUpdate + >; + [TableName.FolderCheckpoint]: KnexOriginal.CompositeTableType< + TFolderCheckpoints, + TFolderCheckpointsInsert, + TFolderCheckpointsUpdate + >; + [TableName.FolderCheckpointResources]: KnexOriginal.CompositeTableType< + TFolderCheckpointResources, + TFolderCheckpointResourcesInsert, + TFolderCheckpointResourcesUpdate + >; + [TableName.FolderTreeCheckpoint]: KnexOriginal.CompositeTableType< + TFolderTreeCheckpoints, + TFolderTreeCheckpointsInsert, + TFolderTreeCheckpointsUpdate + >; + [TableName.FolderTreeCheckpointResources]: KnexOriginal.CompositeTableType< + TFolderTreeCheckpointResources, + TFolderTreeCheckpointResourcesInsert, + TFolderTreeCheckpointResourcesUpdate + >; [TableName.SecretScanningDataSource]: KnexOriginal.CompositeTableType< TSecretScanningDataSources, TSecretScanningDataSourcesInsert, diff --git a/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts new file mode 100644 index 000000000..30d1ec49b --- /dev/null +++ b/backend/src/db/migrations/20250505194916_add-pit-revamp-tables.ts @@ -0,0 +1,166 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; +import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils"; + +export async function up(knex: Knex): Promise { + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + if (!hasFolderCommitTable) { + await knex.schema.createTable(TableName.FolderCommit, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.bigIncrements("commitId"); + t.jsonb("actorMetadata").notNullable(); + t.string("actorType").notNullable(); + t.string("message"); + t.uuid("folderId").notNullable(); + t.uuid("envId").notNullable(); + t.foreign("envId").references("id").inTable(TableName.Environment).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderId"); + t.index("envId"); + }); + } + + const hasFolderCommitChangesTable = await knex.schema.hasTable(TableName.FolderCommitChanges); + if (!hasFolderCommitChangesTable) { + await knex.schema.createTable(TableName.FolderCommitChanges, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.string("changeType").notNullable(); + t.boolean("isUpdate").notNullable().defaultTo(false); + t.uuid("secretVersionId"); + t.foreign("secretVersionId").references("id").inTable(TableName.SecretVersionV2).onDelete("CASCADE"); + t.uuid("folderVersionId"); + t.foreign("folderVersionId").references("id").inTable(TableName.SecretFolderVersion).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderCommitId"); + t.index("secretVersionId"); + t.index("folderVersionId"); + }); + } + + const hasFolderCheckpointTable = await knex.schema.hasTable(TableName.FolderCheckpoint); + if (!hasFolderCheckpointTable) { + await knex.schema.createTable(TableName.FolderCheckpoint, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderCommitId"); + }); + } + + const hasFolderCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderCheckpointResources); + if (!hasFolderCheckpointResourcesTable) { + await knex.schema.createTable(TableName.FolderCheckpointResources, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderCheckpointId").notNullable(); + t.foreign("folderCheckpointId").references("id").inTable(TableName.FolderCheckpoint).onDelete("CASCADE"); + t.uuid("secretVersionId"); + t.foreign("secretVersionId").references("id").inTable(TableName.SecretVersionV2).onDelete("CASCADE"); + t.uuid("folderVersionId"); + t.foreign("folderVersionId").references("id").inTable(TableName.SecretFolderVersion).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderCheckpointId"); + t.index("secretVersionId"); + t.index("folderVersionId"); + }); + } + + const hasFolderTreeCheckpointTable = await knex.schema.hasTable(TableName.FolderTreeCheckpoint); + if (!hasFolderTreeCheckpointTable) { + await knex.schema.createTable(TableName.FolderTreeCheckpoint, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderCommitId"); + }); + } + + const hasFolderTreeCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderTreeCheckpointResources); + if (!hasFolderTreeCheckpointResourcesTable) { + await knex.schema.createTable(TableName.FolderTreeCheckpointResources, (t) => { + t.uuid("id").primary().defaultTo(knex.fn.uuid()); + t.uuid("folderTreeCheckpointId").notNullable(); + t.foreign("folderTreeCheckpointId").references("id").inTable(TableName.FolderTreeCheckpoint).onDelete("CASCADE"); + t.uuid("folderId").notNullable(); + t.uuid("folderCommitId").notNullable(); + t.foreign("folderCommitId").references("id").inTable(TableName.FolderCommit).onDelete("CASCADE"); + t.timestamps(true, true, true); + + t.index("folderTreeCheckpointId"); + t.index("folderId"); + t.index("folderCommitId"); + }); + } + + if (!hasFolderCommitTable) { + await createOnUpdateTrigger(knex, TableName.FolderCommit); + } + + if (!hasFolderCommitChangesTable) { + await createOnUpdateTrigger(knex, TableName.FolderCommitChanges); + } + + if (!hasFolderCheckpointTable) { + await createOnUpdateTrigger(knex, TableName.FolderCheckpoint); + } + + if (!hasFolderCheckpointResourcesTable) { + await createOnUpdateTrigger(knex, TableName.FolderCheckpointResources); + } + + if (!hasFolderTreeCheckpointTable) { + await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); + } + + if (!hasFolderTreeCheckpointResourcesTable) { + await createOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); + } +} + +export async function down(knex: Knex): Promise { + const hasFolderCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderCheckpointResources); + const hasFolderTreeCheckpointResourcesTable = await knex.schema.hasTable(TableName.FolderTreeCheckpointResources); + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + const hasFolderCommitChangesTable = await knex.schema.hasTable(TableName.FolderCommitChanges); + const hasFolderTreeCheckpointTable = await knex.schema.hasTable(TableName.FolderTreeCheckpoint); + const hasFolderCheckpointTable = await knex.schema.hasTable(TableName.FolderCheckpoint); + + if (hasFolderTreeCheckpointResourcesTable) { + await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpointResources); + await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpointResources); + } + + if (hasFolderCheckpointResourcesTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCheckpointResources); + await knex.schema.dropTableIfExists(TableName.FolderCheckpointResources); + } + + if (hasFolderTreeCheckpointTable) { + await dropOnUpdateTrigger(knex, TableName.FolderTreeCheckpoint); + await knex.schema.dropTableIfExists(TableName.FolderTreeCheckpoint); + } + + if (hasFolderCheckpointTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCheckpoint); + await knex.schema.dropTableIfExists(TableName.FolderCheckpoint); + } + + if (hasFolderCommitChangesTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCommitChanges); + await knex.schema.dropTableIfExists(TableName.FolderCommitChanges); + } + + if (hasFolderCommitTable) { + await dropOnUpdateTrigger(knex, TableName.FolderCommit); + await knex.schema.dropTableIfExists(TableName.FolderCommit); + } +} diff --git a/backend/src/db/migrations/20250528110936_add-folder-description-to-versioning.ts b/backend/src/db/migrations/20250528110936_add-folder-description-to-versioning.ts new file mode 100644 index 000000000..8c1da7ebc --- /dev/null +++ b/backend/src/db/migrations/20250528110936_add-folder-description-to-versioning.ts @@ -0,0 +1,19 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasColumn(TableName.SecretFolderVersion, "description"))) { + await knex.schema.alterTable(TableName.SecretFolderVersion, (t) => { + t.string("description").nullable(); + }); + } +} + +export async function down(knex: Knex): Promise { + if (await knex.schema.hasColumn(TableName.SecretFolderVersion, "description")) { + await knex.schema.alterTable(TableName.SecretFolderVersion, (t) => { + t.dropColumn("description"); + }); + } +} diff --git a/backend/src/db/migrations/20250602155451_fix-secret-versions.ts b/backend/src/db/migrations/20250602155451_fix-secret-versions.ts new file mode 100644 index 000000000..f525e85f3 --- /dev/null +++ b/backend/src/db/migrations/20250602155451_fix-secret-versions.ts @@ -0,0 +1,139 @@ +/* eslint-disable no-await-in-loop */ +import { Knex } from "knex"; + +import { chunkArray } from "@app/lib/fn"; +import { selectAllTableCols } from "@app/lib/knex"; +import { logger } from "@app/lib/logger"; + +import { SecretType, TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + logger.info("Starting secret version fix migration"); + + // Get all shared secret IDs first to optimize versions query + const secretIds = await knex(TableName.SecretV2) + .where("type", SecretType.Shared) + .select("id") + .then((rows) => rows.map((row) => row.id)); + + logger.info(`Found ${secretIds.length} shared secrets to process`); + + if (secretIds.length === 0) { + logger.info("No shared secrets found"); + return; + } + + const secretIdChunks = chunkArray(secretIds, 5000); + + for (let chunkIndex = 0; chunkIndex < secretIdChunks.length; chunkIndex += 1) { + const currentSecretIds = secretIdChunks[chunkIndex]; + logger.info(`Processing chunk ${chunkIndex + 1} of ${secretIdChunks.length}`); + + // Get secrets and versions for current chunk + const [sharedSecrets, allVersions] = await Promise.all([ + knex(TableName.SecretV2).whereIn("id", currentSecretIds).select(selectAllTableCols(TableName.SecretV2)), + knex(TableName.SecretVersionV2).whereIn("secretId", currentSecretIds).select("secretId", "version") + ]); + + const versionsBySecretId = new Map(); + + allVersions.forEach((v) => { + const versions = versionsBySecretId.get(v.secretId); + if (versions) { + versions.push(v.version); + } else { + versionsBySecretId.set(v.secretId, [v.version]); + } + }); + + const versionsToAdd = []; + const secretsToUpdate = []; + + // Process each shared secret + for (const secret of sharedSecrets) { + const existingVersions = versionsBySecretId.get(secret.id) || []; + + if (existingVersions.length === 0) { + // No versions exist - add current version + versionsToAdd.push({ + secretId: secret.id, + version: secret.version, + key: secret.key, + encryptedValue: secret.encryptedValue, + encryptedComment: secret.encryptedComment, + reminderNote: secret.reminderNote, + reminderRepeatDays: secret.reminderRepeatDays, + skipMultilineEncoding: secret.skipMultilineEncoding, + metadata: secret.metadata, + folderId: secret.folderId, + actorType: "platform" + }); + } else { + const latestVersion = Math.max(...existingVersions); + + if (latestVersion !== secret.version) { + // Latest version doesn't match - create new version and update secret + const nextVersion = latestVersion + 1; + + versionsToAdd.push({ + secretId: secret.id, + version: nextVersion, + key: secret.key, + encryptedValue: secret.encryptedValue, + encryptedComment: secret.encryptedComment, + reminderNote: secret.reminderNote, + reminderRepeatDays: secret.reminderRepeatDays, + skipMultilineEncoding: secret.skipMultilineEncoding, + metadata: secret.metadata, + folderId: secret.folderId, + actorType: "platform" + }); + + secretsToUpdate.push({ + id: secret.id, + newVersion: nextVersion + }); + } + } + } + + logger.info( + `Chunk ${chunkIndex + 1}: Adding ${versionsToAdd.length} versions, updating ${secretsToUpdate.length} secrets` + ); + + // Batch insert new versions + if (versionsToAdd.length > 0) { + const insertBatches = chunkArray(versionsToAdd, 9000); + for (let i = 0; i < insertBatches.length; i += 1) { + await knex.batchInsert(TableName.SecretVersionV2, insertBatches[i]); + } + } + + if (secretsToUpdate.length > 0) { + const updateBatches = chunkArray(secretsToUpdate, 1000); + + for (const updateBatch of updateBatches) { + const ids = updateBatch.map((u) => u.id); + const versionCases = updateBatch.map((u) => `WHEN '${u.id}' THEN ${u.newVersion}`).join(" "); + + await knex.raw( + ` + UPDATE ${TableName.SecretV2} + SET version = CASE id ${versionCases} END, + "updatedAt" = NOW() + WHERE id IN (${ids.map(() => "?").join(",")}) + `, + ids + ); + } + } + } + + logger.info("Secret version fix migration completed"); +} + +export async function down(): Promise { + logger.info("Rollback not implemented for secret version fix migration"); + // Note: Rolling back this migration would be complex and potentially destructive + // as it would require tracking which version entries were added +} diff --git a/backend/src/db/migrations/20250602155452_pit-projects-commits-initialization.ts b/backend/src/db/migrations/20250602155452_pit-projects-commits-initialization.ts new file mode 100644 index 000000000..dd4034c57 --- /dev/null +++ b/backend/src/db/migrations/20250602155452_pit-projects-commits-initialization.ts @@ -0,0 +1,345 @@ +import { Knex } from "knex"; + +import { chunkArray } from "@app/lib/fn"; +import { selectAllTableCols } from "@app/lib/knex"; +import { logger } from "@app/lib/logger"; +import { ActorType } from "@app/services/auth/auth-type"; +import { ChangeType } from "@app/services/folder-commit/folder-commit-service"; + +import { + ProjectType, + SecretType, + TableName, + TFolderCheckpoints, + TFolderCommits, + TFolderTreeCheckpoints, + TSecretFolders +} from "../schemas"; + +const sortFoldersByHierarchy = (folders: TSecretFolders[]) => { + // Create a map for quick lookup of children by parent ID + const childrenMap = new Map(); + + // Set of all folder IDs + const allFolderIds = new Set(); + + // Build the set of all folder IDs + folders.forEach((folder) => { + if (folder.id) { + allFolderIds.add(folder.id); + } + }); + + // Group folders by their parentId + folders.forEach((folder) => { + if (folder.parentId) { + const children = childrenMap.get(folder.parentId) || []; + children.push(folder); + childrenMap.set(folder.parentId, children); + } + }); + + // Find root folders - those with no parentId or with a parentId that doesn't exist + const rootFolders = folders.filter((folder) => !folder.parentId || !allFolderIds.has(folder.parentId)); + + // Process each level of the hierarchy + const result = []; + let currentLevel = rootFolders; + + while (currentLevel.length > 0) { + result.push(...currentLevel); + + const nextLevel = []; + for (const folder of currentLevel) { + if (folder.id) { + const children = childrenMap.get(folder.id) || []; + nextLevel.push(...children); + } + } + + currentLevel = nextLevel; + } + + return result.reverse(); +}; + +const getSecretsByFolderIds = async (knex: Knex, folderIds: string[]): Promise> => { + const secrets = await knex(TableName.SecretV2) + .whereIn(`${TableName.SecretV2}.folderId`, folderIds) + .where(`${TableName.SecretV2}.type`, SecretType.Shared) + .join(TableName.SecretVersionV2, (queryBuilder) => { + void queryBuilder + .on(`${TableName.SecretVersionV2}.secretId`, `${TableName.SecretV2}.id`) + .andOn(`${TableName.SecretVersionV2}.version`, `${TableName.SecretV2}.version`); + }) + .select(selectAllTableCols(TableName.SecretV2)) + .select(knex.ref("id").withSchema(TableName.SecretVersionV2).as("secretVersionId")); + + const secretsMap: Record = {}; + + secrets.forEach((secret) => { + if (!secretsMap[secret.folderId]) { + secretsMap[secret.folderId] = []; + } + secretsMap[secret.folderId].push(secret.secretVersionId); + }); + + return secretsMap; +}; + +const getFoldersByParentIds = async (knex: Knex, parentIds: string[]): Promise> => { + const folders = await knex(TableName.SecretFolder) + .whereIn(`${TableName.SecretFolder}.parentId`, parentIds) + .where(`${TableName.SecretFolder}.isReserved`, false) + .join(TableName.SecretFolderVersion, (queryBuilder) => { + void queryBuilder + .on(`${TableName.SecretFolderVersion}.folderId`, `${TableName.SecretFolder}.id`) + .andOn(`${TableName.SecretFolderVersion}.version`, `${TableName.SecretFolder}.version`); + }) + .select(selectAllTableCols(TableName.SecretFolder)) + .select(knex.ref("id").withSchema(TableName.SecretFolderVersion).as("folderVersionId")); + + const foldersMap: Record = {}; + + folders.forEach((folder) => { + if (!folder.parentId) { + return; + } + if (!foldersMap[folder.parentId]) { + foldersMap[folder.parentId] = []; + } + foldersMap[folder.parentId].push(folder.folderVersionId); + }); + + return foldersMap; +}; + +export async function up(knex: Knex): Promise { + logger.info("Initializing folder commits"); + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + if (hasFolderCommitTable) { + // Get Projects to Initialize + const projects = await knex(TableName.Project) + .where(`${TableName.Project}.version`, 3) + .where(`${TableName.Project}.type`, ProjectType.SecretManager) + .select(selectAllTableCols(TableName.Project)); + logger.info(`Found ${projects.length} projects to initialize`); + + // Process Projects in batches of 100 + const batches = chunkArray(projects, 100); + let i = 0; + for (const batch of batches) { + i += 1; + logger.info(`Processing project batch ${i} of ${batches.length}`); + let foldersCommitsList = []; + + const rootFoldersMap: Record = {}; + const envRootFoldersMap: Record = {}; + + // Get All Folders for the Project + // eslint-disable-next-line no-await-in-loop + const folders = await knex(TableName.SecretFolder) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .whereIn( + `${TableName.Environment}.projectId`, + batch.map((project) => project.id) + ) + .where(`${TableName.SecretFolder}.isReserved`, false) + .select(selectAllTableCols(TableName.SecretFolder)); + logger.info(`Found ${folders.length} folders to initialize in project batch ${i} of ${batches.length}`); + + // Sort Folders by Hierarchy (parents before nested folders) + const sortedFolders = sortFoldersByHierarchy(folders); + + // eslint-disable-next-line no-await-in-loop + const folderSecretsMap = await getSecretsByFolderIds( + knex, + sortedFolders.map((folder) => folder.id) + ); + // eslint-disable-next-line no-await-in-loop + const folderFoldersMap = await getFoldersByParentIds( + knex, + sortedFolders.map((folder) => folder.id) + ); + + // Get folder commit changes + for (const folder of sortedFolders) { + const subFolderVersionIds = folderFoldersMap[folder.id]; + const secretVersionIds = folderSecretsMap[folder.id]; + const changes = []; + if (subFolderVersionIds) { + changes.push( + ...subFolderVersionIds.map((folderVersionId) => ({ + folderId: folder.id, + changeType: ChangeType.ADD, + secretVersionId: undefined, + folderVersionId, + isUpdate: false + })) + ); + } + if (secretVersionIds) { + changes.push( + ...secretVersionIds.map((secretVersionId) => ({ + folderId: folder.id, + changeType: ChangeType.ADD, + secretVersionId, + folderVersionId: undefined, + isUpdate: false + })) + ); + } + if (changes.length > 0) { + const folderCommit = { + commit: { + actorMetadata: {}, + actorType: ActorType.PLATFORM, + message: "Initialized folder", + folderId: folder.id, + envId: folder.envId + }, + changes + }; + foldersCommitsList.push(folderCommit); + if (!folder.parentId) { + rootFoldersMap[folder.id] = folder.envId; + envRootFoldersMap[folder.envId] = folder.id; + } + } + } + logger.info(`Retrieved folder changes for project batch ${i} of ${batches.length}`); + + const filteredBrokenProjectFolders: string[] = []; + + foldersCommitsList = foldersCommitsList.filter((folderCommit) => { + if (!envRootFoldersMap[folderCommit.commit.envId]) { + filteredBrokenProjectFolders.push(folderCommit.commit.folderId); + return false; + } + return true; + }); + + logger.info( + `Filtered ${filteredBrokenProjectFolders.length} broken project folders: ${JSON.stringify(filteredBrokenProjectFolders)}` + ); + + // Insert New Commits in batches of 9000 + const newCommits = foldersCommitsList.map((folderCommit) => folderCommit.commit); + const commitBatches = chunkArray(newCommits, 9000); + + let j = 0; + for (const commitBatch of commitBatches) { + j += 1; + logger.info(`Inserting folder commits - batch ${j} of ${commitBatches.length}`); + // Create folder commit + // eslint-disable-next-line no-await-in-loop + const newCommitsInserted = (await knex + .batchInsert(TableName.FolderCommit, commitBatch) + .returning("*")) as TFolderCommits[]; + + logger.info(`Finished inserting folder commits - batch ${j} of ${commitBatches.length}`); + + const newCommitsMap: Record = {}; + const newCommitsMapInverted: Record = {}; + const newCheckpointsMap: Record = {}; + newCommitsInserted.forEach((commit) => { + newCommitsMap[commit.folderId] = commit.id; + newCommitsMapInverted[commit.id] = commit.folderId; + }); + + // Create folder checkpoints + // eslint-disable-next-line no-await-in-loop + const newCheckpoints = (await knex + .batchInsert( + TableName.FolderCheckpoint, + Object.values(newCommitsMap).map((commitId) => ({ + folderCommitId: commitId + })) + ) + .returning("*")) as TFolderCheckpoints[]; + + logger.info(`Finished inserting folder checkpoints - batch ${j} of ${commitBatches.length}`); + + newCheckpoints.forEach((checkpoint) => { + newCheckpointsMap[newCommitsMapInverted[checkpoint.folderCommitId]] = checkpoint.id; + }); + + // Create folder commit changes + // eslint-disable-next-line no-await-in-loop + await knex.batchInsert( + TableName.FolderCommitChanges, + foldersCommitsList + .map((folderCommit) => folderCommit.changes) + .flat() + .map((change) => ({ + folderCommitId: newCommitsMap[change.folderId], + changeType: change.changeType, + secretVersionId: change.secretVersionId, + folderVersionId: change.folderVersionId, + isUpdate: false + })) + ); + + logger.info(`Finished inserting folder commit changes - batch ${j} of ${commitBatches.length}`); + + // Create folder checkpoint resources + // eslint-disable-next-line no-await-in-loop + await knex.batchInsert( + TableName.FolderCheckpointResources, + foldersCommitsList + .map((folderCommit) => folderCommit.changes) + .flat() + .map((change) => ({ + folderCheckpointId: newCheckpointsMap[change.folderId], + folderVersionId: change.folderVersionId, + secretVersionId: change.secretVersionId + })) + ); + + logger.info(`Finished inserting folder checkpoint resources - batch ${j} of ${commitBatches.length}`); + + // Create Folder Tree Checkpoint + // eslint-disable-next-line no-await-in-loop + const newTreeCheckpoints = (await knex + .batchInsert( + TableName.FolderTreeCheckpoint, + Object.keys(rootFoldersMap).map((folderId) => ({ + folderCommitId: newCommitsMap[folderId] + })) + ) + .returning("*")) as TFolderTreeCheckpoints[]; + + logger.info(`Finished inserting folder tree checkpoints - batch ${j} of ${commitBatches.length}`); + + const newTreeCheckpointsMap: Record = {}; + newTreeCheckpoints.forEach((checkpoint) => { + newTreeCheckpointsMap[rootFoldersMap[newCommitsMapInverted[checkpoint.folderCommitId]]] = checkpoint.id; + }); + + // Create Folder Tree Checkpoint Resources + // eslint-disable-next-line no-await-in-loop + await knex + .batchInsert( + TableName.FolderTreeCheckpointResources, + newCommitsInserted.map((folderCommit) => ({ + folderTreeCheckpointId: newTreeCheckpointsMap[folderCommit.envId], + folderId: folderCommit.folderId, + folderCommitId: folderCommit.id + })) + ) + .returning("*"); + + logger.info(`Finished inserting folder tree checkpoint resources - batch ${j} of ${commitBatches.length}`); + } + } + } + logger.info("Folder commits initialized"); +} + +export async function down(knex: Knex): Promise { + const hasFolderCommitTable = await knex.schema.hasTable(TableName.FolderCommit); + if (hasFolderCommitTable) { + // delete all existing entries + await knex(TableName.FolderCommit).del(); + } +} diff --git a/backend/src/db/migrations/20250604174128_identity-kubernetes-auth-gateway-reviewer.ts b/backend/src/db/migrations/20250604174128_identity-kubernetes-auth-gateway-reviewer.ts new file mode 100644 index 000000000..da5493153 --- /dev/null +++ b/backend/src/db/migrations/20250604174128_identity-kubernetes-auth-gateway-reviewer.ts @@ -0,0 +1,23 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + const hasTokenReviewModeColumn = await knex.schema.hasColumn(TableName.IdentityKubernetesAuth, "tokenReviewMode"); + + if (!hasTokenReviewModeColumn) { + await knex.schema.alterTable(TableName.IdentityKubernetesAuth, (table) => { + table.string("tokenReviewMode").notNullable().defaultTo("api"); + }); + } +} + +export async function down(knex: Knex): Promise { + const hasTokenReviewModeColumn = await knex.schema.hasColumn(TableName.IdentityKubernetesAuth, "tokenReviewMode"); + + if (hasTokenReviewModeColumn) { + await knex.schema.alterTable(TableName.IdentityKubernetesAuth, (table) => { + table.dropColumn("tokenReviewMode"); + }); + } +} diff --git a/backend/src/db/migrations/20250606134139_add-project-snapshots-legacy-option.ts b/backend/src/db/migrations/20250606134139_add-project-snapshots-legacy-option.ts new file mode 100644 index 000000000..f5f73d7fe --- /dev/null +++ b/backend/src/db/migrations/20250606134139_add-project-snapshots-legacy-option.ts @@ -0,0 +1,21 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + const hasShowSnapshotsLegacyColumn = await knex.schema.hasColumn(TableName.Project, "showSnapshotsLegacy"); + if (!hasShowSnapshotsLegacyColumn) { + await knex.schema.table(TableName.Project, (table) => { + table.boolean("showSnapshotsLegacy").notNullable().defaultTo(false); + }); + } +} + +export async function down(knex: Knex): Promise { + const hasShowSnapshotsLegacyColumn = await knex.schema.hasColumn(TableName.Project, "showSnapshotsLegacy"); + if (hasShowSnapshotsLegacyColumn) { + await knex.schema.table(TableName.Project, (table) => { + table.dropColumn("showSnapshotsLegacy"); + }); + } +} diff --git a/backend/src/db/migrations/20250610143920_add-dynamic-secret-lease-config.ts b/backend/src/db/migrations/20250610143920_add-dynamic-secret-lease-config.ts new file mode 100644 index 000000000..30d6f1854 --- /dev/null +++ b/backend/src/db/migrations/20250610143920_add-dynamic-secret-lease-config.ts @@ -0,0 +1,21 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + const hasConfigColumn = await knex.schema.hasColumn(TableName.DynamicSecretLease, "config"); + if (!hasConfigColumn) { + await knex.schema.alterTable(TableName.DynamicSecretLease, (table) => { + table.jsonb("config"); + }); + } +} + +export async function down(knex: Knex): Promise { + const hasConfigColumn = await knex.schema.hasColumn(TableName.DynamicSecretLease, "config"); + if (hasConfigColumn) { + await knex.schema.alterTable(TableName.DynamicSecretLease, (table) => { + table.dropColumn("config"); + }); + } +} diff --git a/backend/src/db/migrations/20250610173646_identity-kubernetes-auth-optional-host-url.ts b/backend/src/db/migrations/20250610173646_identity-kubernetes-auth-optional-host-url.ts new file mode 100644 index 000000000..6d2dd6664 --- /dev/null +++ b/backend/src/db/migrations/20250610173646_identity-kubernetes-auth-optional-host-url.ts @@ -0,0 +1,45 @@ +import { Knex } from "knex"; + +import { selectAllTableCols } from "@app/lib/knex"; + +import { TableName } from "../schemas"; + +const BATCH_SIZE = 1000; + +export async function up(knex: Knex): Promise { + const hasKubernetesHostColumn = await knex.schema.hasColumn(TableName.IdentityKubernetesAuth, "kubernetesHost"); + + if (hasKubernetesHostColumn) { + await knex.schema.alterTable(TableName.IdentityKubernetesAuth, (table) => { + table.string("kubernetesHost").nullable().alter(); + }); + } +} + +export async function down(knex: Knex): Promise { + const hasKubernetesHostColumn = await knex.schema.hasColumn(TableName.IdentityKubernetesAuth, "kubernetesHost"); + + // find all rows where kubernetesHost is null + const rows = await knex(TableName.IdentityKubernetesAuth) + .whereNull("kubernetesHost") + .select(selectAllTableCols(TableName.IdentityKubernetesAuth)); + + if (rows.length > 0) { + for (let i = 0; i < rows.length; i += BATCH_SIZE) { + const batch = rows.slice(i, i + BATCH_SIZE); + // eslint-disable-next-line no-await-in-loop + await knex(TableName.IdentityKubernetesAuth) + .whereIn( + "id", + batch.map((row) => row.id) + ) + .update({ kubernetesHost: "" }); + } + } + + if (hasKubernetesHostColumn) { + await knex.schema.alterTable(TableName.IdentityKubernetesAuth, (table) => { + table.string("kubernetesHost").notNullable().alter(); + }); + } +} diff --git a/backend/src/db/migrations/utils/services.ts b/backend/src/db/migrations/utils/services.ts index 731f703e2..0e071e6fe 100644 --- a/backend/src/db/migrations/utils/services.ts +++ b/backend/src/db/migrations/utils/services.ts @@ -3,12 +3,27 @@ import { Knex } from "knex"; import { initializeHsmModule } from "@app/ee/services/hsm/hsm-fns"; import { hsmServiceFactory } from "@app/ee/services/hsm/hsm-service"; import { TKeyStoreFactory } from "@app/keystore/keystore"; +import { folderCheckpointDALFactory } from "@app/services/folder-checkpoint/folder-checkpoint-dal"; +import { folderCheckpointResourcesDALFactory } from "@app/services/folder-checkpoint-resources/folder-checkpoint-resources-dal"; +import { folderCommitDALFactory } from "@app/services/folder-commit/folder-commit-dal"; +import { folderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; +import { folderCommitChangesDALFactory } from "@app/services/folder-commit-changes/folder-commit-changes-dal"; +import { folderTreeCheckpointDALFactory } from "@app/services/folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { folderTreeCheckpointResourcesDALFactory } from "@app/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal"; +import { identityDALFactory } from "@app/services/identity/identity-dal"; import { internalKmsDALFactory } from "@app/services/kms/internal-kms-dal"; import { kmskeyDALFactory } from "@app/services/kms/kms-key-dal"; import { kmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal"; import { kmsServiceFactory } from "@app/services/kms/kms-service"; import { orgDALFactory } from "@app/services/org/org-dal"; import { projectDALFactory } from "@app/services/project/project-dal"; +import { resourceMetadataDALFactory } from "@app/services/resource-metadata/resource-metadata-dal"; +import { secretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { secretFolderVersionDALFactory } from "@app/services/secret-folder/secret-folder-version-dal"; +import { secretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; +import { secretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal"; +import { secretVersionV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-version-dal"; +import { userDALFactory } from "@app/services/user/user-dal"; import { TMigrationEnvConfig } from "./env-config"; @@ -50,3 +65,77 @@ export const getMigrationEncryptionServices = async ({ envConfig, db, keyStore } return { kmsService }; }; + +export const getMigrationPITServices = async ({ + db, + keyStore, + envConfig +}: { + db: Knex; + keyStore: TKeyStoreFactory; + envConfig: TMigrationEnvConfig; +}) => { + const projectDAL = projectDALFactory(db); + const folderCommitDAL = folderCommitDALFactory(db); + const folderCommitChangesDAL = folderCommitChangesDALFactory(db); + const folderCheckpointDAL = folderCheckpointDALFactory(db); + const folderTreeCheckpointDAL = folderTreeCheckpointDALFactory(db); + const userDAL = userDALFactory(db); + const identityDAL = identityDALFactory(db); + const folderDAL = secretFolderDALFactory(db); + const folderVersionDAL = secretFolderVersionDALFactory(db); + const secretVersionV2BridgeDAL = secretVersionV2BridgeDALFactory(db); + const folderCheckpointResourcesDAL = folderCheckpointResourcesDALFactory(db); + const secretV2BridgeDAL = secretV2BridgeDALFactory({ db, keyStore }); + const folderTreeCheckpointResourcesDAL = folderTreeCheckpointResourcesDALFactory(db); + const secretTagDAL = secretTagDALFactory(db); + + const orgDAL = orgDALFactory(db); + const kmsRootConfigDAL = kmsRootConfigDALFactory(db); + const kmsDAL = kmskeyDALFactory(db); + const internalKmsDAL = internalKmsDALFactory(db); + const resourceMetadataDAL = resourceMetadataDALFactory(db); + + const hsmModule = initializeHsmModule(envConfig); + hsmModule.initialize(); + + const hsmService = hsmServiceFactory({ + hsmModule: hsmModule.getModule(), + envConfig + }); + + const kmsService = kmsServiceFactory({ + kmsRootConfigDAL, + keyStore, + kmsDAL, + internalKmsDAL, + orgDAL, + projectDAL, + hsmService, + envConfig + }); + + await hsmService.startService(); + await kmsService.startService(); + + const folderCommitService = folderCommitServiceFactory({ + folderCommitDAL, + folderCommitChangesDAL, + folderCheckpointDAL, + folderTreeCheckpointDAL, + userDAL, + identityDAL, + folderDAL, + folderVersionDAL, + secretVersionV2BridgeDAL, + projectDAL, + folderCheckpointResourcesDAL, + secretV2BridgeDAL, + folderTreeCheckpointResourcesDAL, + kmsService, + secretTagDAL, + resourceMetadataDAL + }); + + return { folderCommitService }; +}; diff --git a/backend/src/db/schemas/dynamic-secret-leases.ts b/backend/src/db/schemas/dynamic-secret-leases.ts index 8c16bcb55..ef16b1a30 100644 --- a/backend/src/db/schemas/dynamic-secret-leases.ts +++ b/backend/src/db/schemas/dynamic-secret-leases.ts @@ -16,7 +16,8 @@ export const DynamicSecretLeasesSchema = z.object({ statusDetails: z.string().nullable().optional(), dynamicSecretId: z.string().uuid(), createdAt: z.date(), - updatedAt: z.date() + updatedAt: z.date(), + config: z.unknown().nullable().optional() }); export type TDynamicSecretLeases = z.infer; diff --git a/backend/src/db/schemas/folder-checkpoint-resources.ts b/backend/src/db/schemas/folder-checkpoint-resources.ts new file mode 100644 index 000000000..5fa8215cf --- /dev/null +++ b/backend/src/db/schemas/folder-checkpoint-resources.ts @@ -0,0 +1,23 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCheckpointResourcesSchema = z.object({ + id: z.string().uuid(), + folderCheckpointId: z.string().uuid(), + secretVersionId: z.string().uuid().nullable().optional(), + folderVersionId: z.string().uuid().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCheckpointResources = z.infer; +export type TFolderCheckpointResourcesInsert = Omit, TImmutableDBKeys>; +export type TFolderCheckpointResourcesUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/folder-checkpoints.ts b/backend/src/db/schemas/folder-checkpoints.ts new file mode 100644 index 000000000..ba0ce6f71 --- /dev/null +++ b/backend/src/db/schemas/folder-checkpoints.ts @@ -0,0 +1,19 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCheckpointsSchema = z.object({ + id: z.string().uuid(), + folderCommitId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCheckpoints = z.infer; +export type TFolderCheckpointsInsert = Omit, TImmutableDBKeys>; +export type TFolderCheckpointsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/folder-commit-changes.ts b/backend/src/db/schemas/folder-commit-changes.ts new file mode 100644 index 000000000..2bee0c5b3 --- /dev/null +++ b/backend/src/db/schemas/folder-commit-changes.ts @@ -0,0 +1,23 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCommitChangesSchema = z.object({ + id: z.string().uuid(), + folderCommitId: z.string().uuid(), + changeType: z.string(), + isUpdate: z.boolean().default(false), + secretVersionId: z.string().uuid().nullable().optional(), + folderVersionId: z.string().uuid().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCommitChanges = z.infer; +export type TFolderCommitChangesInsert = Omit, TImmutableDBKeys>; +export type TFolderCommitChangesUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/folder-commits.ts b/backend/src/db/schemas/folder-commits.ts new file mode 100644 index 000000000..ade480eda --- /dev/null +++ b/backend/src/db/schemas/folder-commits.ts @@ -0,0 +1,24 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderCommitsSchema = z.object({ + id: z.string().uuid(), + commitId: z.coerce.bigint(), + actorMetadata: z.unknown(), + actorType: z.string(), + message: z.string().nullable().optional(), + folderId: z.string().uuid(), + envId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderCommits = z.infer; +export type TFolderCommitsInsert = Omit, TImmutableDBKeys>; +export type TFolderCommitsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/folder-tree-checkpoint-resources.ts b/backend/src/db/schemas/folder-tree-checkpoint-resources.ts new file mode 100644 index 000000000..06d5770cc --- /dev/null +++ b/backend/src/db/schemas/folder-tree-checkpoint-resources.ts @@ -0,0 +1,26 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderTreeCheckpointResourcesSchema = z.object({ + id: z.string().uuid(), + folderTreeCheckpointId: z.string().uuid(), + folderId: z.string().uuid(), + folderCommitId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderTreeCheckpointResources = z.infer; +export type TFolderTreeCheckpointResourcesInsert = Omit< + z.input, + TImmutableDBKeys +>; +export type TFolderTreeCheckpointResourcesUpdate = Partial< + Omit, TImmutableDBKeys> +>; diff --git a/backend/src/db/schemas/folder-tree-checkpoints.ts b/backend/src/db/schemas/folder-tree-checkpoints.ts new file mode 100644 index 000000000..ea500af6b --- /dev/null +++ b/backend/src/db/schemas/folder-tree-checkpoints.ts @@ -0,0 +1,19 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const FolderTreeCheckpointsSchema = z.object({ + id: z.string().uuid(), + folderCommitId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TFolderTreeCheckpoints = z.infer; +export type TFolderTreeCheckpointsInsert = Omit, TImmutableDBKeys>; +export type TFolderTreeCheckpointsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/identity-kubernetes-auths.ts b/backend/src/db/schemas/identity-kubernetes-auths.ts index 00d1fd771..deb78bf8a 100644 --- a/backend/src/db/schemas/identity-kubernetes-auths.ts +++ b/backend/src/db/schemas/identity-kubernetes-auths.ts @@ -18,7 +18,7 @@ export const IdentityKubernetesAuthsSchema = z.object({ createdAt: z.date(), updatedAt: z.date(), identityId: z.string().uuid(), - kubernetesHost: z.string(), + kubernetesHost: z.string().nullable().optional(), encryptedCaCert: z.string().nullable().optional(), caCertIV: z.string().nullable().optional(), caCertTag: z.string().nullable().optional(), @@ -31,7 +31,8 @@ export const IdentityKubernetesAuthsSchema = z.object({ encryptedKubernetesTokenReviewerJwt: zodBuffer.nullable().optional(), encryptedKubernetesCaCertificate: zodBuffer.nullable().optional(), gatewayId: z.string().uuid().nullable().optional(), - accessTokenPeriod: z.coerce.number().default(0) + accessTokenPeriod: z.coerce.number().default(0), + tokenReviewMode: z.string().default("api") }); export type TIdentityKubernetesAuths = z.infer; diff --git a/backend/src/db/schemas/index.ts b/backend/src/db/schemas/index.ts index 6743a23cc..7a27caf9e 100644 --- a/backend/src/db/schemas/index.ts +++ b/backend/src/db/schemas/index.ts @@ -24,6 +24,12 @@ export * from "./dynamic-secrets"; export * from "./external-certificate-authorities"; export * from "./external-group-org-role-mappings"; export * from "./external-kms"; +export * from "./folder-checkpoint-resources"; +export * from "./folder-checkpoints"; +export * from "./folder-commit-changes"; +export * from "./folder-commits"; +export * from "./folder-tree-checkpoint-resources"; +export * from "./folder-tree-checkpoints"; export * from "./gateways"; export * from "./git-app-install-sessions"; export * from "./git-app-org"; diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 6722ce235..df0a858b9 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -160,6 +160,12 @@ export enum TableName { ProjectMicrosoftTeamsConfigs = "project_microsoft_teams_configs", SecretReminderRecipients = "secret_reminder_recipients", GithubOrgSyncConfig = "github_org_sync_configs", + FolderCommit = "folder_commits", + FolderCommitChanges = "folder_commit_changes", + FolderCheckpoint = "folder_checkpoints", + FolderCheckpointResources = "folder_checkpoint_resources", + FolderTreeCheckpoint = "folder_tree_checkpoints", + FolderTreeCheckpointResources = "folder_tree_checkpoint_resources", SecretScanningDataSource = "secret_scanning_data_sources", SecretScanningResource = "secret_scanning_resources", SecretScanningScan = "secret_scanning_scans", @@ -167,7 +173,7 @@ export enum TableName { SecretScanningConfig = "secret_scanning_configs" } -export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt"; +export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt" | "commitId"; export const UserDeviceSchema = z .object({ diff --git a/backend/src/db/schemas/projects.ts b/backend/src/db/schemas/projects.ts index c1e96e8ce..b4c98d8a2 100644 --- a/backend/src/db/schemas/projects.ts +++ b/backend/src/db/schemas/projects.ts @@ -28,7 +28,8 @@ export const ProjectsSchema = z.object({ type: z.string(), enforceCapitalization: z.boolean().default(false), hasDeleteProtection: z.boolean().default(false).nullable().optional(), - secretSharing: z.boolean().default(true) + secretSharing: z.boolean().default(true), + showSnapshotsLegacy: z.boolean().default(false) }); export type TProjects = z.infer; diff --git a/backend/src/db/schemas/secret-folder-versions.ts b/backend/src/db/schemas/secret-folder-versions.ts index 8bef6e83f..3d444c566 100644 --- a/backend/src/db/schemas/secret-folder-versions.ts +++ b/backend/src/db/schemas/secret-folder-versions.ts @@ -14,7 +14,8 @@ export const SecretFolderVersionsSchema = z.object({ createdAt: z.date(), updatedAt: z.date(), envId: z.string().uuid(), - folderId: z.string().uuid() + folderId: z.string().uuid(), + description: z.string().nullable().optional() }); export type TSecretFolderVersions = z.infer; diff --git a/backend/src/ee/routes/v1/dynamic-secret-lease-router.ts b/backend/src/ee/routes/v1/dynamic-secret-lease-router.ts index 7c42c7f99..26c27d0d3 100644 --- a/backend/src/ee/routes/v1/dynamic-secret-lease-router.ts +++ b/backend/src/ee/routes/v1/dynamic-secret-lease-router.ts @@ -36,7 +36,8 @@ export const registerDynamicSecretLeaseRouter = async (server: FastifyZodProvide ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than a day" }); }), path: z.string().trim().default("/").transform(removeTrailingSlash).describe(DYNAMIC_SECRET_LEASES.CREATE.path), - environmentSlug: z.string().min(1).describe(DYNAMIC_SECRET_LEASES.CREATE.path) + environmentSlug: z.string().min(1).describe(DYNAMIC_SECRET_LEASES.CREATE.environmentSlug), + config: z.any().optional() }), response: { 200: z.object({ diff --git a/backend/src/ee/routes/v1/dynamic-secret-lease-routers/kubernetes-lease-router.ts b/backend/src/ee/routes/v1/dynamic-secret-lease-routers/kubernetes-lease-router.ts new file mode 100644 index 000000000..f2751c635 --- /dev/null +++ b/backend/src/ee/routes/v1/dynamic-secret-lease-routers/kubernetes-lease-router.ts @@ -0,0 +1,67 @@ +import { z } from "zod"; + +import { DynamicSecretLeasesSchema } from "@app/db/schemas"; +import { ApiDocsTags, DYNAMIC_SECRET_LEASES } from "@app/lib/api-docs"; +import { daysToMillisecond } from "@app/lib/dates"; +import { removeTrailingSlash } from "@app/lib/fn"; +import { ms } from "@app/lib/ms"; +import { writeLimit } from "@app/server/config/rateLimiter"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { SanitizedDynamicSecretSchema } from "@app/server/routes/sanitizedSchemas"; +import { AuthMode } from "@app/services/auth/auth-type"; + +export const registerKubernetesDynamicSecretLeaseRouter = async (server: FastifyZodProvider) => { + server.route({ + method: "POST", + url: "/", + config: { + rateLimit: writeLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.DynamicSecrets], + body: z.object({ + dynamicSecretName: z.string().min(1).describe(DYNAMIC_SECRET_LEASES.CREATE.dynamicSecretName).toLowerCase(), + projectSlug: z.string().min(1).describe(DYNAMIC_SECRET_LEASES.CREATE.projectSlug), + ttl: z + .string() + .optional() + .describe(DYNAMIC_SECRET_LEASES.CREATE.ttl) + .superRefine((val, ctx) => { + if (!val) return; + const valMs = ms(val); + if (valMs < 60 * 1000) + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be greater than 1min" }); + if (valMs > daysToMillisecond(1)) + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than a day" }); + }), + path: z.string().trim().default("/").transform(removeTrailingSlash).describe(DYNAMIC_SECRET_LEASES.CREATE.path), + environmentSlug: z.string().min(1).describe(DYNAMIC_SECRET_LEASES.CREATE.environmentSlug), + config: z + .object({ + namespace: z.string().min(1).optional().describe(DYNAMIC_SECRET_LEASES.KUBERNETES.CREATE.config.namespace) + }) + .optional() + }), + response: { + 200: z.object({ + lease: DynamicSecretLeasesSchema, + dynamicSecret: SanitizedDynamicSecretSchema, + data: z.unknown() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { data, lease, dynamicSecret } = await server.services.dynamicSecretLease.create({ + actor: req.permission.type, + actorId: req.permission.id, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + name: req.body.dynamicSecretName, + ...req.body + }); + return { lease, data, dynamicSecret }; + } + }); +}; diff --git a/backend/src/ee/routes/v1/dynamic-secret-router.ts b/backend/src/ee/routes/v1/dynamic-secret-router.ts index bf5cce7d5..b916bab67 100644 --- a/backend/src/ee/routes/v1/dynamic-secret-router.ts +++ b/backend/src/ee/routes/v1/dynamic-secret-router.ts @@ -23,7 +23,10 @@ const validateUsernameTemplateCharacters = characterValidator([ CharacterType.CloseBrace, CharacterType.CloseBracket, CharacterType.OpenBracket, - CharacterType.Fullstop + CharacterType.Fullstop, + CharacterType.SingleQuote, + CharacterType.Spaces, + CharacterType.Pipe ]); const userTemplateSchema = z @@ -33,7 +36,7 @@ const userTemplateSchema = z .refine((el) => validateUsernameTemplateCharacters(el)) .refine((el) => isValidHandleBarTemplate(el, { - allowedExpressions: (val) => ["randomUsername", "unixTimestamp"].includes(val) + allowedExpressions: (val) => ["randomUsername", "unixTimestamp", "identity.name"].includes(val) }) ); diff --git a/backend/src/ee/routes/v1/index.ts b/backend/src/ee/routes/v1/index.ts index 0b8c78586..8f3b69dfa 100644 --- a/backend/src/ee/routes/v1/index.ts +++ b/backend/src/ee/routes/v1/index.ts @@ -6,6 +6,7 @@ import { registerAssumePrivilegeRouter } from "./assume-privilege-router"; import { registerAuditLogStreamRouter } from "./audit-log-stream-router"; import { registerCaCrlRouter } from "./certificate-authority-crl-router"; import { registerDynamicSecretLeaseRouter } from "./dynamic-secret-lease-router"; +import { registerKubernetesDynamicSecretLeaseRouter } from "./dynamic-secret-lease-routers/kubernetes-lease-router"; import { registerDynamicSecretRouter } from "./dynamic-secret-router"; import { registerExternalKmsRouter } from "./external-kms-router"; import { registerGatewayRouter } from "./gateway-router"; @@ -18,6 +19,7 @@ import { registerLdapRouter } from "./ldap-router"; import { registerLicenseRouter } from "./license-router"; import { registerOidcRouter } from "./oidc-router"; import { registerOrgRoleRouter } from "./org-role-router"; +import { registerPITRouter } from "./pit-router"; import { registerProjectRoleRouter } from "./project-role-router"; import { registerProjectRouter } from "./project-router"; import { registerRateLimitRouter } from "./rate-limit-router"; @@ -53,6 +55,7 @@ export const registerV1EERoutes = async (server: FastifyZodProvider) => { { prefix: "/workspace" } ); await server.register(registerSnapshotRouter, { prefix: "/secret-snapshot" }); + await server.register(registerPITRouter, { prefix: "/pit" }); await server.register(registerSecretApprovalPolicyRouter, { prefix: "/secret-approvals" }); await server.register(registerSecretApprovalRequestRouter, { prefix: "/secret-approval-requests" @@ -69,6 +72,7 @@ export const registerV1EERoutes = async (server: FastifyZodProvider) => { async (dynamicSecretRouter) => { await dynamicSecretRouter.register(registerDynamicSecretRouter); await dynamicSecretRouter.register(registerDynamicSecretLeaseRouter, { prefix: "/leases" }); + await dynamicSecretRouter.register(registerKubernetesDynamicSecretLeaseRouter, { prefix: "/leases/kubernetes" }); }, { prefix: "/dynamic-secrets" } ); diff --git a/backend/src/ee/routes/v1/pit-router.ts b/backend/src/ee/routes/v1/pit-router.ts new file mode 100644 index 000000000..f993e31d7 --- /dev/null +++ b/backend/src/ee/routes/v1/pit-router.ts @@ -0,0 +1,416 @@ +/* eslint-disable @typescript-eslint/no-base-to-string */ +import { z } from "zod"; + +import { EventType } from "@app/ee/services/audit-log/audit-log-types"; +import { removeTrailingSlash } from "@app/lib/fn"; +import { readLimit } from "@app/server/config/rateLimiter"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { booleanSchema } from "@app/server/routes/sanitizedSchemas"; +import { AuthMode } from "@app/services/auth/auth-type"; +import { commitChangesResponseSchema, resourceChangeSchema } from "@app/services/folder-commit/folder-commit-schemas"; + +const commitHistoryItemSchema = z.object({ + id: z.string(), + folderId: z.string(), + actorType: z.string(), + actorMetadata: z.unknown().optional(), + message: z.string().optional().nullable(), + commitId: z.string(), + createdAt: z.string().or(z.date()), + envId: z.string() +}); + +const folderStateSchema = z.array( + z.object({ + type: z.string(), + id: z.string(), + versionId: z.string(), + secretKey: z.string().optional(), + secretVersion: z.number().optional(), + folderName: z.string().optional(), + folderVersion: z.number().optional() + }) +); + +export const registerPITRouter = async (server: FastifyZodProvider) => { + // Get commits count for a folder + server.route({ + method: "GET", + url: "/commits/count", + config: { + rateLimit: readLimit + }, + schema: { + querystring: z.object({ + environment: z.string().trim(), + path: z.string().trim().default("/").transform(removeTrailingSlash), + projectId: z.string().trim() + }), + response: { + 200: z.object({ + count: z.number(), + folderId: z.string() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.getCommitsCount({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.query.projectId, + environment: req.query.environment, + path: req.query.path + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.query.projectId, + event: { + type: EventType.GET_PROJECT_PIT_COMMIT_COUNT, + metadata: { + environment: req.query.environment, + path: req.query.path, + commitCount: result.count.toString() + } + } + }); + + return result; + } + }); + + // Get all commits for a folder + server.route({ + method: "GET", + url: "/commits", + config: { + rateLimit: readLimit + }, + schema: { + querystring: z.object({ + environment: z.string().trim(), + path: z.string().trim().default("/").transform(removeTrailingSlash), + projectId: z.string().trim(), + offset: z.coerce.number().min(0).default(0), + limit: z.coerce.number().min(1).max(100).default(20), + search: z.string().trim().optional(), + sort: z.enum(["asc", "desc"]).default("desc") + }), + response: { + 200: z.object({ + commits: commitHistoryItemSchema.array(), + total: z.number(), + hasMore: z.boolean() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.getCommitsForFolder({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.query.projectId, + environment: req.query.environment, + path: req.query.path, + offset: req.query.offset, + limit: req.query.limit, + search: req.query.search, + sort: req.query.sort + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.query.projectId, + event: { + type: EventType.GET_PROJECT_PIT_COMMITS, + metadata: { + environment: req.query.environment, + path: req.query.path, + commitCount: result.commits.length.toString(), + offset: req.query.offset.toString(), + limit: req.query.limit.toString(), + search: req.query.search, + sort: req.query.sort + } + } + }); + + return result; + } + }); + + // Get commit changes for a specific commit + server.route({ + method: "GET", + url: "/commits/:commitId/changes", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + commitId: z.string().trim() + }), + querystring: z.object({ + projectId: z.string().trim() + }), + response: { + 200: commitChangesResponseSchema + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.getCommitChanges({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.query.projectId, + commitId: req.params.commitId + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.query.projectId, + event: { + type: EventType.GET_PROJECT_PIT_COMMIT_CHANGES, + metadata: { + commitId: req.params.commitId, + changesCount: (result.changes.changes?.length || 0).toString() + } + } + }); + + return result; + } + }); + + // Retrieve rollback changes for a commit + server.route({ + method: "GET", + url: "/commits/:commitId/compare", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + commitId: z.string().trim() + }), + querystring: z.object({ + folderId: z.string().trim(), + environment: z.string().trim(), + deepRollback: booleanSchema.default(false), + secretPath: z.string().trim().default("/").transform(removeTrailingSlash), + projectId: z.string().trim() + }), + response: { + 200: z.array( + z.object({ + folderId: z.string(), + folderName: z.string(), + folderPath: z.string().optional(), + changes: z.array(resourceChangeSchema) + }) + ) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.compareCommitChanges({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.query.projectId, + commitId: req.params.commitId, + folderId: req.query.folderId, + environment: req.query.environment, + deepRollback: req.query.deepRollback, + secretPath: req.query.secretPath + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.query.projectId, + event: { + type: EventType.PIT_COMPARE_FOLDER_STATES, + metadata: { + targetCommitId: req.params.commitId, + folderId: req.query.folderId, + deepRollback: req.query.deepRollback, + diffsCount: result.length.toString(), + environment: req.query.environment, + folderPath: req.query.secretPath + } + } + }); + + return result; + } + }); + + // Rollback to a previous commit + server.route({ + method: "POST", + url: "/commits/:commitId/rollback", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + commitId: z.string().trim() + }), + body: z.object({ + folderId: z.string().trim(), + deepRollback: z.boolean().default(false), + message: z.string().max(256).trim().optional(), + environment: z.string().trim(), + projectId: z.string().trim() + }), + response: { + 200: z.object({ + success: z.boolean(), + secretChangesCount: z.number().optional(), + folderChangesCount: z.number().optional(), + totalChanges: z.number().optional() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.rollbackToCommit({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.body.projectId, + commitId: req.params.commitId, + folderId: req.body.folderId, + deepRollback: req.body.deepRollback, + message: req.body.message, + environment: req.body.environment + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.body.projectId, + event: { + type: EventType.PIT_ROLLBACK_COMMIT, + metadata: { + targetCommitId: req.params.commitId, + environment: req.body.environment, + folderId: req.body.folderId, + deepRollback: req.body.deepRollback, + message: req.body.message || "Rollback to previous commit", + totalChanges: result.totalChanges?.toString() || "0" + } + } + }); + + return result; + } + }); + + // Revert commit + server.route({ + method: "POST", + url: "/commits/:commitId/revert", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + commitId: z.string().trim() + }), + body: z.object({ + projectId: z.string().trim() + }), + response: { + 200: z.object({ + success: z.boolean(), + message: z.string(), + originalCommitId: z.string(), + revertCommitId: z.string().optional(), + changesReverted: z.number().optional() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.revertCommit({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.body.projectId, + commitId: req.params.commitId + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.body.projectId, + event: { + type: EventType.PIT_REVERT_COMMIT, + metadata: { + commitId: req.params.commitId, + revertCommitId: result.revertCommitId, + changesReverted: result.changesReverted?.toString() + } + } + }); + + return result; + } + }); + + // Folder state at commit + server.route({ + method: "GET", + url: "/commits/:commitId", + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + commitId: z.string().trim() + }), + querystring: z.object({ + folderId: z.string().trim(), + projectId: z.string().trim() + }), + response: { + 200: folderStateSchema + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const result = await server.services.pit.getFolderStateAtCommit({ + actor: req.permission?.type, + actorId: req.permission?.id, + actorOrgId: req.permission?.orgId, + actorAuthMethod: req.permission?.authMethod, + projectId: req.query.projectId, + commitId: req.params.commitId + }); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.query.projectId, + event: { + type: EventType.PIT_GET_FOLDER_STATE, + metadata: { + commitId: req.params.commitId, + folderId: req.query.folderId, + resourceCount: result.length.toString() + } + } + }); + + return result; + } + }); +}; diff --git a/backend/src/ee/routes/v1/snapshot-router.ts b/backend/src/ee/routes/v1/snapshot-router.ts index 3ee80adce..c14d97192 100644 --- a/backend/src/ee/routes/v1/snapshot-router.ts +++ b/backend/src/ee/routes/v1/snapshot-router.ts @@ -65,9 +65,10 @@ export const registerSnapshotRouter = async (server: FastifyZodProvider) => { rateLimit: writeLimit }, schema: { - hide: false, + hide: true, + deprecated: true, tags: [ApiDocsTags.Projects], - description: "Roll back project secrets to those captured in a secret snapshot version.", + description: "(Deprecated) Roll back project secrets to those captured in a secret snapshot version.", security: [ { bearerAuth: [] @@ -84,6 +85,10 @@ export const registerSnapshotRouter = async (server: FastifyZodProvider) => { }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { + throw new Error( + "This endpoint is deprecated. Please use the new PIT recovery system. More information is available at: https://infisical.com/docs/documentation/platform/pit-recovery." + ); + const secretSnapshot = await server.services.snapshot.rollbackSnapshot({ actor: req.permission.type, actorId: req.permission.id, diff --git a/backend/src/ee/services/audit-log/audit-log-types.ts b/backend/src/ee/services/audit-log/audit-log-types.ts index cfc01741f..1b07982ca 100644 --- a/backend/src/ee/services/audit-log/audit-log-types.ts +++ b/backend/src/ee/services/audit-log/audit-log-types.ts @@ -44,6 +44,7 @@ import { TSecretSyncRaw, TUpdateSecretSyncDTO } from "@app/services/secret-sync/secret-sync-types"; +import { TWebhookPayloads } from "@app/services/webhook/webhook-types"; import { WorkflowIntegration } from "@app/services/workflow-integration/workflow-integration-types"; import { KmipPermission } from "../kmip/kmip-enum"; @@ -206,6 +207,7 @@ export enum EventType { CREATE_WEBHOOK = "create-webhook", UPDATE_WEBHOOK_STATUS = "update-webhook-status", DELETE_WEBHOOK = "delete-webhook", + WEBHOOK_TRIGGERED = "webhook-triggered", GET_SECRET_IMPORTS = "get-secret-imports", GET_SECRET_IMPORT = "get-secret-import", CREATE_SECRET_IMPORT = "create-secret-import", @@ -393,6 +395,13 @@ export enum EventType { PROJECT_ASSUME_PRIVILEGE_SESSION_START = "project-assume-privileges-session-start", PROJECT_ASSUME_PRIVILEGE_SESSION_END = "project-assume-privileges-session-end", + GET_PROJECT_PIT_COMMITS = "get-project-pit-commits", + GET_PROJECT_PIT_COMMIT_CHANGES = "get-project-pit-commit-changes", + GET_PROJECT_PIT_COMMIT_COUNT = "get-project-pit-commit-count", + PIT_ROLLBACK_COMMIT = "pit-rollback-commit", + PIT_REVERT_COMMIT = "pit-revert-commit", + PIT_GET_FOLDER_STATE = "pit-get-folder-state", + PIT_COMPARE_FOLDER_STATES = "pit-compare-folder-states", SECRET_SCANNING_DATA_SOURCE_LIST = "secret-scanning-data-source-list", SECRET_SCANNING_DATA_SOURCE_CREATE = "secret-scanning-data-source-create", SECRET_SCANNING_DATA_SOURCE_UPDATE = "secret-scanning-data-source-update", @@ -1440,6 +1449,14 @@ interface DeleteWebhookEvent { }; } +export interface WebhookTriggeredEvent { + type: EventType.WEBHOOK_TRIGGERED; + metadata: { + webhookId: string; + status: string; + } & TWebhookPayloads; +} + interface GetSecretImportsEvent { type: EventType.GET_SECRET_IMPORTS; metadata: { @@ -2979,6 +2996,78 @@ interface MicrosoftTeamsWorkflowIntegrationUpdateEvent { }; } +interface GetProjectPitCommitsEvent { + type: EventType.GET_PROJECT_PIT_COMMITS; + metadata: { + commitCount: string; + environment: string; + path: string; + offset: string; + limit: string; + search?: string; + sort: string; + }; +} + +interface GetProjectPitCommitChangesEvent { + type: EventType.GET_PROJECT_PIT_COMMIT_CHANGES; + metadata: { + changesCount: string; + commitId: string; + }; +} + +interface GetProjectPitCommitCountEvent { + type: EventType.GET_PROJECT_PIT_COMMIT_COUNT; + metadata: { + environment: string; + path: string; + commitCount: string; + }; +} + +interface PitRollbackCommitEvent { + type: EventType.PIT_ROLLBACK_COMMIT; + metadata: { + targetCommitId: string; + folderId: string; + deepRollback: boolean; + message: string; + totalChanges: string; + environment: string; + }; +} + +interface PitRevertCommitEvent { + type: EventType.PIT_REVERT_COMMIT; + metadata: { + commitId: string; + revertCommitId?: string; + changesReverted?: string; + }; +} + +interface PitGetFolderStateEvent { + type: EventType.PIT_GET_FOLDER_STATE; + metadata: { + commitId: string; + folderId: string; + resourceCount: string; + }; +} + +interface PitCompareFolderStatesEvent { + type: EventType.PIT_COMPARE_FOLDER_STATES; + metadata: { + targetCommitId: string; + folderId: string; + deepRollback: boolean; + diffsCount: string; + environment: string; + folderPath: string; + }; +} + interface SecretScanningDataSourceListEvent { type: EventType.SECRET_SCANNING_DATA_SOURCE_LIST; metadata: { @@ -3221,6 +3310,7 @@ export type Event = | CreateWebhookEvent | UpdateWebhookStatusEvent | DeleteWebhookEvent + | WebhookTriggeredEvent | GetSecretImportsEvent | GetSecretImportEvent | CreateSecretImportEvent @@ -3397,6 +3487,13 @@ export type Event = | MicrosoftTeamsWorkflowIntegrationGetEvent | MicrosoftTeamsWorkflowIntegrationListEvent | MicrosoftTeamsWorkflowIntegrationUpdateEvent + | GetProjectPitCommitsEvent + | GetProjectPitCommitChangesEvent + | PitRollbackCommitEvent + | GetProjectPitCommitCountEvent + | PitRevertCommitEvent + | PitCompareFolderStatesEvent + | PitGetFolderStateEvent | SecretScanningDataSourceListEvent | SecretScanningDataSourceGetEvent | SecretScanningDataSourceCreateEvent diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts index fa1a80ac3..497e94311 100644 --- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts +++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts @@ -10,6 +10,7 @@ import { TDynamicSecretDALFactory } from "../dynamic-secret/dynamic-secret-dal"; import { DynamicSecretStatus } from "../dynamic-secret/dynamic-secret-types"; import { DynamicSecretProviders, TDynamicProviderFns } from "../dynamic-secret/providers/models"; import { TDynamicSecretLeaseDALFactory } from "./dynamic-secret-lease-dal"; +import { TDynamicSecretLeaseConfig } from "./dynamic-secret-lease-types"; type TDynamicSecretLeaseQueueServiceFactoryDep = { queueService: TQueueServiceFactory; @@ -99,7 +100,9 @@ export const dynamicSecretLeaseQueueServiceFactory = ({ secretManagerDecryptor({ cipherTextBlob: dynamicSecretCfg.encryptedInput }).toString() ) as object; - await selectedProvider.revoke(decryptedStoredInput, dynamicSecretLease.externalEntityId); + await selectedProvider.revoke(decryptedStoredInput, dynamicSecretLease.externalEntityId, { + projectId: folder.projectId + }); await dynamicSecretLeaseDAL.deleteById(dynamicSecretLease.id); return; } @@ -132,8 +135,15 @@ export const dynamicSecretLeaseQueueServiceFactory = ({ await Promise.all(dynamicSecretLeases.map(({ id }) => unsetLeaseRevocation(id))); await Promise.all( - dynamicSecretLeases.map(({ externalEntityId }) => - selectedProvider.revoke(decryptedStoredInput, externalEntityId) + dynamicSecretLeases.map(({ externalEntityId, config }) => + selectedProvider.revoke( + decryptedStoredInput, + externalEntityId, + { + projectId: folder.projectId + }, + config as TDynamicSecretLeaseConfig + ) ) ); } diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts index f3f3f3acd..4b72ff7e9 100644 --- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts +++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts @@ -1,4 +1,5 @@ import { ForbiddenError, subject } from "@casl/ability"; +import RE2 from "re2"; import { ActionProjectType } from "@app/db/schemas"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; @@ -11,10 +12,13 @@ import { getConfig } from "@app/lib/config/env"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { logger } from "@app/lib/logger"; import { ms } from "@app/lib/ms"; +import { ActorType } from "@app/services/auth/auth-type"; +import { TIdentityDALFactory } from "@app/services/identity/identity-dal"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectDALFactory } from "@app/services/project/project-dal"; import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { TUserDALFactory } from "@app/services/user/user-dal"; import { TDynamicSecretDALFactory } from "../dynamic-secret/dynamic-secret-dal"; import { DynamicSecretProviders, TDynamicProviderFns } from "../dynamic-secret/providers/models"; @@ -25,6 +29,7 @@ import { TCreateDynamicSecretLeaseDTO, TDeleteDynamicSecretLeaseDTO, TDetailsDynamicSecretLeaseDTO, + TDynamicSecretLeaseConfig, TListDynamicSecretLeasesDTO, TRenewDynamicSecretLeaseDTO } from "./dynamic-secret-lease-types"; @@ -39,6 +44,8 @@ type TDynamicSecretLeaseServiceFactoryDep = { permissionService: Pick; projectDAL: Pick; kmsService: Pick; + userDAL: Pick; + identityDAL: TIdentityDALFactory; }; export type TDynamicSecretLeaseServiceFactory = ReturnType; @@ -52,8 +59,16 @@ export const dynamicSecretLeaseServiceFactory = ({ dynamicSecretQueueService, projectDAL, licenseService, - kmsService + kmsService, + userDAL, + identityDAL }: TDynamicSecretLeaseServiceFactoryDep) => { + const extractEmailUsername = (email: string) => { + const regex = new RE2(/^([^@]+)/); + const match = email.match(regex); + return match ? match[1] : email; + }; + const create = async ({ environmentSlug, path, @@ -63,7 +78,8 @@ export const dynamicSecretLeaseServiceFactory = ({ actorId, actorOrgId, actorAuthMethod, - ttl + ttl, + config }: TCreateDynamicSecretLeaseDTO) => { const appCfg = getConfig(); const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); @@ -132,10 +148,25 @@ export const dynamicSecretLeaseServiceFactory = ({ let result; try { + const identity: { name: string } = { name: "" }; + if (actor === ActorType.USER) { + const user = await userDAL.findById(actorId); + if (user) { + identity.name = extractEmailUsername(user.username); + } + } else if (actor === ActorType.Machine) { + const machineIdentity = await identityDAL.findById(actorId); + if (machineIdentity) { + identity.name = machineIdentity.name; + } + } result = await selectedProvider.create({ inputs: decryptedStoredInput, expireAt: expireAt.getTime(), - usernameTemplate: dynamicSecretCfg.usernameTemplate + usernameTemplate: dynamicSecretCfg.usernameTemplate, + identity, + metadata: { projectId }, + config }); } catch (error: unknown) { if (error && typeof error === "object" && error !== null && "sqlMessage" in error) { @@ -149,8 +180,10 @@ export const dynamicSecretLeaseServiceFactory = ({ expireAt, version: 1, dynamicSecretId: dynamicSecretCfg.id, - externalEntityId: entityId + externalEntityId: entityId, + config }); + await dynamicSecretQueueService.setLeaseRevocation(dynamicSecretLease.id, Number(expireAt) - Number(new Date())); return { lease: dynamicSecretLease, dynamicSecret: dynamicSecretCfg, data }; }; @@ -237,7 +270,8 @@ export const dynamicSecretLeaseServiceFactory = ({ const { entityId } = await selectedProvider.renew( decryptedStoredInput, dynamicSecretLease.externalEntityId, - expireAt.getTime() + expireAt.getTime(), + { projectId } ); await dynamicSecretQueueService.unsetLeaseRevocation(dynamicSecretLease.id); @@ -313,7 +347,12 @@ export const dynamicSecretLeaseServiceFactory = ({ ) as object; const revokeResponse = await selectedProvider - .revoke(decryptedStoredInput, dynamicSecretLease.externalEntityId) + .revoke( + decryptedStoredInput, + dynamicSecretLease.externalEntityId, + { projectId }, + dynamicSecretLease.config as TDynamicSecretLeaseConfig + ) .catch(async (err) => { // only propogate this error if forced is false if (!isForced) return { error: err as Error }; diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-types.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-types.ts index bf182b349..f6d9f6297 100644 --- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-types.ts +++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-types.ts @@ -10,6 +10,7 @@ export type TCreateDynamicSecretLeaseDTO = { environmentSlug: string; ttl?: string; projectSlug: string; + config?: TDynamicSecretLeaseConfig; } & Omit; export type TDetailsDynamicSecretLeaseDTO = { @@ -41,3 +42,9 @@ export type TRenewDynamicSecretLeaseDTO = { ttl?: string; projectSlug: string; } & Omit; + +export type TDynamicSecretKubernetesLeaseConfig = { + namespace?: string; +}; + +export type TDynamicSecretLeaseConfig = TDynamicSecretKubernetesLeaseConfig; diff --git a/backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts b/backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts index 16ac10716..b502bf9f3 100644 --- a/backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts +++ b/backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts @@ -116,7 +116,7 @@ export const dynamicSecretServiceFactory = ({ throw new BadRequestError({ message: "Provided dynamic secret already exist under the folder" }); const selectedProvider = dynamicSecretProviders[provider.type]; - const inputs = await selectedProvider.validateProviderInputs(provider.inputs); + const inputs = await selectedProvider.validateProviderInputs(provider.inputs, { projectId }); let selectedGatewayId: string | null = null; if (inputs && typeof inputs === "object" && "gatewayId" in inputs && inputs.gatewayId) { @@ -146,7 +146,7 @@ export const dynamicSecretServiceFactory = ({ selectedGatewayId = gateway.id; } - const isConnected = await selectedProvider.validateConnection(provider.inputs); + const isConnected = await selectedProvider.validateConnection(provider.inputs, { projectId }); if (!isConnected) throw new BadRequestError({ message: "Provider connection failed" }); const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({ @@ -272,7 +272,7 @@ export const dynamicSecretServiceFactory = ({ secretManagerDecryptor({ cipherTextBlob: dynamicSecretCfg.encryptedInput }).toString() ) as object; const newInput = { ...decryptedStoredInput, ...(inputs || {}) }; - const updatedInput = await selectedProvider.validateProviderInputs(newInput); + const updatedInput = await selectedProvider.validateProviderInputs(newInput, { projectId }); let selectedGatewayId: string | null = null; if (updatedInput && typeof updatedInput === "object" && "gatewayId" in updatedInput && updatedInput?.gatewayId) { @@ -301,7 +301,7 @@ export const dynamicSecretServiceFactory = ({ selectedGatewayId = gateway.id; } - const isConnected = await selectedProvider.validateConnection(newInput); + const isConnected = await selectedProvider.validateConnection(newInput, { projectId }); if (!isConnected) throw new BadRequestError({ message: "Provider connection failed" }); const updatedDynamicCfg = await dynamicSecretDAL.transaction(async (tx) => { @@ -472,7 +472,9 @@ export const dynamicSecretServiceFactory = ({ secretManagerDecryptor({ cipherTextBlob: dynamicSecretCfg.encryptedInput }).toString() ) as object; const selectedProvider = dynamicSecretProviders[dynamicSecretCfg.type as DynamicSecretProviders]; - const providerInputs = (await selectedProvider.validateProviderInputs(decryptedStoredInput)) as object; + const providerInputs = (await selectedProvider.validateProviderInputs(decryptedStoredInput, { + projectId + })) as object; return { ...dynamicSecretCfg, inputs: providerInputs }; }; diff --git a/backend/src/ee/services/dynamic-secret/providers/aws-elasticache.ts b/backend/src/ee/services/dynamic-secret/providers/aws-elasticache.ts index 56fa110d1..89371f1bd 100644 --- a/backend/src/ee/services/dynamic-secret/providers/aws-elasticache.ts +++ b/backend/src/ee/services/dynamic-secret/providers/aws-elasticache.ts @@ -16,6 +16,7 @@ import { BadRequestError } from "@app/lib/errors"; import { validateHandlebarTemplate } from "@app/lib/template/validate-handlebars"; import { DynamicSecretAwsElastiCacheSchema, TDynamicProviderFns } from "./models"; +import { compileUsernameTemplate } from "./templateUtils"; const CreateElastiCacheUserSchema = z.object({ UserId: z.string().trim().min(1), @@ -132,14 +133,14 @@ const generatePassword = () => { return customAlphabet(charset, 64)(); }; -const generateUsername = (usernameTemplate?: string | null) => { +const generateUsername = (usernameTemplate?: string | null, identity?: { name: string }) => { const charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-"; const randomUsername = `inf-${customAlphabet(charset, 32)()}`; if (!usernameTemplate) return randomUsername; - - return handlebars.compile(usernameTemplate)({ + return compileUsernameTemplate({ + usernameTemplate, randomUsername, - unixTimestamp: Math.floor(Date.now() / 100) + identity }); }; @@ -174,14 +175,21 @@ export const AwsElastiCacheDatabaseProvider = (): TDynamicProviderFns => { return true; }; - const create = async (data: { inputs: unknown; expireAt: number; usernameTemplate?: string | null }) => { - const { inputs, expireAt, usernameTemplate } = data; + const create = async (data: { + inputs: unknown; + expireAt: number; + usernameTemplate?: string | null; + identity?: { + name: string; + }; + }) => { + const { inputs, expireAt, usernameTemplate, identity } = data; const providerInputs = await validateProviderInputs(inputs); if (!(await validateConnection(providerInputs))) { throw new BadRequestError({ message: "Failed to establish connection" }); } - const leaseUsername = generateUsername(usernameTemplate); + const leaseUsername = generateUsername(usernameTemplate, identity); const leasePassword = generatePassword(); const leaseExpiration = new Date(expireAt).toISOString(); diff --git a/backend/src/ee/services/dynamic-secret/providers/aws-iam.ts b/backend/src/ee/services/dynamic-secret/providers/aws-iam.ts index 9d8e10f60..f7383d4ac 100644 --- a/backend/src/ee/services/dynamic-secret/providers/aws-iam.ts +++ b/backend/src/ee/services/dynamic-secret/providers/aws-iam.ts @@ -16,21 +16,25 @@ import { PutUserPolicyCommand, RemoveUserFromGroupCommand } from "@aws-sdk/client-iam"; -import handlebars from "handlebars"; +import { AssumeRoleCommand, STSClient } from "@aws-sdk/client-sts"; +import { randomUUID } from "crypto"; import { z } from "zod"; +import { getConfig } from "@app/lib/config/env"; import { BadRequestError } from "@app/lib/errors"; import { alphaNumericNanoId } from "@app/lib/nanoid"; -import { DynamicSecretAwsIamSchema, TDynamicProviderFns } from "./models"; +import { AwsIamAuthType, DynamicSecretAwsIamSchema, TDynamicProviderFns } from "./models"; +import { compileUsernameTemplate } from "./templateUtils"; -const generateUsername = (usernameTemplate?: string | null) => { +const generateUsername = (usernameTemplate?: string | null, identity?: { name: string }) => { const randomUsername = alphaNumericNanoId(32); if (!usernameTemplate) return randomUsername; - return handlebars.compile(usernameTemplate)({ + return compileUsernameTemplate({ + usernameTemplate, randomUsername, - unixTimestamp: Math.floor(Date.now() / 100) + identity }); }; @@ -40,7 +44,43 @@ export const AwsIamProvider = (): TDynamicProviderFns => { return providerInputs; }; - const $getClient = async (providerInputs: z.infer) => { + const $getClient = async (providerInputs: z.infer, projectId: string) => { + const appCfg = getConfig(); + if (providerInputs.method === AwsIamAuthType.AssumeRole) { + const stsClient = new STSClient({ + region: providerInputs.region, + credentials: + appCfg.DYNAMIC_SECRET_AWS_ACCESS_KEY_ID && appCfg.DYNAMIC_SECRET_AWS_SECRET_ACCESS_KEY + ? { + accessKeyId: appCfg.DYNAMIC_SECRET_AWS_ACCESS_KEY_ID, + secretAccessKey: appCfg.DYNAMIC_SECRET_AWS_SECRET_ACCESS_KEY + } + : undefined // if hosting on AWS + }); + + const command = new AssumeRoleCommand({ + RoleArn: providerInputs.roleArn, + RoleSessionName: `infisical-dynamic-secret-${randomUUID()}`, + DurationSeconds: 900, // 15 mins + ExternalId: projectId + }); + + const assumeRes = await stsClient.send(command); + + if (!assumeRes.Credentials?.AccessKeyId || !assumeRes.Credentials?.SecretAccessKey) { + throw new BadRequestError({ message: "Failed to assume role - verify credentials and role configuration" }); + } + const client = new IAMClient({ + region: providerInputs.region, + credentials: { + accessKeyId: assumeRes.Credentials?.AccessKeyId, + secretAccessKey: assumeRes.Credentials?.SecretAccessKey, + sessionToken: assumeRes.Credentials?.SessionToken + } + }); + return client; + } + const client = new IAMClient({ region: providerInputs.region, credentials: { @@ -52,21 +92,41 @@ export const AwsIamProvider = (): TDynamicProviderFns => { return client; }; - const validateConnection = async (inputs: unknown) => { + const validateConnection = async (inputs: unknown, { projectId }: { projectId: string }) => { const providerInputs = await validateProviderInputs(inputs); - const client = await $getClient(providerInputs); - - const isConnected = await client.send(new GetUserCommand({})).then(() => true); + const client = await $getClient(providerInputs, projectId); + const isConnected = await client + .send(new GetUserCommand({})) + .then(() => true) + .catch((err) => { + const message = (err as Error)?.message; + if ( + providerInputs.method === AwsIamAuthType.AssumeRole && + // assume role will throw an error asking to provider username, but if so this has access in aws correctly + message.includes("Must specify userName when calling with non-User credentials") + ) { + return true; + } + throw err; + }); return isConnected; }; - const create = async (data: { inputs: unknown; expireAt: number; usernameTemplate?: string | null }) => { - const { inputs, usernameTemplate } = data; + const create = async (data: { + inputs: unknown; + expireAt: number; + usernameTemplate?: string | null; + identity?: { + name: string; + }; + metadata: { projectId: string }; + }) => { + const { inputs, usernameTemplate, metadata, identity } = data; const providerInputs = await validateProviderInputs(inputs); - const client = await $getClient(providerInputs); + const client = await $getClient(providerInputs, metadata.projectId); - const username = generateUsername(usernameTemplate); + const username = generateUsername(usernameTemplate, identity); const { policyArns, userGroups, policyDocument, awsPath, permissionBoundaryPolicyArn } = providerInputs; const createUserRes = await client.send( new CreateUserCommand({ @@ -76,6 +136,7 @@ export const AwsIamProvider = (): TDynamicProviderFns => { UserName: username }) ); + if (!createUserRes.User) throw new BadRequestError({ message: "Failed to create AWS IAM User" }); if (userGroups) { await Promise.all( @@ -125,9 +186,9 @@ export const AwsIamProvider = (): TDynamicProviderFns => { }; }; - const revoke = async (inputs: unknown, entityId: string) => { + const revoke = async (inputs: unknown, entityId: string, metadata: { projectId: string }) => { const providerInputs = await validateProviderInputs(inputs); - const client = await $getClient(providerInputs); + const client = await $getClient(providerInputs, metadata.projectId); const username = entityId; diff --git a/backend/src/ee/services/dynamic-secret/providers/cassandra.ts b/backend/src/ee/services/dynamic-secret/providers/cassandra.ts index fce23b56f..b939dcad6 100644 --- a/backend/src/ee/services/dynamic-secret/providers/cassandra.ts +++ b/backend/src/ee/services/dynamic-secret/providers/cassandra.ts @@ -8,19 +8,20 @@ import { validateHandlebarTemplate } from "@app/lib/template/validate-handlebars import { verifyHostInputValidity } from "../dynamic-secret-fns"; import { DynamicSecretCassandraSchema, TDynamicProviderFns } from "./models"; +import { compileUsernameTemplate } from "./templateUtils"; const generatePassword = (size = 48) => { const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.~!*"; return customAlphabet(charset, 48)(size); }; -const generateUsername = (usernameTemplate?: string | null) => { +const generateUsername = (usernameTemplate?: string | null, identity?: { name: string }) => { const randomUsername = alphaNumericNanoId(32); // Username must start with an ascii letter, so we prepend the username with "inf-" if (!usernameTemplate) return randomUsername; - - return handlebars.compile(usernameTemplate)({ + return compileUsernameTemplate({ + usernameTemplate, randomUsername, - unixTimestamp: Math.floor(Date.now() / 100) + identity }); }; @@ -75,12 +76,17 @@ export const CassandraProvider = (): TDynamicProviderFns => { return isConnected; }; - const create = async (data: { inputs: unknown; expireAt: number; usernameTemplate?: string | null }) => { - const { inputs, expireAt, usernameTemplate } = data; + const create = async (data: { + inputs: unknown; + expireAt: number; + usernameTemplate?: string | null; + identity?: { name: string }; + }) => { + const { inputs, expireAt, usernameTemplate, identity } = data; const providerInputs = await validateProviderInputs(inputs); const client = await $getClient(providerInputs); - const username = generateUsername(usernameTemplate); + const username = generateUsername(usernameTemplate, identity); const password = generatePassword(); const { keyspace } = providerInputs; const expiration = new Date(expireAt).toISOString(); diff --git a/backend/src/ee/services/dynamic-secret/providers/elastic-search.ts b/backend/src/ee/services/dynamic-secret/providers/elastic-search.ts index 066822827..32d21ee76 100644 --- a/backend/src/ee/services/dynamic-secret/providers/elastic-search.ts +++ b/backend/src/ee/services/dynamic-secret/providers/elastic-search.ts @@ -1,5 +1,4 @@ import { Client as ElasticSearchClient } from "@elastic/elasticsearch"; -import handlebars from "handlebars"; import { customAlphabet } from "nanoid"; import { z } from "zod"; @@ -7,19 +6,20 @@ import { alphaNumericNanoId } from "@app/lib/nanoid"; import { verifyHostInputValidity } from "../dynamic-secret-fns"; import { DynamicSecretElasticSearchSchema, ElasticSearchAuthTypes, TDynamicProviderFns } from "./models"; +import { compileUsernameTemplate } from "./templateUtils"; const generatePassword = () => { const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.~!*"; return customAlphabet(charset, 64)(); }; -const generateUsername = (usernameTemplate?: string | null) => { +const generateUsername = (usernameTemplate?: string | null, identity?: { name: string }) => { const randomUsername = alphaNumericNanoId(32); // Username must start with an ascii letter, so we prepend the username with "inf-" if (!usernameTemplate) return randomUsername; - - return handlebars.compile(usernameTemplate)({ + return compileUsernameTemplate({ + usernameTemplate, randomUsername, - unixTimestamp: Math.floor(Date.now() / 100) + identity }); }; @@ -71,12 +71,12 @@ export const ElasticSearchProvider = (): TDynamicProviderFns => { return infoResponse; }; - const create = async (data: { inputs: unknown; usernameTemplate?: string | null }) => { - const { inputs, usernameTemplate } = data; + const create = async (data: { inputs: unknown; usernameTemplate?: string | null; identity?: { name: string } }) => { + const { inputs, usernameTemplate, identity } = data; const providerInputs = await validateProviderInputs(inputs); const connection = await $getClient(providerInputs); - const username = generateUsername(usernameTemplate); + const username = generateUsername(usernameTemplate, identity); const password = generatePassword(); await connection.security.putUser({ diff --git a/backend/src/ee/services/dynamic-secret/providers/kubernetes.ts b/backend/src/ee/services/dynamic-secret/providers/kubernetes.ts index 8a54ba089..45cc06e7c 100644 --- a/backend/src/ee/services/dynamic-secret/providers/kubernetes.ts +++ b/backend/src/ee/services/dynamic-secret/providers/kubernetes.ts @@ -1,24 +1,46 @@ -import axios from "axios"; +import axios, { AxiosError } from "axios"; +import handlebars from "handlebars"; import https from "https"; -import { InternalServerError } from "@app/lib/errors"; -import { withGatewayProxy } from "@app/lib/gateway"; +import { BadRequestError, InternalServerError } from "@app/lib/errors"; +import { GatewayHttpProxyActions, GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway"; +import { alphaNumericNanoId } from "@app/lib/nanoid"; import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator"; import { TKubernetesTokenRequest } from "@app/services/identity-kubernetes-auth/identity-kubernetes-auth-types"; +import { TDynamicSecretKubernetesLeaseConfig } from "../../dynamic-secret-lease/dynamic-secret-lease-types"; import { TGatewayServiceFactory } from "../../gateway/gateway-service"; -import { DynamicSecretKubernetesSchema, TDynamicProviderFns } from "./models"; +import { + DynamicSecretKubernetesSchema, + KubernetesAuthMethod, + KubernetesCredentialType, + KubernetesRoleType, + TDynamicProviderFns +} from "./models"; const EXTERNAL_REQUEST_TIMEOUT = 10 * 1000; +// This value is just a placeholder. When using gateway auth method, the url is irrelevant. +const GATEWAY_AUTH_DEFAULT_URL = "https://kubernetes.default.svc.cluster.local"; + type TKubernetesProviderDTO = { gatewayService: Pick; }; +const generateUsername = (usernameTemplate?: string | null) => { + const randomUsername = `dynamic-secret-sa-${alphaNumericNanoId(10).toLowerCase()}`; + if (!usernameTemplate) return randomUsername; + + return handlebars.compile(usernameTemplate)({ + randomUsername, + unixTimestamp: Math.floor(Date.now() / 100) + }); +}; + export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): TDynamicProviderFns => { const validateProviderInputs = async (inputs: unknown) => { const providerInputs = await DynamicSecretKubernetesSchema.parseAsync(inputs); - if (!providerInputs.gatewayId) { + if (!providerInputs.gatewayId && providerInputs.url) { await blockLocalAndPrivateIpAddresses(providerInputs.url); } @@ -30,19 +52,27 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): gatewayId: string; targetHost: string; targetPort: number; + caCert?: string; + reviewTokenThroughGateway: boolean; + enableSsl: boolean; }, - gatewayCallback: (host: string, port: number) => Promise + gatewayCallback: (host: string, port: number, httpsAgent?: https.Agent) => Promise ): Promise => { const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(inputs.gatewayId); const [relayHost, relayPort] = relayDetails.relayAddress.split(":"); const callbackResult = await withGatewayProxy( - async (port) => { + async (port, httpsAgent) => { // Needs to be https protocol or the kubernetes API server will fail with "Client sent an HTTP request to an HTTPS server" - const res = await gatewayCallback("https://localhost", port); + const res = await gatewayCallback( + inputs.reviewTokenThroughGateway ? "http://localhost" : "https://localhost", + port, + httpsAgent + ); return res; }, { + protocol: inputs.reviewTokenThroughGateway ? GatewayProxyProtocol.Http : GatewayProxyProtocol.Tcp, targetHost: inputs.targetHost, targetPort: inputs.targetPort, relayHost, @@ -53,7 +83,12 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): ca: relayDetails.certChain, cert: relayDetails.certificate, key: relayDetails.privateKey.toString() - } + }, + // we always pass this, because its needed for both tcp and http protocol + httpsAgent: new https.Agent({ + ca: inputs.caCert, + rejectUnauthorized: inputs.enableSsl + }) } ); @@ -63,7 +98,189 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): const validateConnection = async (inputs: unknown) => { const providerInputs = await validateProviderInputs(inputs); - const serviceAccountGetCallback = async (host: string, port: number) => { + const serviceAccountDynamicCallback = async (host: string, port: number, httpsAgent?: https.Agent) => { + if (providerInputs.credentialType !== KubernetesCredentialType.Dynamic) { + throw new Error("invalid callback"); + } + + const baseUrl = port ? `${host}:${port}` : host; + const serviceAccountName = generateUsername(); + const roleBindingName = `${serviceAccountName}-role-binding`; + + const namespaces = providerInputs.namespace.split(",").map((namespace) => namespace.trim()); + + // Test each namespace sequentially instead of in parallel to simplify cleanup + for await (const namespace of namespaces) { + try { + // 1. Create a test service account + await axios.post( + `${baseUrl}/api/v1/namespaces/${namespace}/serviceaccounts`, + { + metadata: { + name: serviceAccountName, + namespace + } + }, + { + headers: { + "Content-Type": "application/json", + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) + }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), + signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), + timeout: EXTERNAL_REQUEST_TIMEOUT + } + ); + + // 2. Create a test role binding + const roleBindingUrl = + providerInputs.roleType === KubernetesRoleType.ClusterRole + ? `${baseUrl}/apis/rbac.authorization.k8s.io/v1/clusterrolebindings` + : `${baseUrl}/apis/rbac.authorization.k8s.io/v1/namespaces/${namespace}/rolebindings`; + + const roleBindingMetadata = { + name: roleBindingName, + ...(providerInputs.roleType !== KubernetesRoleType.ClusterRole && { namespace }) + }; + + await axios.post( + roleBindingUrl, + { + metadata: roleBindingMetadata, + roleRef: { + kind: providerInputs.roleType === KubernetesRoleType.ClusterRole ? "ClusterRole" : "Role", + name: providerInputs.role, + apiGroup: "rbac.authorization.k8s.io" + }, + subjects: [ + { + kind: "ServiceAccount", + name: serviceAccountName, + namespace + } + ] + }, + { + headers: { + "Content-Type": "application/json", + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) + }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), + signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), + timeout: EXTERNAL_REQUEST_TIMEOUT + } + ); + + // 3. Request a token for the test service account + await axios.post( + `${baseUrl}/api/v1/namespaces/${namespace}/serviceaccounts/${serviceAccountName}/token`, + { + spec: { + expirationSeconds: 600, // 10 minutes + ...(providerInputs.audiences?.length ? { audiences: providerInputs.audiences } : {}) + } + }, + { + headers: { + "Content-Type": "application/json", + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) + }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), + signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), + timeout: EXTERNAL_REQUEST_TIMEOUT + } + ); + + // 4. Cleanup: delete role binding and service account + if (providerInputs.roleType === KubernetesRoleType.Role) { + await axios.delete( + `${baseUrl}/apis/rbac.authorization.k8s.io/v1/namespaces/${namespace}/rolebindings/${roleBindingName}`, + { + headers: { + "Content-Type": "application/json", + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) + }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), + signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), + timeout: EXTERNAL_REQUEST_TIMEOUT + } + ); + } else { + await axios.delete(`${baseUrl}/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/${roleBindingName}`, { + headers: { + "Content-Type": "application/json", + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) + }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), + signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), + timeout: EXTERNAL_REQUEST_TIMEOUT + }); + } + + await axios.delete(`${baseUrl}/api/v1/namespaces/${namespace}/serviceaccounts/${serviceAccountName}`, { + headers: { + "Content-Type": "application/json", + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) + }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), + signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), + timeout: EXTERNAL_REQUEST_TIMEOUT + }); + } catch (error) { + const cleanupInfo = `You may need to manually clean up the following resources in namespace "${namespace}": Service Account - ${serviceAccountName}, ${providerInputs.roleType === KubernetesRoleType.Role ? "Role" : "Cluster Role"} Binding - ${roleBindingName}.`; + let mainErrorMessage = "Unknown error"; + if (error instanceof AxiosError) { + mainErrorMessage = (error.response?.data as { message: string })?.message; + } else if (error instanceof Error) { + mainErrorMessage = error.message; + } + + throw new Error(`${mainErrorMessage}. ${cleanupInfo}`); + } + } + }; + + const serviceAccountStaticCallback = async (host: string, port: number, httpsAgent?: https.Agent) => { + if (providerInputs.credentialType !== KubernetesCredentialType.Static) { + throw new Error("invalid callback"); + } + const baseUrl = port ? `${host}:${port}` : host; await axios.get( @@ -71,36 +288,63 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): { headers: { "Content-Type": "application/json", - Authorization: `Bearer ${providerInputs.clusterToken}` + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), - timeout: EXTERNAL_REQUEST_TIMEOUT, - httpsAgent: new https.Agent({ - ca: providerInputs.ca, - rejectUnauthorized: providerInputs.sslEnabled - }) + timeout: EXTERNAL_REQUEST_TIMEOUT } ); }; - const url = new URL(providerInputs.url); + const rawUrl = + providerInputs.authMethod === KubernetesAuthMethod.Gateway ? GATEWAY_AUTH_DEFAULT_URL : providerInputs.url || ""; + const url = new URL(rawUrl); + const k8sGatewayHost = url.hostname; const k8sPort = url.port ? Number(url.port) : 443; + const k8sHost = `${url.protocol}//${url.hostname}`; try { if (providerInputs.gatewayId) { - const k8sHost = url.hostname; - - await $gatewayProxyWrapper( - { - gatewayId: providerInputs.gatewayId, - targetHost: k8sHost, - targetPort: k8sPort - }, - serviceAccountGetCallback - ); + if (providerInputs.authMethod === KubernetesAuthMethod.Gateway) { + await $gatewayProxyWrapper( + { + gatewayId: providerInputs.gatewayId, + targetHost: k8sHost, + targetPort: k8sPort, + enableSsl: providerInputs.sslEnabled, + caCert: providerInputs.ca, + reviewTokenThroughGateway: true + }, + providerInputs.credentialType === KubernetesCredentialType.Static + ? serviceAccountStaticCallback + : serviceAccountDynamicCallback + ); + } else { + await $gatewayProxyWrapper( + { + gatewayId: providerInputs.gatewayId, + targetHost: k8sGatewayHost, + targetPort: k8sPort, + enableSsl: providerInputs.sslEnabled, + caCert: providerInputs.ca, + reviewTokenThroughGateway: false + }, + providerInputs.credentialType === KubernetesCredentialType.Static + ? serviceAccountStaticCallback + : serviceAccountDynamicCallback + ); + } + } else if (providerInputs.credentialType === KubernetesCredentialType.Static) { + await serviceAccountStaticCallback(k8sHost, k8sPort); } else { - const k8sHost = `${url.protocol}//${url.hostname}`; - await serviceAccountGetCallback(k8sHost, k8sPort); + await serviceAccountDynamicCallback(k8sHost, k8sPort); } return true; @@ -116,10 +360,153 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): } }; - const create = async ({ inputs, expireAt }: { inputs: unknown; expireAt: number }) => { + const create = async ({ + inputs, + expireAt, + usernameTemplate, + config + }: { + inputs: unknown; + expireAt: number; + usernameTemplate?: string | null; + config?: TDynamicSecretKubernetesLeaseConfig; + }) => { const providerInputs = await validateProviderInputs(inputs); - const tokenRequestCallback = async (host: string, port: number) => { + const serviceAccountDynamicCallback = async (host: string, port: number, httpsAgent?: https.Agent) => { + if (providerInputs.credentialType !== KubernetesCredentialType.Dynamic) { + throw new Error("invalid callback"); + } + + const baseUrl = port ? `${host}:${port}` : host; + const serviceAccountName = generateUsername(usernameTemplate); + const roleBindingName = `${serviceAccountName}-role-binding`; + const allowedNamespaces = providerInputs.namespace.split(",").map((namespace) => namespace.trim()); + + if (config?.namespace && !allowedNamespaces?.includes(config?.namespace)) { + throw new BadRequestError({ + message: `Namespace ${config?.namespace} is not allowed. Allowed namespaces: ${allowedNamespaces?.join(", ")}` + }); + } + + const namespace = config?.namespace || allowedNamespaces[0]; + if (!namespace) { + throw new BadRequestError({ + message: "No namespace provided" + }); + } + + // 1. Create the service account + await axios.post( + `${baseUrl}/api/v1/namespaces/${namespace}/serviceaccounts`, + { + metadata: { + name: serviceAccountName, + namespace + } + }, + { + headers: { + "Content-Type": "application/json", + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) + }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), + signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), + timeout: EXTERNAL_REQUEST_TIMEOUT + } + ); + + // 2. Create the role binding + const roleBindingUrl = + providerInputs.roleType === KubernetesRoleType.ClusterRole + ? `${baseUrl}/apis/rbac.authorization.k8s.io/v1/clusterrolebindings` + : `${baseUrl}/apis/rbac.authorization.k8s.io/v1/namespaces/${namespace}/rolebindings`; + + const roleBindingMetadata = { + name: roleBindingName, + ...(providerInputs.roleType !== KubernetesRoleType.ClusterRole && { namespace }) + }; + + await axios.post( + roleBindingUrl, + { + metadata: roleBindingMetadata, + roleRef: { + kind: providerInputs.roleType === KubernetesRoleType.ClusterRole ? "ClusterRole" : "Role", + name: providerInputs.role, + apiGroup: "rbac.authorization.k8s.io" + }, + subjects: [ + { + kind: "ServiceAccount", + name: serviceAccountName, + namespace + } + ] + }, + { + headers: { + "Content-Type": "application/json", + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) + }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), + signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), + timeout: EXTERNAL_REQUEST_TIMEOUT + } + ); + + // 3. Request a token for the service account + const res = await axios.post( + `${baseUrl}/api/v1/namespaces/${namespace}/serviceaccounts/${serviceAccountName}/token`, + { + spec: { + expirationSeconds: Math.floor((expireAt - Date.now()) / 1000), + ...(providerInputs.audiences?.length ? { audiences: providerInputs.audiences } : {}) + } + }, + { + headers: { + "Content-Type": "application/json", + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) + }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), + signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), + timeout: EXTERNAL_REQUEST_TIMEOUT + } + ); + + return { ...res.data, serviceAccountName }; + }; + + const tokenRequestStaticCallback = async (host: string, port: number, httpsAgent?: https.Agent) => { + if (providerInputs.credentialType !== KubernetesCredentialType.Static) { + throw new Error("invalid callback"); + } + + if (config?.namespace && config.namespace !== providerInputs.namespace) { + throw new BadRequestError({ + message: `Namespace ${config?.namespace} is not allowed. Allowed namespace: ${providerInputs.namespace}.` + }); + } + const baseUrl = port ? `${host}:${port}` : host; const res = await axios.post( @@ -133,39 +520,71 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): { headers: { "Content-Type": "application/json", - Authorization: `Bearer ${providerInputs.clusterToken}` + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), - timeout: EXTERNAL_REQUEST_TIMEOUT, - httpsAgent: new https.Agent({ - ca: providerInputs.ca, - rejectUnauthorized: providerInputs.sslEnabled - }) + timeout: EXTERNAL_REQUEST_TIMEOUT } ); - return res.data; + return { ...res.data, serviceAccountName: providerInputs.serviceAccountName }; }; - const url = new URL(providerInputs.url); + const rawUrl = + providerInputs.authMethod === KubernetesAuthMethod.Gateway ? GATEWAY_AUTH_DEFAULT_URL : providerInputs.url || ""; + const url = new URL(rawUrl); const k8sHost = `${url.protocol}//${url.hostname}`; const k8sGatewayHost = url.hostname; const k8sPort = url.port ? Number(url.port) : 443; try { - const tokenData = providerInputs.gatewayId - ? await $gatewayProxyWrapper( + let tokenData; + if (providerInputs.gatewayId) { + if (providerInputs.authMethod === KubernetesAuthMethod.Gateway) { + tokenData = await $gatewayProxyWrapper( + { + gatewayId: providerInputs.gatewayId, + targetHost: k8sHost, + targetPort: k8sPort, + enableSsl: providerInputs.sslEnabled, + caCert: providerInputs.ca, + reviewTokenThroughGateway: true + }, + providerInputs.credentialType === KubernetesCredentialType.Static + ? tokenRequestStaticCallback + : serviceAccountDynamicCallback + ); + } else { + tokenData = await $gatewayProxyWrapper( { gatewayId: providerInputs.gatewayId, targetHost: k8sGatewayHost, - targetPort: k8sPort + targetPort: k8sPort, + enableSsl: providerInputs.sslEnabled, + caCert: providerInputs.ca, + reviewTokenThroughGateway: false }, - tokenRequestCallback - ) - : await tokenRequestCallback(k8sHost, k8sPort); + providerInputs.credentialType === KubernetesCredentialType.Static + ? tokenRequestStaticCallback + : serviceAccountDynamicCallback + ); + } + } else { + tokenData = + providerInputs.credentialType === KubernetesCredentialType.Static + ? await tokenRequestStaticCallback(k8sHost, k8sPort) + : await serviceAccountDynamicCallback(k8sHost, k8sPort); + } return { - entityId: providerInputs.serviceAccountName, + entityId: tokenData.serviceAccountName, data: { TOKEN: tokenData.status.token } }; } catch (error) { @@ -180,7 +599,122 @@ export const KubernetesProvider = ({ gatewayService }: TKubernetesProviderDTO): } }; - const revoke = async (_inputs: unknown, entityId: string) => { + const revoke = async ( + inputs: unknown, + entityId: string, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _metadata: { projectId: string }, + config?: TDynamicSecretKubernetesLeaseConfig + ) => { + const providerInputs = await validateProviderInputs(inputs); + + const serviceAccountDynamicCallback = async (host: string, port: number, httpsAgent?: https.Agent) => { + if (providerInputs.credentialType !== KubernetesCredentialType.Dynamic) { + throw new Error("invalid callback"); + } + + const baseUrl = port ? `${host}:${port}` : host; + const roleBindingName = `${entityId}-role-binding`; + + const namespace = config?.namespace ?? providerInputs.namespace.split(",")[0].trim(); + + if (providerInputs.roleType === KubernetesRoleType.Role) { + await axios.delete( + `${baseUrl}/apis/rbac.authorization.k8s.io/v1/namespaces/${namespace}/rolebindings/${roleBindingName}`, + { + headers: { + "Content-Type": "application/json", + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) + }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), + signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), + timeout: EXTERNAL_REQUEST_TIMEOUT + } + ); + } else { + await axios.delete(`${baseUrl}/apis/rbac.authorization.k8s.io/v1/clusterrolebindings/${roleBindingName}`, { + headers: { + "Content-Type": "application/json", + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) + }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), + signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), + timeout: EXTERNAL_REQUEST_TIMEOUT + }); + } + + // Delete the service account + await axios.delete(`${baseUrl}/api/v1/namespaces/${namespace}/serviceaccounts/${entityId}`, { + headers: { + "Content-Type": "application/json", + ...(providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? { "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount } + : { Authorization: `Bearer ${providerInputs.clusterToken}` }) + }, + ...(providerInputs.authMethod === KubernetesAuthMethod.Api + ? { + httpsAgent + } + : {}), + signal: AbortSignal.timeout(EXTERNAL_REQUEST_TIMEOUT), + timeout: EXTERNAL_REQUEST_TIMEOUT + }); + }; + + if (providerInputs.credentialType === KubernetesCredentialType.Dynamic) { + const rawUrl = + providerInputs.authMethod === KubernetesAuthMethod.Gateway + ? GATEWAY_AUTH_DEFAULT_URL + : providerInputs.url || ""; + + const url = new URL(rawUrl); + const k8sGatewayHost = url.hostname; + const k8sPort = url.port ? Number(url.port) : 443; + const k8sHost = `${url.protocol}//${url.hostname}`; + + if (providerInputs.gatewayId) { + if (providerInputs.authMethod === KubernetesAuthMethod.Gateway) { + await $gatewayProxyWrapper( + { + gatewayId: providerInputs.gatewayId, + targetHost: k8sHost, + targetPort: k8sPort, + enableSsl: providerInputs.sslEnabled, + caCert: providerInputs.ca, + reviewTokenThroughGateway: true + }, + serviceAccountDynamicCallback + ); + } else { + await $gatewayProxyWrapper( + { + gatewayId: providerInputs.gatewayId, + targetHost: k8sGatewayHost, + targetPort: k8sPort, + enableSsl: providerInputs.sslEnabled, + caCert: providerInputs.ca, + reviewTokenThroughGateway: false + }, + serviceAccountDynamicCallback + ); + } + } else { + await serviceAccountDynamicCallback(k8sHost, k8sPort); + } + } + return { entityId }; }; diff --git a/backend/src/ee/services/dynamic-secret/providers/ldap.ts b/backend/src/ee/services/dynamic-secret/providers/ldap.ts index d0e3fbe66..1de8aa1e6 100644 --- a/backend/src/ee/services/dynamic-secret/providers/ldap.ts +++ b/backend/src/ee/services/dynamic-secret/providers/ldap.ts @@ -9,6 +9,7 @@ import { BadRequestError } from "@app/lib/errors"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { LdapCredentialType, LdapSchema, TDynamicProviderFns } from "./models"; +import { compileUsernameTemplate } from "./templateUtils"; const generatePassword = () => { const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.~!*$#"; @@ -22,13 +23,13 @@ const encodePassword = (password?: string) => { return base64Password; }; -const generateUsername = (usernameTemplate?: string | null) => { +const generateUsername = (usernameTemplate?: string | null, identity?: { name: string }) => { const randomUsername = alphaNumericNanoId(32); // Username must start with an ascii letter, so we prepend the username with "inf-" if (!usernameTemplate) return randomUsername; - - return handlebars.compile(usernameTemplate)({ + return compileUsernameTemplate({ + usernameTemplate, randomUsername, - unixTimestamp: Math.floor(Date.now() / 100) + identity }); }; @@ -196,8 +197,8 @@ export const LdapProvider = (): TDynamicProviderFns => { return dnArray; }; - const create = async (data: { inputs: unknown; usernameTemplate?: string | null }) => { - const { inputs, usernameTemplate } = data; + const create = async (data: { inputs: unknown; usernameTemplate?: string | null; identity?: { name: string } }) => { + const { inputs, usernameTemplate, identity } = data; const providerInputs = await validateProviderInputs(inputs); const client = await $getClient(providerInputs); @@ -224,7 +225,7 @@ export const LdapProvider = (): TDynamicProviderFns => { }); } } else { - const username = generateUsername(usernameTemplate); + const username = generateUsername(usernameTemplate, identity); const password = generatePassword(); const generatedLdif = generateLDIF({ username, password, ldifTemplate: providerInputs.creationLdif }); diff --git a/backend/src/ee/services/dynamic-secret/providers/models.ts b/backend/src/ee/services/dynamic-secret/providers/models.ts index 91d26da32..32cc46d22 100644 --- a/backend/src/ee/services/dynamic-secret/providers/models.ts +++ b/backend/src/ee/services/dynamic-secret/providers/models.ts @@ -1,5 +1,10 @@ +import RE2 from "re2"; import { z } from "zod"; +import { CharacterType, characterValidator } from "@app/lib/validator/validate-string"; + +import { TDynamicSecretLeaseConfig } from "../../dynamic-secret-lease/dynamic-secret-lease-types"; + export type PasswordRequirements = { length: number; required: { @@ -20,6 +25,11 @@ export enum SqlProviders { Vertica = "vertica" } +export enum AwsIamAuthType { + AssumeRole = "assume-role", + AccessKey = "access-key" +} + export enum ElasticSearchAuthTypes { User = "user", ApiKey = "api-key" @@ -31,7 +41,18 @@ export enum LdapCredentialType { } export enum KubernetesCredentialType { - Static = "static" + Static = "static", + Dynamic = "dynamic" +} + +export enum KubernetesRoleType { + ClusterRole = "cluster-role", + Role = "role" +} + +export enum KubernetesAuthMethod { + Gateway = "gateway", + Api = "api" } export enum TotpConfigType { @@ -168,16 +189,38 @@ export const DynamicSecretSapAseSchema = z.object({ revocationStatement: z.string().trim() }); -export const DynamicSecretAwsIamSchema = z.object({ - accessKey: z.string().trim().min(1), - secretAccessKey: z.string().trim().min(1), - region: z.string().trim().min(1), - awsPath: z.string().trim().optional(), - permissionBoundaryPolicyArn: z.string().trim().optional(), - policyDocument: z.string().trim().optional(), - userGroups: z.string().trim().optional(), - policyArns: z.string().trim().optional() -}); +export const DynamicSecretAwsIamSchema = z.preprocess( + (val) => { + if (typeof val === "object" && val !== null && !Object.hasOwn(val, "method")) { + // eslint-disable-next-line no-param-reassign + (val as { method: string }).method = AwsIamAuthType.AccessKey; + } + return val; + }, + z.discriminatedUnion("method", [ + z.object({ + method: z.literal(AwsIamAuthType.AccessKey), + accessKey: z.string().trim().min(1), + secretAccessKey: z.string().trim().min(1), + region: z.string().trim().min(1), + awsPath: z.string().trim().optional(), + permissionBoundaryPolicyArn: z.string().trim().optional(), + policyDocument: z.string().trim().optional(), + userGroups: z.string().trim().optional(), + policyArns: z.string().trim().optional() + }), + z.object({ + method: z.literal(AwsIamAuthType.AssumeRole), + roleArn: z.string().trim().min(1, "Role ARN required"), + region: z.string().trim().min(1), + awsPath: z.string().trim().optional(), + permissionBoundaryPolicyArn: z.string().trim().optional(), + policyDocument: z.string().trim().optional(), + userGroups: z.string().trim().optional(), + policyArns: z.string().trim().optional() + }) + ]) +); export const DynamicSecretMongoAtlasSchema = z.object({ adminPublicKey: z.string().trim().min(1).describe("Admin user public api key"), @@ -282,17 +325,89 @@ export const LdapSchema = z.union([ }) ]); -export const DynamicSecretKubernetesSchema = z.object({ - url: z.string().url().trim().min(1), - gatewayId: z.string().nullable().optional(), - sslEnabled: z.boolean().default(true), - clusterToken: z.string().trim().min(1), - ca: z.string().optional(), - serviceAccountName: z.string().trim().min(1), - credentialType: z.literal(KubernetesCredentialType.Static), - namespace: z.string().trim().min(1), - audiences: z.array(z.string().trim().min(1)) -}); +export const DynamicSecretKubernetesSchema = z + .discriminatedUnion("credentialType", [ + z.object({ + url: z + .string() + .optional() + .refine((val: string | undefined) => !val || new RE2(/^https?:\/\/.+/).test(val), { + message: "Invalid URL. Must start with http:// or https:// (e.g. https://example.com)" + }), + clusterToken: z.string().trim().optional(), + ca: z.string().optional(), + sslEnabled: z.boolean().default(false), + credentialType: z.literal(KubernetesCredentialType.Static), + serviceAccountName: z.string().trim().min(1), + namespace: z + .string() + .trim() + .min(1) + .refine((val) => !val.includes(","), "Namespace must be a single value, not a comma-separated list") + .refine( + (val) => characterValidator([CharacterType.AlphaNumeric, CharacterType.Hyphen])(val), + "Invalid namespace format" + ), + gatewayId: z.string().optional(), + audiences: z.array(z.string().trim().min(1)), + authMethod: z.nativeEnum(KubernetesAuthMethod).default(KubernetesAuthMethod.Api) + }), + z.object({ + url: z + .string() + .url() + .optional() + .refine((val: string | undefined) => !val || new RE2(/^https?:\/\/.+/).test(val), { + message: "Invalid URL. Must start with http:// or https:// (e.g. https://example.com)" + }), + clusterToken: z.string().trim().optional(), + ca: z.string().optional(), + sslEnabled: z.boolean().default(false), + credentialType: z.literal(KubernetesCredentialType.Dynamic), + namespace: z + .string() + .trim() + .min(1) + .refine((val) => { + const namespaces = val.split(",").map((ns) => ns.trim()); + return ( + namespaces.length > 0 && + namespaces.every((ns) => ns.length > 0) && + namespaces.every((ns) => characterValidator([CharacterType.AlphaNumeric, CharacterType.Hyphen])(ns)) + ); + }, "Must be a valid comma-separated list of namespace values"), + gatewayId: z.string().optional(), + audiences: z.array(z.string().trim().min(1)), + roleType: z.nativeEnum(KubernetesRoleType), + role: z.string().trim().min(1), + authMethod: z.nativeEnum(KubernetesAuthMethod).default(KubernetesAuthMethod.Api) + }) + ]) + .superRefine((data, ctx) => { + if (data.authMethod === KubernetesAuthMethod.Gateway && !data.gatewayId) { + ctx.addIssue({ + path: ["gatewayId"], + code: z.ZodIssueCode.custom, + message: "When auth method is set to Gateway, a gateway must be selected" + }); + } + if (data.authMethod === KubernetesAuthMethod.Api || !data.authMethod) { + if (!data.clusterToken) { + ctx.addIssue({ + path: ["clusterToken"], + code: z.ZodIssueCode.custom, + message: "When auth method is set to Token, a cluster token must be provided" + }); + } + if (!data.url) { + ctx.addIssue({ + path: ["url"], + code: z.ZodIssueCode.custom, + message: "When auth method is set to Token, a cluster URL must be provided" + }); + } + } + }); export const DynamicSecretVerticaSchema = z.object({ host: z.string().trim().toLowerCase(), @@ -400,9 +515,24 @@ export type TDynamicProviderFns = { inputs: unknown; expireAt: number; usernameTemplate?: string | null; + identity?: { + name: string; + }; + metadata: { projectId: string }; + config?: TDynamicSecretLeaseConfig; }) => Promise<{ entityId: string; data: unknown }>; - validateConnection: (inputs: unknown) => Promise; - validateProviderInputs: (inputs: object) => Promise; - revoke: (inputs: unknown, entityId: string) => Promise<{ entityId: string }>; - renew: (inputs: unknown, entityId: string, expireAt: number) => Promise<{ entityId: string }>; + validateConnection: (inputs: unknown, metadata: { projectId: string }) => Promise; + validateProviderInputs: (inputs: object, metadata: { projectId: string }) => Promise; + revoke: ( + inputs: unknown, + entityId: string, + metadata: { projectId: string }, + config?: TDynamicSecretLeaseConfig + ) => Promise<{ entityId: string }>; + renew: ( + inputs: unknown, + entityId: string, + expireAt: number, + metadata: { projectId: string } + ) => Promise<{ entityId: string }>; }; diff --git a/backend/src/ee/services/dynamic-secret/providers/mongo-atlas.ts b/backend/src/ee/services/dynamic-secret/providers/mongo-atlas.ts index 8f8bf9430..6da8b4b4e 100644 --- a/backend/src/ee/services/dynamic-secret/providers/mongo-atlas.ts +++ b/backend/src/ee/services/dynamic-secret/providers/mongo-atlas.ts @@ -1,5 +1,4 @@ import axios, { AxiosError } from "axios"; -import handlebars from "handlebars"; import { customAlphabet } from "nanoid"; import { z } from "zod"; @@ -7,19 +6,20 @@ import { createDigestAuthRequestInterceptor } from "@app/lib/axios/digest-auth"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { DynamicSecretMongoAtlasSchema, TDynamicProviderFns } from "./models"; +import { compileUsernameTemplate } from "./templateUtils"; const generatePassword = (size = 48) => { const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.~!*"; return customAlphabet(charset, 48)(size); }; -const generateUsername = (usernameTemplate?: string | null) => { +const generateUsername = (usernameTemplate?: string | null, identity?: { name: string }) => { const randomUsername = alphaNumericNanoId(32); if (!usernameTemplate) return randomUsername; - - return handlebars.compile(usernameTemplate)({ + return compileUsernameTemplate({ + usernameTemplate, randomUsername, - unixTimestamp: Math.floor(Date.now() / 100) + identity }); }; @@ -64,12 +64,17 @@ export const MongoAtlasProvider = (): TDynamicProviderFns => { return isConnected; }; - const create = async (data: { inputs: unknown; expireAt: number; usernameTemplate?: string | null }) => { - const { inputs, expireAt, usernameTemplate } = data; + const create = async (data: { + inputs: unknown; + expireAt: number; + usernameTemplate?: string | null; + identity?: { name: string }; + }) => { + const { inputs, expireAt, usernameTemplate, identity } = data; const providerInputs = await validateProviderInputs(inputs); const client = await $getClient(providerInputs); - const username = generateUsername(usernameTemplate); + const username = generateUsername(usernameTemplate, identity); const password = generatePassword(); const expiration = new Date(expireAt).toISOString(); await client({ diff --git a/backend/src/ee/services/dynamic-secret/providers/mongo-db.ts b/backend/src/ee/services/dynamic-secret/providers/mongo-db.ts index 0a15209e0..331a355a7 100644 --- a/backend/src/ee/services/dynamic-secret/providers/mongo-db.ts +++ b/backend/src/ee/services/dynamic-secret/providers/mongo-db.ts @@ -1,4 +1,3 @@ -import handlebars from "handlebars"; import { MongoClient } from "mongodb"; import { customAlphabet } from "nanoid"; import { z } from "zod"; @@ -7,19 +6,20 @@ import { alphaNumericNanoId } from "@app/lib/nanoid"; import { verifyHostInputValidity } from "../dynamic-secret-fns"; import { DynamicSecretMongoDBSchema, TDynamicProviderFns } from "./models"; +import { compileUsernameTemplate } from "./templateUtils"; const generatePassword = (size = 48) => { const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.~!*"; return customAlphabet(charset, 48)(size); }; -const generateUsername = (usernameTemplate?: string | null) => { +const generateUsername = (usernameTemplate?: string | null, identity?: { name: string }) => { const randomUsername = alphaNumericNanoId(32); if (!usernameTemplate) return randomUsername; - - return handlebars.compile(usernameTemplate)({ + return compileUsernameTemplate({ + usernameTemplate, randomUsername, - unixTimestamp: Math.floor(Date.now() / 100) + identity }); }; @@ -60,12 +60,12 @@ export const MongoDBProvider = (): TDynamicProviderFns => { return isConnected; }; - const create = async (data: { inputs: unknown; usernameTemplate?: string | null }) => { - const { inputs, usernameTemplate } = data; + const create = async (data: { inputs: unknown; usernameTemplate?: string | null; identity?: { name: string } }) => { + const { inputs, usernameTemplate, identity } = data; const providerInputs = await validateProviderInputs(inputs); const client = await $getClient(providerInputs); - const username = generateUsername(usernameTemplate); + const username = generateUsername(usernameTemplate, identity); const password = generatePassword(); const db = client.db(providerInputs.database); diff --git a/backend/src/ee/services/dynamic-secret/providers/rabbit-mq.ts b/backend/src/ee/services/dynamic-secret/providers/rabbit-mq.ts index e7d90d272..76081c86c 100644 --- a/backend/src/ee/services/dynamic-secret/providers/rabbit-mq.ts +++ b/backend/src/ee/services/dynamic-secret/providers/rabbit-mq.ts @@ -1,5 +1,4 @@ import axios, { Axios } from "axios"; -import handlebars from "handlebars"; import https from "https"; import { customAlphabet } from "nanoid"; import { z } from "zod"; @@ -9,19 +8,20 @@ import { alphaNumericNanoId } from "@app/lib/nanoid"; import { verifyHostInputValidity } from "../dynamic-secret-fns"; import { DynamicSecretRabbitMqSchema, TDynamicProviderFns } from "./models"; +import { compileUsernameTemplate } from "./templateUtils"; const generatePassword = () => { const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.~!*"; return customAlphabet(charset, 64)(); }; -const generateUsername = (usernameTemplate?: string | null) => { +const generateUsername = (usernameTemplate?: string | null, identity?: { name: string }) => { const randomUsername = alphaNumericNanoId(32); // Username must start with an ascii letter, so we prepend the username with "inf-" if (!usernameTemplate) return randomUsername; - - return handlebars.compile(usernameTemplate)({ + return compileUsernameTemplate({ + usernameTemplate, randomUsername, - unixTimestamp: Math.floor(Date.now() / 100) + identity }); }; @@ -117,12 +117,12 @@ export const RabbitMqProvider = (): TDynamicProviderFns => { return infoResponse; }; - const create = async (data: { inputs: unknown; usernameTemplate?: string | null }) => { - const { inputs, usernameTemplate } = data; + const create = async (data: { inputs: unknown; usernameTemplate?: string | null; identity?: { name: string } }) => { + const { inputs, usernameTemplate, identity } = data; const providerInputs = await validateProviderInputs(inputs); const connection = await $getClient(providerInputs); - const username = generateUsername(usernameTemplate); + const username = generateUsername(usernameTemplate, identity); const password = generatePassword(); await createRabbitMqUser({ diff --git a/backend/src/ee/services/dynamic-secret/providers/redis.ts b/backend/src/ee/services/dynamic-secret/providers/redis.ts index 855af2e29..989ed96dc 100644 --- a/backend/src/ee/services/dynamic-secret/providers/redis.ts +++ b/backend/src/ee/services/dynamic-secret/providers/redis.ts @@ -9,19 +9,20 @@ import { validateHandlebarTemplate } from "@app/lib/template/validate-handlebars import { verifyHostInputValidity } from "../dynamic-secret-fns"; import { DynamicSecretRedisDBSchema, TDynamicProviderFns } from "./models"; +import { compileUsernameTemplate } from "./templateUtils"; const generatePassword = () => { const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.~!*"; return customAlphabet(charset, 64)(); }; -const generateUsername = (usernameTemplate?: string | null) => { +const generateUsername = (usernameTemplate?: string | null, identity?: { name: string }) => { const randomUsername = alphaNumericNanoId(32); // Username must start with an ascii letter, so we prepend the username with "inf-" if (!usernameTemplate) return randomUsername; - - return handlebars.compile(usernameTemplate)({ + return compileUsernameTemplate({ + usernameTemplate, randomUsername, - unixTimestamp: Math.floor(Date.now() / 100) + identity }); }; @@ -121,12 +122,17 @@ export const RedisDatabaseProvider = (): TDynamicProviderFns => { return pingResponse; }; - const create = async (data: { inputs: unknown; expireAt: number; usernameTemplate?: string | null }) => { - const { inputs, expireAt, usernameTemplate } = data; + const create = async (data: { + inputs: unknown; + expireAt: number; + usernameTemplate?: string | null; + identity?: { name: string }; + }) => { + const { inputs, expireAt, usernameTemplate, identity } = data; const providerInputs = await validateProviderInputs(inputs); const connection = await $getClient(providerInputs); - const username = generateUsername(usernameTemplate); + const username = generateUsername(usernameTemplate, identity); const password = generatePassword(); const expiration = new Date(expireAt).toISOString(); diff --git a/backend/src/ee/services/dynamic-secret/providers/sap-ase.ts b/backend/src/ee/services/dynamic-secret/providers/sap-ase.ts index af2431058..9c13d3efc 100644 --- a/backend/src/ee/services/dynamic-secret/providers/sap-ase.ts +++ b/backend/src/ee/services/dynamic-secret/providers/sap-ase.ts @@ -9,19 +9,20 @@ import { validateHandlebarTemplate } from "@app/lib/template/validate-handlebars import { verifyHostInputValidity } from "../dynamic-secret-fns"; import { DynamicSecretSapAseSchema, TDynamicProviderFns } from "./models"; +import { compileUsernameTemplate } from "./templateUtils"; const generatePassword = (size = 48) => { const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; return customAlphabet(charset, 48)(size); }; -const generateUsername = (usernameTemplate?: string | null) => { +const generateUsername = (usernameTemplate?: string | null, identity?: { name: string }) => { const randomUsername = `inf_${alphaNumericNanoId(25)}`; // Username must start with an ascii letter, so we prepend the username with "inf-" if (!usernameTemplate) return randomUsername; - - return handlebars.compile(usernameTemplate)({ + return compileUsernameTemplate({ + usernameTemplate, randomUsername, - unixTimestamp: Math.floor(Date.now() / 100) + identity }); }; @@ -87,11 +88,11 @@ export const SapAseProvider = (): TDynamicProviderFns => { return true; }; - const create = async (data: { inputs: unknown; usernameTemplate?: string | null }) => { - const { inputs, usernameTemplate } = data; + const create = async (data: { inputs: unknown; usernameTemplate?: string | null; identity?: { name: string } }) => { + const { inputs, usernameTemplate, identity } = data; const providerInputs = await validateProviderInputs(inputs); - const username = generateUsername(usernameTemplate); + const username = generateUsername(usernameTemplate, identity); const password = generatePassword(); const client = await $getClient(providerInputs); diff --git a/backend/src/ee/services/dynamic-secret/providers/sap-hana.ts b/backend/src/ee/services/dynamic-secret/providers/sap-hana.ts index 654e2d144..5c8a75555 100644 --- a/backend/src/ee/services/dynamic-secret/providers/sap-hana.ts +++ b/backend/src/ee/services/dynamic-secret/providers/sap-hana.ts @@ -15,19 +15,20 @@ import { validateHandlebarTemplate } from "@app/lib/template/validate-handlebars import { verifyHostInputValidity } from "../dynamic-secret-fns"; import { DynamicSecretSapHanaSchema, TDynamicProviderFns } from "./models"; +import { compileUsernameTemplate } from "./templateUtils"; const generatePassword = (size = 48) => { const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; return customAlphabet(charset, 48)(size); }; -const generateUsername = (usernameTemplate?: string | null) => { +const generateUsername = (usernameTemplate?: string | null, identity?: { name: string }) => { const randomUsername = alphaNumericNanoId(32); // Username must start with an ascii letter, so we prepend the username with "inf-" if (!usernameTemplate) return randomUsername; - - return handlebars.compile(usernameTemplate)({ + return compileUsernameTemplate({ + usernameTemplate, randomUsername, - unixTimestamp: Math.floor(Date.now() / 100) + identity }); }; @@ -97,11 +98,16 @@ export const SapHanaProvider = (): TDynamicProviderFns => { return testResult; }; - const create = async (data: { inputs: unknown; expireAt: number; usernameTemplate?: string | null }) => { - const { inputs, expireAt, usernameTemplate } = data; + const create = async (data: { + inputs: unknown; + expireAt: number; + usernameTemplate?: string | null; + identity?: { name: string }; + }) => { + const { inputs, expireAt, usernameTemplate, identity } = data; const providerInputs = await validateProviderInputs(inputs); - const username = generateUsername(usernameTemplate); + const username = generateUsername(usernameTemplate, identity); const password = generatePassword(); const expiration = new Date(expireAt).toISOString(); diff --git a/backend/src/ee/services/dynamic-secret/providers/snowflake.ts b/backend/src/ee/services/dynamic-secret/providers/snowflake.ts index 571d488c9..9e97ecd30 100644 --- a/backend/src/ee/services/dynamic-secret/providers/snowflake.ts +++ b/backend/src/ee/services/dynamic-secret/providers/snowflake.ts @@ -8,6 +8,7 @@ import { alphaNumericNanoId } from "@app/lib/nanoid"; import { validateHandlebarTemplate } from "@app/lib/template/validate-handlebars"; import { DynamicSecretSnowflakeSchema, TDynamicProviderFns } from "./models"; +import { compileUsernameTemplate } from "./templateUtils"; // destroy client requires callback... const noop = () => {}; @@ -17,13 +18,13 @@ const generatePassword = (size = 48) => { return customAlphabet(charset, 48)(size); }; -const generateUsername = (usernameTemplate?: string | null) => { +const generateUsername = (usernameTemplate?: string | null, identity?: { name: string }) => { const randomUsername = `infisical_${alphaNumericNanoId(32)}`; // Username must start with an ascii letter, so we prepend the username with "inf-" if (!usernameTemplate) return randomUsername; - - return handlebars.compile(usernameTemplate)({ + return compileUsernameTemplate({ + usernameTemplate, randomUsername, - unixTimestamp: Math.floor(Date.now() / 100) + identity }); }; @@ -88,13 +89,18 @@ export const SnowflakeProvider = (): TDynamicProviderFns => { return isValidConnection; }; - const create = async (data: { inputs: unknown; expireAt: number; usernameTemplate?: string | null }) => { - const { inputs, expireAt, usernameTemplate } = data; + const create = async (data: { + inputs: unknown; + expireAt: number; + usernameTemplate?: string | null; + identity?: { name: string }; + }) => { + const { inputs, expireAt, usernameTemplate, identity } = data; const providerInputs = await validateProviderInputs(inputs); const client = await $getClient(providerInputs); - const username = generateUsername(usernameTemplate); + const username = generateUsername(usernameTemplate, identity); const password = generatePassword(); try { diff --git a/backend/src/ee/services/dynamic-secret/providers/sql-database.ts b/backend/src/ee/services/dynamic-secret/providers/sql-database.ts index ce16a1237..d3217be37 100644 --- a/backend/src/ee/services/dynamic-secret/providers/sql-database.ts +++ b/backend/src/ee/services/dynamic-secret/providers/sql-database.ts @@ -3,13 +3,14 @@ import handlebars from "handlebars"; import knex from "knex"; import { z } from "zod"; -import { withGatewayProxy } from "@app/lib/gateway"; +import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { validateHandlebarTemplate } from "@app/lib/template/validate-handlebars"; import { TGatewayServiceFactory } from "../../gateway/gateway-service"; import { verifyHostInputValidity } from "../dynamic-secret-fns"; import { DynamicSecretSqlDBSchema, PasswordRequirements, SqlProviders, TDynamicProviderFns } from "./models"; +import { compileUsernameTemplate } from "./templateUtils"; const EXTERNAL_REQUEST_TIMEOUT = 10 * 1000; @@ -104,9 +105,8 @@ const generatePassword = (provider: SqlProviders, requirements?: PasswordRequire } }; -const generateUsername = (provider: SqlProviders, usernameTemplate?: string | null) => { +const generateUsername = (provider: SqlProviders, usernameTemplate?: string | null, identity?: { name: string }) => { let randomUsername = ""; - // For oracle, the client assumes everything is upper case when not using quotes around the password if (provider === SqlProviders.Oracle) { randomUsername = alphaNumericNanoId(32).toUpperCase(); @@ -114,10 +114,13 @@ const generateUsername = (provider: SqlProviders, usernameTemplate?: string | nu randomUsername = alphaNumericNanoId(32); } if (!usernameTemplate) return randomUsername; - - return handlebars.compile(usernameTemplate)({ + return compileUsernameTemplate({ + usernameTemplate, randomUsername, - unixTimestamp: Math.floor(Date.now() / 100) + identity, + options: { + toUpperCase: provider === SqlProviders.Oracle + } }); }; @@ -185,6 +188,7 @@ export const SqlDatabaseProvider = ({ gatewayService }: TSqlDatabaseProviderDTO) await gatewayCallback("localhost", port); }, { + protocol: GatewayProxyProtocol.Tcp, targetHost: providerInputs.host, targetPort: providerInputs.port, relayHost, @@ -220,11 +224,16 @@ export const SqlDatabaseProvider = ({ gatewayService }: TSqlDatabaseProviderDTO) return isConnected; }; - const create = async (data: { inputs: unknown; expireAt: number; usernameTemplate?: string | null }) => { - const { inputs, expireAt, usernameTemplate } = data; + const create = async (data: { + inputs: unknown; + expireAt: number; + usernameTemplate?: string | null; + identity?: { name: string }; + }) => { + const { inputs, expireAt, usernameTemplate, identity } = data; const providerInputs = await validateProviderInputs(inputs); - const username = generateUsername(providerInputs.client, usernameTemplate); + const username = generateUsername(providerInputs.client, usernameTemplate, identity); const password = generatePassword(providerInputs.client, providerInputs.passwordRequirements); const gatewayCallback = async (host = providerInputs.host, port = providerInputs.port) => { diff --git a/backend/src/ee/services/dynamic-secret/providers/templateUtils.ts b/backend/src/ee/services/dynamic-secret/providers/templateUtils.ts new file mode 100644 index 000000000..70a083dbf --- /dev/null +++ b/backend/src/ee/services/dynamic-secret/providers/templateUtils.ts @@ -0,0 +1,80 @@ +/* eslint-disable func-names */ +import handlebars from "handlebars"; +import RE2 from "re2"; + +import { logger } from "@app/lib/logger"; +import { alphaNumericNanoId } from "@app/lib/nanoid"; + +export const compileUsernameTemplate = ({ + usernameTemplate, + randomUsername, + identity, + unixTimestamp, + options +}: { + usernameTemplate: string; + randomUsername: string; + identity?: { name: string }; + unixTimestamp?: number; + options?: { + toUpperCase?: boolean; + }; +}): string => { + // Create isolated handlebars instance + const hbs = handlebars.create(); + + // Register random helper on local instance + hbs.registerHelper("random", function (length: number) { + if (typeof length !== "number" || length <= 0 || length > 100) { + return ""; + } + return alphaNumericNanoId(length); + }); + + // Register replace helper on local instance + hbs.registerHelper("replace", function (text: string, searchValue: string, replaceValue: string) { + // Convert to string if it's not already + const textStr = String(text || ""); + if (!textStr) { + return textStr; + } + + try { + const re2Pattern = new RE2(searchValue, "g"); + // Replace all occurrences + return re2Pattern.replace(textStr, replaceValue); + } catch (error) { + logger.error(error, "RE2 pattern failed, using original template"); + return textStr; + } + }); + + // Register truncate helper on local instance + hbs.registerHelper("truncate", function (text: string, length: number) { + // Convert to string if it's not already + const textStr = String(text || ""); + if (!textStr) { + return textStr; + } + + if (typeof length !== "number" || length <= 0) return textStr; + return textStr.substring(0, length); + }); + + // Compile template with context using local instance + const context = { + randomUsername, + unixTimestamp: unixTimestamp || Math.floor(Date.now() / 100), + identity: { + name: identity?.name + } + }; + + const result = hbs.compile(usernameTemplate)(context); + + if (options?.toUpperCase) { + return result.toUpperCase(); + } + + return result; +}; diff --git a/backend/src/ee/services/dynamic-secret/providers/vertica.ts b/backend/src/ee/services/dynamic-secret/providers/vertica.ts index 9e283ab41..e361ab329 100644 --- a/backend/src/ee/services/dynamic-secret/providers/vertica.ts +++ b/backend/src/ee/services/dynamic-secret/providers/vertica.ts @@ -4,7 +4,7 @@ import knex, { Knex } from "knex"; import { z } from "zod"; import { BadRequestError } from "@app/lib/errors"; -import { withGatewayProxy } from "@app/lib/gateway"; +import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { validateHandlebarTemplate } from "@app/lib/template/validate-handlebars"; @@ -196,6 +196,7 @@ export const VerticaProvider = ({ gatewayService }: TVerticaProviderDTO): TDynam await gatewayCallback("localhost", port); }, { + protocol: GatewayProxyProtocol.Tcp, targetHost: providerInputs.host, targetPort: providerInputs.port, relayHost, diff --git a/backend/src/ee/services/group/group-types.ts b/backend/src/ee/services/group/group-types.ts index 9424075ca..1d7c5fc71 100644 --- a/backend/src/ee/services/group/group-types.ts +++ b/backend/src/ee/services/group/group-types.ts @@ -42,6 +42,10 @@ export type TListGroupUsersDTO = { filter?: EFilterReturnedUsers; } & TGenericPermission; +export type TListProjectGroupUsersDTO = TListGroupUsersDTO & { + projectId: string; +}; + export type TAddUserToGroupDTO = { id: string; username: string; diff --git a/backend/src/ee/services/license/license-service.ts b/backend/src/ee/services/license/license-service.ts index e2cf09bb1..80e58815e 100644 --- a/backend/src/ee/services/license/license-service.ts +++ b/backend/src/ee/services/license/license-service.ts @@ -709,6 +709,10 @@ export const licenseServiceFactory = ({ return licenses; }; + const invalidateGetPlan = async (orgId: string) => { + await keyStore.deleteItem(FEATURE_CACHE_KEY(orgId)); + }; + return { generateOrgCustomerId, removeOrgCustomer, @@ -723,6 +727,7 @@ export const licenseServiceFactory = ({ return onPremFeatures; }, getPlan, + invalidateGetPlan, updateSubscriptionOrgMemberCount, refreshPlan, getOrgPlan, diff --git a/backend/src/ee/services/permission/default-roles.ts b/backend/src/ee/services/permission/default-roles.ts index 40c5310ae..cca4efaf2 100644 --- a/backend/src/ee/services/permission/default-roles.ts +++ b/backend/src/ee/services/permission/default-roles.ts @@ -4,6 +4,7 @@ import { ProjectPermissionActions, ProjectPermissionCertificateActions, ProjectPermissionCmekActions, + ProjectPermissionCommitsActions, ProjectPermissionDynamicSecretActions, ProjectPermissionGroupActions, ProjectPermissionIdentityActions, @@ -90,6 +91,11 @@ const buildAdminPermissionRules = () => { ProjectPermissionSub.Certificates ); + can( + [ProjectPermissionCommitsActions.Read, ProjectPermissionCommitsActions.PerformRollback], + ProjectPermissionSub.Commits + ); + can( [ ProjectPermissionSshHostActions.Edit, @@ -292,6 +298,11 @@ const buildMemberPermissionRules = () => { ProjectPermissionSub.SecretImports ); + can( + [ProjectPermissionCommitsActions.Read, ProjectPermissionCommitsActions.PerformRollback], + ProjectPermissionSub.Commits + ); + can([ProjectPermissionActions.Read], ProjectPermissionSub.SecretApproval); can([ProjectPermissionSecretRotationActions.Read], ProjectPermissionSub.SecretRotation); @@ -479,6 +490,7 @@ const buildViewerPermissionRules = () => { can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificates); can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificateTemplates); can(ProjectPermissionSecretSyncActions.Read, ProjectPermissionSub.SecretSyncs); + can(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits); can( [ diff --git a/backend/src/ee/services/permission/project-permission.ts b/backend/src/ee/services/permission/project-permission.ts index 34cffcd98..f61c4b1a4 100644 --- a/backend/src/ee/services/permission/project-permission.ts +++ b/backend/src/ee/services/permission/project-permission.ts @@ -17,6 +17,11 @@ export enum ProjectPermissionActions { Delete = "delete" } +export enum ProjectPermissionCommitsActions { + Read = "read", + PerformRollback = "perform-rollback" +} + export enum ProjectPermissionCertificateActions { Read = "read", Create = "create", @@ -172,6 +177,7 @@ export enum ProjectPermissionSub { SecretRollback = "secret-rollback", SecretApproval = "secret-approval", SecretRotation = "secret-rotation", + Commits = "commits", Identity = "identity", CertificateAuthorities = "certificate-authorities", Certificates = "certificates", @@ -325,6 +331,7 @@ export type ProjectPermissionSet = | [ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback] | [ProjectPermissionActions.Create, ProjectPermissionSub.SecretRollback] | [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms] + | [ProjectPermissionCommitsActions, ProjectPermissionSub.Commits] | [ProjectPermissionSecretScanningDataSourceActions, ProjectPermissionSub.SecretScanningDataSources] | [ProjectPermissionSecretScanningFindingActions, ProjectPermissionSub.SecretScanningFindings] | [ProjectPermissionSecretScanningConfigActions, ProjectPermissionSub.SecretScanningConfigs]; @@ -376,7 +383,8 @@ const DynamicSecretConditionV2Schema = z .object({ [PermissionConditionOperators.$EQ]: PermissionConditionSchema[PermissionConditionOperators.$EQ], [PermissionConditionOperators.$NEQ]: PermissionConditionSchema[PermissionConditionOperators.$NEQ], - [PermissionConditionOperators.$IN]: PermissionConditionSchema[PermissionConditionOperators.$IN] + [PermissionConditionOperators.$IN]: PermissionConditionSchema[PermissionConditionOperators.$IN], + [PermissionConditionOperators.$GLOB]: PermissionConditionSchema[PermissionConditionOperators.$GLOB] }) .partial() ]), @@ -404,6 +412,23 @@ const DynamicSecretConditionV2Schema = z }) .partial(); +const SecretImportConditionSchema = z + .object({ + environment: z.union([ + z.string(), + z + .object({ + [PermissionConditionOperators.$EQ]: PermissionConditionSchema[PermissionConditionOperators.$EQ], + [PermissionConditionOperators.$NEQ]: PermissionConditionSchema[PermissionConditionOperators.$NEQ], + [PermissionConditionOperators.$IN]: PermissionConditionSchema[PermissionConditionOperators.$IN], + [PermissionConditionOperators.$GLOB]: PermissionConditionSchema[PermissionConditionOperators.$GLOB] + }) + .partial() + ]), + secretPath: SECRET_PATH_PERMISSION_OPERATOR_SCHEMA + }) + .partial(); + const SecretConditionV2Schema = z .object({ environment: z.union([ @@ -658,6 +683,12 @@ const GeneralPermissionSchema = [ "Describe what action an entity can take." ) }), + z.object({ + subject: z.literal(ProjectPermissionSub.Commits).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionCommitsActions).describe( + "Describe what action an entity can take." + ) + }), z.object({ subject: z .literal(ProjectPermissionSub.SecretScanningDataSources) @@ -741,7 +772,7 @@ export const ProjectPermissionV2Schema = z.discriminatedUnion("subject", [ action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionActions).describe( "Describe what action an entity can take." ), - conditions: SecretConditionV1Schema.describe( + conditions: SecretImportConditionSchema.describe( "When specified, only matching conditions will be allowed to access given resource." ).optional() }), diff --git a/backend/src/ee/services/pit/pit-service.ts b/backend/src/ee/services/pit/pit-service.ts new file mode 100644 index 000000000..160729123 --- /dev/null +++ b/backend/src/ee/services/pit/pit-service.ts @@ -0,0 +1,485 @@ +/* eslint-disable no-await-in-loop */ +import { ForbiddenError } from "@casl/ability"; + +import { ActionProjectType } from "@app/db/schemas"; +import { ProjectPermissionCommitsActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { NotFoundError } from "@app/lib/errors"; +import { logger } from "@app/lib/logger"; +import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type"; +import { ResourceType, TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; +import { + isFolderCommitChange, + isSecretCommitChange +} from "@app/services/folder-commit-changes/folder-commit-changes-dal"; +import { TProjectEnvDALFactory } from "@app/services/project-env/project-env-dal"; +import { TSecretServiceFactory } from "@app/services/secret/secret-service"; +import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { TSecretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service"; + +import { TPermissionServiceFactory } from "../permission/permission-service"; + +type TPitServiceFactoryDep = { + folderCommitService: TFolderCommitServiceFactory; + secretService: Pick; + folderService: Pick; + permissionService: Pick; + folderDAL: Pick; + projectEnvDAL: Pick; +}; + +export type TPitServiceFactory = ReturnType; + +export const pitServiceFactory = ({ + folderCommitService, + secretService, + folderService, + permissionService, + folderDAL, + projectEnvDAL +}: TPitServiceFactoryDep) => { + const getCommitsCount = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + environment, + path + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + environment: string; + path: string; + }) => { + const result = await folderCommitService.getCommitsCount({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + environment, + path + }); + + return result; + }; + + const getCommitsForFolder = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + environment, + path, + offset, + limit, + search, + sort + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + environment: string; + path: string; + offset: number; + limit: number; + search?: string; + sort: "asc" | "desc"; + }) => { + const result = await folderCommitService.getCommitsForFolder({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + environment, + path, + offset, + limit, + search, + sort + }); + + return { + commits: result.commits.map((commit) => ({ + ...commit, + commitId: commit.commitId.toString() + })), + total: result.total, + hasMore: result.hasMore + }; + }; + + const getCommitChanges = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + commitId: string; + }) => { + const changes = await folderCommitService.getCommitChanges({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId + }); + + const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(projectId, [changes.folderId]); + + for (const change of changes.changes) { + if (isSecretCommitChange(change)) { + change.versions = await secretService.getChangeVersions( + { + secretVersion: change.secretVersion, + secretId: change.secretId, + id: change.id, + isUpdate: change.isUpdate, + changeType: change.changeType + }, + (Number.parseInt(change.secretVersion, 10) - 1).toString(), + actorId, + actor, + actorOrgId, + actorAuthMethod, + changes.envId, + projectId, + folderWithPath?.path || "" + ); + } else if (isFolderCommitChange(change)) { + change.versions = await folderService.getFolderVersions( + change, + (Number.parseInt(change.folderVersion, 10) - 1).toString(), + change.folderChangeId + ); + } + } + + return { + changes: { + ...changes, + commitId: changes.commitId.toString() + } + }; + }; + + const compareCommitChanges = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId, + folderId, + environment, + deepRollback, + secretPath + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + commitId: string; + folderId: string; + environment: string; + deepRollback: boolean; + secretPath: string; + }) => { + const latestCommit = await folderCommitService.getLatestCommit({ + folderId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId + }); + + const targetCommit = await folderCommitService.getCommitById({ + commitId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId + }); + + const env = await projectEnvDAL.findOne({ + projectId, + slug: environment + }); + + if (!latestCommit) { + throw new NotFoundError({ message: "Latest commit not found" }); + } + + let diffs; + if (deepRollback) { + diffs = await folderCommitService.deepCompareFolder({ + targetCommitId: targetCommit.id, + envId: env.id, + projectId + }); + } else { + const folderData = await folderService.getFolderById({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + id: folderId + }); + + diffs = [ + { + folderId: folderData.id, + folderName: folderData.name, + folderPath: secretPath, + changes: await folderCommitService.compareFolderStates({ + targetCommitId: commitId, + currentCommitId: latestCommit.id + }) + } + ]; + } + + for (const diff of diffs) { + for (const change of diff.changes) { + // Use discriminated union type checking + if (change.type === ResourceType.SECRET) { + // TypeScript now knows this is a SecretChange + if (change.secretKey && change.secretVersion && change.secretId) { + change.versions = await secretService.getChangeVersions( + { + secretVersion: change.secretVersion, + secretId: change.secretId, + id: change.id, + isUpdate: change.isUpdate, + changeType: change.changeType + }, + change.fromVersion || "1", + actorId, + actor, + actorOrgId, + actorAuthMethod, + env.id, + projectId, + diff.folderPath || "" + ); + } + } else if (change.type === ResourceType.FOLDER) { + // TypeScript now knows this is a FolderChange + if (change.folderVersion) { + change.versions = await folderService.getFolderVersions(change, change.fromVersion || "1", change.id); + } + } + } + } + + return diffs; + }; + + const rollbackToCommit = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId, + folderId, + deepRollback, + message, + environment + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + commitId: string; + folderId: string; + deepRollback: boolean; + message?: string; + environment: string; + }) => { + const { permission: userPermission } = await permissionService.getProjectPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId, + actionProjectType: ActionProjectType.SecretManager + }); + + ForbiddenError.from(userPermission).throwUnlessCan( + ProjectPermissionCommitsActions.PerformRollback, + ProjectPermissionSub.Commits + ); + + const latestCommit = await folderCommitService.getLatestCommit({ + folderId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId + }); + + if (!latestCommit) { + throw new NotFoundError({ message: "Latest commit not found" }); + } + + logger.info(`PIT - Attempting to rollback folder ${folderId} from commit ${latestCommit.id} to commit ${commitId}`); + + const targetCommit = await folderCommitService.getCommitById({ + commitId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + + const env = await projectEnvDAL.findOne({ + projectId, + slug: environment + }); + + if (!targetCommit || targetCommit.folderId !== folderId || targetCommit.envId !== env.id) { + throw new NotFoundError({ message: "Target commit not found" }); + } + + if (!latestCommit || latestCommit.envId !== env.id) { + throw new NotFoundError({ message: "Latest commit not found" }); + } + + if (deepRollback) { + await folderCommitService.deepRollbackFolder(commitId, env.id, actorId, actor, projectId, message); + return { success: true }; + } + + const diff = await folderCommitService.compareFolderStates({ + currentCommitId: latestCommit.id, + targetCommitId: commitId + }); + + const response = await folderCommitService.applyFolderStateDifferences({ + differences: diff, + actorInfo: { + actorType: actor, + actorId, + message: message || "Rollback to previous commit" + }, + folderId, + projectId, + reconstructNewFolders: deepRollback + }); + + return { + success: true, + secretChangesCount: response.secretChangesCount, + folderChangesCount: response.folderChangesCount, + totalChanges: response.totalChanges + }; + }; + + const revertCommit = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + commitId: string; + }) => { + const response = await folderCommitService.revertCommitChanges({ + commitId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + + return response; + }; + + const getFolderStateAtCommit = async ({ + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + commitId + }: { + actor: ActorType; + actorId: string; + actorOrgId: string; + actorAuthMethod: ActorAuthMethod; + projectId: string; + commitId: string; + }) => { + const commit = await folderCommitService.getCommitById({ + commitId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId + }); + + if (!commit) { + throw new NotFoundError({ message: `Commit with ID ${commitId} not found` }); + } + + const response = await folderCommitService.reconstructFolderState(commitId); + + return response.map((item) => { + if (item.type === ResourceType.SECRET) { + return { + ...item, + secretVersion: Number(item.secretVersion) + }; + } + + if (item.type === ResourceType.FOLDER) { + return { + ...item, + folderVersion: Number(item.folderVersion) + }; + } + + return item; + }); + }; + + return { + getCommitsCount, + getCommitsForFolder, + getCommitChanges, + compareCommitChanges, + rollbackToCommit, + revertCommit, + getFolderStateAtCommit + }; +}; diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 217181281..9f1ee5307 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-nested-ternary */ import { ForbiddenError, subject } from "@casl/ability"; import { @@ -20,6 +21,7 @@ import { EnforcementLevel } from "@app/lib/types"; import { triggerWorkflowIntegrationNotification } from "@app/lib/workflow-integrations/trigger-notification"; import { TriggerFeature } from "@app/lib/workflow-integrations/types"; import { ActorType } from "@app/services/auth/auth-type"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TMicrosoftTeamsServiceFactory } from "@app/services/microsoft-teams/microsoft-teams-service"; @@ -130,6 +132,7 @@ type TSecretApprovalRequestServiceFactoryDep = { licenseService: Pick; projectMicrosoftTeamsConfigDAL: Pick; microsoftTeamsService: Pick; + folderCommitService: Pick; }; export type TSecretApprovalRequestServiceFactory = ReturnType; @@ -161,7 +164,8 @@ export const secretApprovalRequestServiceFactory = ({ projectSlackConfigDAL, resourceMetadataDAL, projectMicrosoftTeamsConfigDAL, - microsoftTeamsService + microsoftTeamsService, + folderCommitService }: TSecretApprovalRequestServiceFactoryDep) => { const requestCount = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod }: TApprovalRequestCountDTO) => { if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" }); @@ -243,7 +247,7 @@ export const secretApprovalRequestServiceFactory = ({ const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); const { policy } = secretApprovalRequest; - const { hasRole } = await permissionService.getProjectPermission({ + const { hasRole, permission } = await permissionService.getProjectPermission({ actor, actorId, projectId, @@ -259,6 +263,12 @@ export const secretApprovalRequestServiceFactory = ({ throw new ForbiddenRequestError({ message: "User has insufficient privileges" }); } + const hasSecretReadAccess = permission.can( + ProjectPermissionSecretActions.DescribeAndReadValue, + ProjectPermissionSub.Secrets + ); + const hiddenSecretValue = "******"; + let secrets; if (shouldUseSecretV2Bridge) { const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ @@ -275,9 +285,9 @@ export const secretApprovalRequestServiceFactory = ({ version: el.version, secretMetadata: el.secretMetadata as ResourceMetadataDTO, isRotatedSecret: el.secret?.isRotatedSecret ?? false, - secretValue: - // eslint-disable-next-line no-nested-ternary - el.secret && el.secret.isRotatedSecret + secretValue: !hasSecretReadAccess + ? hiddenSecretValue + : el.secret && el.secret.isRotatedSecret ? undefined : el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() @@ -290,9 +300,11 @@ export const secretApprovalRequestServiceFactory = ({ secretKey: el.secret.key, id: el.secret.id, version: el.secret.version, - secretValue: el.secret.encryptedValue - ? secretManagerDecryptor({ cipherTextBlob: el.secret.encryptedValue }).toString() - : "", + secretValue: !hasSecretReadAccess + ? hiddenSecretValue + : el.secret.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: el.secret.encryptedValue }).toString() + : "", secretComment: el.secret.encryptedComment ? secretManagerDecryptor({ cipherTextBlob: el.secret.encryptedComment }).toString() : "" @@ -303,9 +315,11 @@ export const secretApprovalRequestServiceFactory = ({ secretKey: el.secretVersion.key, id: el.secretVersion.id, version: el.secretVersion.version, - secretValue: el.secretVersion.encryptedValue - ? secretManagerDecryptor({ cipherTextBlob: el.secretVersion.encryptedValue }).toString() - : "", + secretValue: !hasSecretReadAccess + ? hiddenSecretValue + : el.secretVersion.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: el.secretVersion.encryptedValue }).toString() + : "", secretComment: el.secretVersion.encryptedComment ? secretManagerDecryptor({ cipherTextBlob: el.secretVersion.encryptedComment }).toString() : "", @@ -597,6 +611,10 @@ export const secretApprovalRequestServiceFactory = ({ ? await fnSecretV2BridgeBulkInsert({ tx, folderId, + actor: { + actorId, + type: actor + }, orgId: actorOrgId, inputSecrets: secretCreationCommits.map((el) => ({ tagIds: el?.tags.map(({ id }) => id), @@ -619,13 +637,18 @@ export const secretApprovalRequestServiceFactory = ({ secretDAL: secretV2BridgeDAL, secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, - secretVersionTagDAL: secretVersionTagV2BridgeDAL + secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService }) : []; const updatedSecrets = secretUpdationCommits.length ? await fnSecretV2BridgeBulkUpdate({ folderId, orgId: actorOrgId, + actor: { + actorId, + type: actor + }, tx, inputSecrets: secretUpdationCommits.map((el) => { const encryptedValue = @@ -659,7 +682,8 @@ export const secretApprovalRequestServiceFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }) : []; const deletedSecret = secretDeletionCommits.length @@ -667,10 +691,13 @@ export const secretApprovalRequestServiceFactory = ({ projectId, folderId, tx, - actorId: "", + actorId, + actorType: actor, secretDAL: secretV2BridgeDAL, secretQueueService, - inputSecrets: secretDeletionCommits.map(({ key }) => ({ secretKey: key, type: SecretType.Shared })) + inputSecrets: secretDeletionCommits.map(({ key }) => ({ secretKey: key, type: SecretType.Shared })), + folderCommitService, + secretVersionDAL: secretVersionV2BridgeDAL }) : []; const updatedSecretApproval = await secretApprovalRequestDAL.updateById( diff --git a/backend/src/ee/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts index 90fdf561e..628f8e310 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-service.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -10,6 +10,7 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { QueueName, TQueueServiceFactory } from "@app/queue"; import { ActorType } from "@app/services/auth/auth-type"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -87,6 +88,7 @@ type TSecretReplicationServiceFactoryDep = { projectBotService: Pick; kmsService: Pick; + folderCommitService: Pick; }; export type TSecretReplicationServiceFactory = ReturnType; @@ -132,6 +134,7 @@ export const secretReplicationServiceFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, kmsService, + folderCommitService, resourceMetadataDAL }: TSecretReplicationServiceFactoryDep) => { const $getReplicatedSecrets = ( @@ -419,7 +422,7 @@ export const secretReplicationServiceFactory = ({ return { op: operation, requestId: approvalRequestDoc.id, - metadata: doc.metadata, + metadata: doc.metadata ? JSON.stringify(doc.metadata) : [], secretMetadata: JSON.stringify(doc.secretMetadata), key: doc.key, encryptedValue: doc.encryptedValue, @@ -446,11 +449,12 @@ export const secretReplicationServiceFactory = ({ tx, secretTagDAL, resourceMetadataDAL, + folderCommitService, secretVersionTagDAL: secretVersionV2TagBridgeDAL, inputSecrets: locallyCreatedSecrets.map((doc) => { return { type: doc.type, - metadata: doc.metadata, + metadata: doc.metadata ? JSON.stringify(doc.metadata) : [], key: doc.key, encryptedValue: doc.encryptedValue, encryptedComment: doc.encryptedComment, @@ -466,6 +470,7 @@ export const secretReplicationServiceFactory = ({ orgId, folderId: destinationReplicationFolderId, secretVersionDAL: secretVersionV2BridgeDAL, + folderCommitService, secretDAL: secretV2BridgeDAL, tx, resourceMetadataDAL, @@ -479,7 +484,7 @@ export const secretReplicationServiceFactory = ({ }, data: { type: doc.type, - metadata: doc.metadata, + metadata: doc.metadata ? JSON.stringify(doc.metadata) : [], key: doc.key, encryptedValue: doc.encryptedValue as Buffer, encryptedComment: doc.encryptedComment, diff --git a/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.ts b/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.ts index 037df50ac..e2969ac1d 100644 --- a/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.ts +++ b/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.ts @@ -101,10 +101,56 @@ export const azureClientSecretRotationFactory: TRotationFactory< } }; + /** + * Checks if a credential with the given keyId exists. + */ + const credentialExists = async (keyId: string): Promise => { + const accessToken = await getAzureConnectionAccessToken(connection.id, appConnectionDAL, kmsService); + const endpoint = `${GRAPH_API_BASE}/applications/${objectId}/passwordCredentials`; + + try { + const { data } = await request.get<{ value: Array<{ keyId: string }> }>(endpoint, { + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + + return data.value?.some((credential) => credential.keyId === keyId) || false; + } catch (error: unknown) { + if (error instanceof AxiosError) { + let message; + if ( + error.response?.data && + typeof error.response.data === "object" && + "error" in error.response.data && + typeof (error.response.data as AzureErrorResponse).error.message === "string" + ) { + message = (error.response.data as AzureErrorResponse).error.message; + } + throw new BadRequestError({ + message: `Failed to check credential existence for app ${objectId}: ${ + message || error.message || "Unknown error" + }` + }); + } + throw new BadRequestError({ + message: "Unable to validate connection: verify credentials" + }); + } + }; + /** * Revokes a client secret from the Azure app using its keyId. + * First checks if the credential exists before attempting revocation. */ const revokeCredential = async (keyId: string) => { + // Check if credential exists before attempting revocation + const exists = await credentialExists(keyId); + if (!exists) { + return; // Credential doesn't exist, nothing to revoke + } + const accessToken = await getAzureConnectionAccessToken(connection.id, appConnectionDAL, kmsService); const endpoint = `${GRAPH_API_BASE}/applications/${objectId}/removePassword`; diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts index 6bf9c9b77..84a3435b6 100644 --- a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts +++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts @@ -63,6 +63,7 @@ import { TAppConnectionDALFactory } from "@app/services/app-connection/app-conne import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns"; import { TAppConnectionServiceFactory } from "@app/services/app-connection/app-connection-service"; import { ActorType } from "@app/services/auth/auth-type"; +import { TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -98,7 +99,7 @@ export type TSecretRotationV2ServiceFactoryDep = { TSecretV2BridgeDALFactory, "bulkUpdate" | "insertMany" | "deleteMany" | "upsertSecretReferences" | "find" | "invalidateSecretCacheByProjectId" >; - secretVersionV2BridgeDAL: Pick; + secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; secretTagDAL: Pick; @@ -106,6 +107,7 @@ export type TSecretRotationV2ServiceFactoryDep = { snapshotService: Pick; queueService: Pick; appConnectionDAL: Pick; + folderCommitService: Pick; }; export type TSecretRotationV2ServiceFactory = ReturnType; @@ -145,6 +147,7 @@ export const secretRotationV2ServiceFactory = ({ snapshotService, keyStore, queueService, + folderCommitService, appConnectionDAL }: TSecretRotationV2ServiceFactoryDep) => { const $queueSendSecretRotationStatusNotification = async (secretRotation: TSecretRotationV2Raw) => { @@ -538,7 +541,12 @@ export const secretRotationV2ServiceFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, secretTagDAL, - resourceMetadataDAL + folderCommitService, + resourceMetadataDAL, + actor: { + type: actor.type, + actorId: actor.id + } }); await secretRotationV2DAL.insertSecretMappings( @@ -674,7 +682,12 @@ export const secretRotationV2ServiceFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, secretTagDAL, - resourceMetadataDAL + folderCommitService, + resourceMetadataDAL, + actor: { + type: actor.type, + actorId: actor.id + } }); secretsMappingUpdated = true; @@ -792,6 +805,9 @@ export const secretRotationV2ServiceFactory = ({ projectId, folderId, actorId: actor.id, // not actually used since rotated secrets are shared + actorType: actor.type, + folderCommitService, + secretVersionDAL: secretVersionV2BridgeDAL, tx }); } @@ -935,6 +951,10 @@ export const secretRotationV2ServiceFactory = ({ secretDAL: secretV2BridgeDAL, secretVersionDAL: secretVersionV2BridgeDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService, + actor: { + type: ActorType.PLATFORM + }, secretTagDAL, resourceMetadataDAL }); diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts index 2c6124348..d792ac6e6 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts @@ -14,6 +14,7 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { ActorType } from "@app/services/auth/auth-type"; +import { CommitType, TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -53,6 +54,7 @@ type TSecretRotationQueueFactoryDep = { secretVersionV2BridgeDAL: Pick; telemetryService: Pick; kmsService: Pick; + folderCommitService: Pick; }; // These error should stop the repeatable job and ask user to reconfigure rotation @@ -77,6 +79,7 @@ export const secretRotationQueueFactory = ({ telemetryService, secretV2BridgeDAL, secretVersionV2BridgeDAL, + folderCommitService, kmsService }: TSecretRotationQueueFactoryDep) => { const addToQueue = async (rotationId: string, interval: number) => { @@ -330,7 +333,7 @@ export const secretRotationQueueFactory = ({ })), tx ); - await secretVersionV2BridgeDAL.insertMany( + const secretVersions = await secretVersionV2BridgeDAL.insertMany( updatedSecrets.map(({ id, updatedAt, createdAt, ...el }) => ({ ...el, actorType: ActorType.PLATFORM, @@ -338,6 +341,22 @@ export const secretRotationQueueFactory = ({ })), tx ); + + await folderCommitService.createCommit( + { + actor: { + type: ActorType.PLATFORM + }, + message: "Changed by Secret rotation", + folderId: secretVersions[0].folderId, + changes: secretVersions.map((sv) => ({ + type: CommitType.ADD, + isUpdate: true, + secretVersionId: sv.id + })) + }, + tx + ); }); await secretV2BridgeDAL.invalidateSecretCacheByProjectId(secretRotation.projectId); diff --git a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts index 015a8d420..8cae3dfb5 100644 --- a/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts +++ b/backend/src/ee/services/secret-snapshot/secret-snapshot-service.ts @@ -8,6 +8,7 @@ import { InternalServerError, NotFoundError } from "@app/lib/errors"; import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { ActorType } from "@app/services/auth/auth-type"; +import { CommitType, TFolderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -51,8 +52,8 @@ type TSecretSnapshotServiceFactoryDep = { snapshotSecretV2BridgeDAL: TSnapshotSecretV2DALFactory; snapshotFolderDAL: TSnapshotFolderDALFactory; secretVersionDAL: Pick; - secretVersionV2BridgeDAL: Pick; - folderVersionDAL: Pick; + secretVersionV2BridgeDAL: Pick; + folderVersionDAL: Pick; secretDAL: Pick; secretV2BridgeDAL: Pick; secretTagDAL: Pick; @@ -63,6 +64,7 @@ type TSecretSnapshotServiceFactoryDep = { licenseService: Pick; kmsService: Pick; projectBotService: Pick; + folderCommitService: Pick; }; export type TSecretSnapshotServiceFactory = ReturnType; @@ -84,7 +86,8 @@ export const secretSnapshotServiceFactory = ({ snapshotSecretV2BridgeDAL, secretVersionV2TagBridgeDAL, kmsService, - projectBotService + projectBotService, + folderCommitService }: TSecretSnapshotServiceFactoryDep) => { const projectSecretSnapshotCount = async ({ environment, @@ -403,6 +406,18 @@ export const secretSnapshotServiceFactory = ({ .filter((el) => el.isRotatedSecret) .map((el) => el.secretId); + const deletedSecretsChanges = new Map(); // secretId -> version info + const deletedFoldersChanges = new Map(); // folderId -> version info + const addedSecretsChanges = new Map(); // secretId -> version info + const addedFoldersChanges = new Map(); // folderId -> version info + const commitChanges: { + type: string; + secretVersionId?: string; + folderVersionId?: string; + isUpdate?: boolean; + folderId?: string; + }[] = []; + // this will remove all secrets in current folder except rotated secrets which we ignore const deletedTopLevelSecs = await secretV2BridgeDAL.delete( { @@ -424,7 +439,35 @@ export const secretSnapshotServiceFactory = ({ }, tx ); + + await Promise.all( + deletedTopLevelSecs.map(async (sec) => { + const version = await secretVersionV2BridgeDAL.findOne({ secretId: sec.id, version: sec.version }, tx); + deletedSecretsChanges.set(sec.id, { + id: sec.id, + version: sec.version, + // Store the version ID if available from the snapshot + versionId: version?.id + }); + }) + ); + const deletedTopLevelSecsGroupById = groupBy(deletedTopLevelSecs, (item) => item.id); + + const deletedFoldersData = await folderDAL.delete({ parentId: snapshot.folderId, isReserved: false }, tx); + + await Promise.all( + deletedFoldersData.map(async (folder) => { + const version = await folderVersionDAL.findOne({ folderId: folder.id, version: folder.version }, tx); + deletedFoldersChanges.set(folder.id, { + id: folder.id, + version: folder.version, + // Store the version ID if available + versionId: version?.id + }); + }) + ); + // this will remove all secrets and folders on child // due to sql foreign key and link list connection removing the folders removes everything below too const deletedFolders = await folderDAL.delete({ parentId: snapshot.folderId, isReserved: false }, tx); @@ -489,14 +532,21 @@ export const secretSnapshotServiceFactory = ({ }); await secretTagDAL.saveTagsToSecretV2(secretTagsToBeInsert, tx); const folderVersions = await folderVersionDAL.insertMany( - folders.map(({ version, name, id, envId }) => ({ + folders.map(({ version, name, id, envId, description }) => ({ name, version, folderId: id, - envId + envId, + description })), tx ); + + // Track added folders + folderVersions.forEach((fv) => { + addedFoldersChanges.set(fv.folderId, fv); + }); + const userActorId = actor === ActorType.USER ? actorId : undefined; const identityActorId = actor !== ActorType.USER ? actorId : undefined; const actorType = actor || ActorType.PLATFORM; @@ -511,6 +561,11 @@ export const secretSnapshotServiceFactory = ({ })), tx ); + + secretVersions.forEach((sv) => { + addedSecretsChanges.set(sv.secretId, sv); + }); + await secretVersionV2TagBridgeDAL.insertMany( secretVersions.flatMap(({ secretId, id }) => secretVerTagToBeInsert?.[secretId]?.length @@ -522,6 +577,70 @@ export const secretSnapshotServiceFactory = ({ ), tx ); + + // Compute commit changes + // Handle secrets + deletedSecretsChanges.forEach((deletedInfo, secretId) => { + const addedSecret = addedSecretsChanges.get(secretId); + if (addedSecret) { + // Secret was deleted and re-added - this is an update only if versions are different + if (deletedInfo.versionId !== addedSecret.id) { + commitChanges.push({ + type: CommitType.ADD, // In the commit system, updates are tracked as "add" with isUpdate=true + secretVersionId: addedSecret.id, + isUpdate: true + }); + } + // Remove from addedSecrets since we've handled it + addedSecretsChanges.delete(secretId); + } else if (deletedInfo.versionId) { + // Secret was only deleted + commitChanges.push({ + type: CommitType.DELETE, + secretVersionId: deletedInfo.versionId + }); + } + }); + // Add remaining new secrets (not updates) + addedSecretsChanges.forEach((addedSecret) => { + commitChanges.push({ + type: CommitType.ADD, + secretVersionId: addedSecret.id + }); + }); + + // Handle folders + deletedFoldersChanges.forEach((deletedInfo, folderId) => { + const addedFolder = addedFoldersChanges.get(folderId); + if (addedFolder) { + // Folder was deleted and re-added - this is an update only if versions are different + if (deletedInfo.versionId !== addedFolder.id) { + commitChanges.push({ + type: CommitType.ADD, + folderVersionId: addedFolder.id, + isUpdate: true + }); + } + // Remove from addedFolders since we've handled it + addedFoldersChanges.delete(folderId); + } else if (deletedInfo.versionId) { + // Folder was only deleted + commitChanges.push({ + type: CommitType.DELETE, + folderVersionId: deletedInfo.versionId, + folderId: deletedInfo.id + }); + } + }); + + // Add remaining new folders (not updates) + addedFoldersChanges.forEach((addedFolder) => { + commitChanges.push({ + type: CommitType.ADD, + folderVersionId: addedFolder.id + }); + }); + const newSnapshot = await snapshotDAL.create( { folderId: snapshot.folderId, @@ -550,6 +669,22 @@ export const secretSnapshotServiceFactory = ({ })), tx ); + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType, + metadata: { + id: userActorId || identityActorId + } + }, + message: "Rollback to snapshot", + folderId: snapshot.folderId, + changes: commitChanges + }, + tx + ); + } return { ...newSnapshot, snapshotSecrets, snapshotFolders }; }); @@ -609,11 +744,12 @@ export const secretSnapshotServiceFactory = ({ }); await secretTagDAL.saveTagsToSecret(secretTagsToBeInsert, tx); const folderVersions = await folderVersionDAL.insertMany( - folders.map(({ version, name, id, envId }) => ({ + folders.map(({ version, name, id, envId, description }) => ({ name, version, folderId: id, - envId + envId, + description })), tx ); diff --git a/backend/src/ee/services/secret-sync/oci-vault/oci-vault-sync-fns.ts b/backend/src/ee/services/secret-sync/oci-vault/oci-vault-sync-fns.ts index 5b05b2301..5fb39a0ec 100644 --- a/backend/src/ee/services/secret-sync/oci-vault/oci-vault-sync-fns.ts +++ b/backend/src/ee/services/secret-sync/oci-vault/oci-vault-sync-fns.ts @@ -117,6 +117,7 @@ export const OCIVaultSyncFns = { syncSecrets: async (secretSync: TOCIVaultSyncWithCredentials, secretMap: TSecretMap) => { const { connection, + environment, destinationConfig: { compartmentOcid, vaultOcid, keyOcid } } = secretSync; @@ -213,7 +214,7 @@ export const OCIVaultSyncFns = { // Update and delete secrets for await (const [key, variable] of Object.entries(variables)) { // eslint-disable-next-line no-continue - if (!matchesSchema(key, secretSync.syncOptions.keySchema)) continue; + if (!matchesSchema(key, environment?.slug || "", secretSync.syncOptions.keySchema)) continue; // Only update / delete active secrets if (variable.lifecycleState === vault.models.SecretSummary.LifecycleState.Active) { diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index a7eec1078..d3f19168a 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -10,7 +10,8 @@ export const PgSqlLock = { KmsRootKeyInit: 2025, OrgGatewayRootCaInit: (orgId: string) => pgAdvisoryLockHashText(`org-gateway-root-ca:${orgId}`), OrgGatewayCertExchange: (orgId: string) => pgAdvisoryLockHashText(`org-gateway-cert-exchange:${orgId}`), - SecretRotationV2Creation: (folderId: string) => pgAdvisoryLockHashText(`secret-rotation-v2-creation:${folderId}`) + SecretRotationV2Creation: (folderId: string) => pgAdvisoryLockHashText(`secret-rotation-v2-creation:${folderId}`), + CreateProject: (orgId: string) => pgAdvisoryLockHashText(`create-project:${orgId}`) } as const; export type TKeyStoreFactory = ReturnType; @@ -26,6 +27,7 @@ export const KeyStorePrefixes = { KmsOrgDataKeyCreation: "kms-org-data-key-creation-lock", WaitUntilReadyKmsOrgKeyCreation: "wait-until-ready-kms-org-key-creation-", WaitUntilReadyKmsOrgDataKeyCreation: "wait-until-ready-kms-org-data-key-creation-", + FolderTreeCheckpoint: (envId: string) => `folder-tree-checkpoint-${envId}`, WaitUntilReadyProjectEnvironmentOperation: (projectId: string) => `wait-until-ready-project-environments-operation-${projectId}`, diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index b37153d34..93a7217d4 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -89,6 +89,7 @@ export const GROUPS = { limit: "The number of users to return.", username: "The username to search for.", search: "The text string that user email or name will be filtered by.", + projectId: "The ID of the project the group belongs to.", filterUsers: "Whether to filter the list of returned users. 'existingMembers' will only return existing users in the group, 'nonMembers' will only return users not in the group, undefined will return all users in the organization." }, @@ -400,6 +401,8 @@ export const KUBERNETES_AUTH = { caCert: "The PEM-encoded CA cert for the Kubernetes API server.", tokenReviewerJwt: "Optional JWT token for accessing Kubernetes TokenReview API. If provided, this long-lived token will be used to validate service account tokens during authentication. If omitted, the client's own JWT will be used instead, which requires the client to have the system:auth-delegator ClusterRole binding.", + tokenReviewMode: + "The mode to use for token review. Must be one of: 'api', 'gateway'. If gateway is selected, the gateway must be deployed in Kubernetes, and the gateway must have the system:auth-delegator ClusterRole binding.", allowedNamespaces: "The comma-separated list of trusted namespaces that service accounts must belong to authenticate with Infisical.", allowedNames: "The comma-separated list of trusted service account names that can authenticate with Infisical.", @@ -417,6 +420,8 @@ export const KUBERNETES_AUTH = { caCert: "The new PEM-encoded CA cert for the Kubernetes API server.", tokenReviewerJwt: "Optional JWT token for accessing Kubernetes TokenReview API. If provided, this long-lived token will be used to validate service account tokens during authentication. If omitted, the client's own JWT will be used instead, which requires the client to have the system:auth-delegator ClusterRole binding.", + tokenReviewMode: + "The mode to use for token review. Must be one of: 'api', 'gateway'. If gateway is selected, the gateway must be deployed in Kubernetes, and the gateway must have the system:auth-delegator ClusterRole binding.", allowedNamespaces: "The new comma-separated list of trusted namespaces that service accounts must belong to authenticate with Infisical.", allowedNames: "The new comma-separated list of trusted service account names that can authenticate with Infisical.", @@ -621,7 +626,8 @@ export const PROJECTS = { autoCapitalization: "Disable or enable auto-capitalization for the project.", slug: "An optional slug for the project. (must be unique within the organization)", hasDeleteProtection: "Enable or disable delete protection for the project.", - secretSharing: "Enable or disable secret sharing for the project." + secretSharing: "Enable or disable secret sharing for the project.", + showSnapshotsLegacy: "Enable or disable legacy snapshots for the project." }, GET_KEY: { workspaceId: "The ID of the project to get the key from." @@ -1107,6 +1113,14 @@ export const DYNAMIC_SECRET_LEASES = { leaseId: "The ID of the dynamic secret lease.", isForced: "A boolean flag to delete the the dynamic secret from Infisical without trying to remove it from external provider. Used when the dynamic secret got modified externally." + }, + KUBERNETES: { + CREATE: { + config: { + namespace: + "The Kubernetes namespace to create the lease in. If not specified, the first namespace defined in the configuration will be used." + } + } } } as const; export const SECRET_TAGS = { @@ -2281,7 +2295,8 @@ export const SecretSyncs = { }, GCP: { scope: "The Google project scope that secrets should be synced to.", - projectId: "The ID of the Google project secrets should be synced to." + projectId: "The ID of the Google project secrets should be synced to.", + locationId: 'The ID of the Google project location secrets should be synced to (ie "us-west4").' }, DATABRICKS: { scope: "The Databricks secret scope that secrets should be synced to." diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index cb53a71a6..6b9d33c2a 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -213,6 +213,12 @@ const envSchema = z GATEWAY_RELAY_AUTH_SECRET: zpStr(z.string().optional()), DYNAMIC_SECRET_ALLOW_INTERNAL_IP: zodStrBool.default("false"), + DYNAMIC_SECRET_AWS_ACCESS_KEY_ID: zpStr(z.string().optional()).default( + process.env.INF_APP_CONNECTION_AWS_ACCESS_KEY_ID + ), + DYNAMIC_SECRET_AWS_SECRET_ACCESS_KEY: zpStr(z.string().optional()).default( + process.env.INF_APP_CONNECTION_AWS_SECRET_ACCESS_KEY + ), /* ----------------------------------------------------------------------------- */ /* App Connections ----------------------------------------------------------------------------- */ @@ -255,6 +261,10 @@ const envSchema = z DATADOG_SERVICE: zpStr(z.string().optional().default("infisical-core")), DATADOG_HOSTNAME: zpStr(z.string().optional()), + // PIT + PIT_CHECKPOINT_WINDOW: zpStr(z.string().optional().default("2")), + PIT_TREE_CHECKPOINT_WINDOW: zpStr(z.string().optional().default("30")), + /* CORS ----------------------------------------------------------------------------- */ CORS_ALLOWED_ORIGINS: zpStr( z diff --git a/backend/src/lib/gateway/gateway.ts b/backend/src/lib/gateway/gateway.ts new file mode 100644 index 000000000..46481a049 --- /dev/null +++ b/backend/src/lib/gateway/gateway.ts @@ -0,0 +1,428 @@ +/* eslint-disable no-await-in-loop */ +import crypto from "node:crypto"; +import net from "node:net"; + +import quicDefault, * as quicModule from "@infisical/quic"; +import axios from "axios"; +import https from "https"; + +import { BadRequestError } from "../errors"; +import { logger } from "../logger"; +import { + GatewayProxyProtocol, + IGatewayProxyOptions, + IGatewayProxyServer, + TGatewayTlsOptions, + TPingGatewayAndVerifyDTO +} from "./types"; + +const DEFAULT_MAX_RETRIES = 3; +const DEFAULT_RETRY_DELAY = 1000; // 1 second + +const quic = quicDefault || quicModule; + +const parseSubjectDetails = (data: string) => { + const values: Record = {}; + data.split("\n").forEach((el) => { + const [key, value] = el.split("="); + values[key.trim()] = value.trim(); + }); + return values; +}; + +const createQuicConnection = async ( + relayHost: string, + relayPort: number, + tlsOptions: TGatewayTlsOptions, + identityId: string, + orgId: string +) => { + const client = await quic.QUICClient.createQUICClient({ + host: relayHost, + port: relayPort, + config: { + ca: tlsOptions.ca, + cert: tlsOptions.cert, + key: tlsOptions.key, + applicationProtos: ["infisical-gateway"], + verifyPeer: true, + verifyCallback: async (certs) => { + if (!certs || certs.length === 0) return quic.native.CryptoError.CertificateRequired; + const serverCertificate = new crypto.X509Certificate(Buffer.from(certs[0])); + const caCertificate = new crypto.X509Certificate(tlsOptions.ca); + const isValidServerCertificate = serverCertificate.verify(caCertificate.publicKey); + if (!isValidServerCertificate) return quic.native.CryptoError.BadCertificate; + + const subjectDetails = parseSubjectDetails(serverCertificate.subject); + if (subjectDetails.OU !== "Gateway" || subjectDetails.CN !== identityId || subjectDetails.O !== orgId) { + return quic.native.CryptoError.CertificateUnknown; + } + + if (new Date() > new Date(serverCertificate.validTo) || new Date() < new Date(serverCertificate.validFrom)) { + return quic.native.CryptoError.CertificateExpired; + } + + const formatedRelayHost = + process.env.NODE_ENV === "development" ? relayHost.replace("host.docker.internal", "127.0.0.1") : relayHost; + if (!serverCertificate.checkIP(formatedRelayHost)) return quic.native.CryptoError.BadCertificate; + }, + maxIdleTimeout: 90000, + keepAliveIntervalTime: 30000 + }, + crypto: { + ops: { + randomBytes: async (data) => { + crypto.getRandomValues(new Uint8Array(data)); + } + } + } + }); + return client; +}; + +export const pingGatewayAndVerify = async ({ + relayHost, + relayPort, + tlsOptions, + maxRetries = DEFAULT_MAX_RETRIES, + identityId, + orgId +}: TPingGatewayAndVerifyDTO) => { + let lastError: Error | null = null; + const quicClient = await createQuicConnection(relayHost, relayPort, tlsOptions, identityId, orgId).catch((err) => { + throw new BadRequestError({ + message: (err as Error)?.message, + error: err as Error + }); + }); + + for (let attempt = 1; attempt <= maxRetries; attempt += 1) { + try { + const stream = quicClient.connection.newStream("bidi"); + const pingWriter = stream.writable.getWriter(); + await pingWriter.write(Buffer.from("PING\n")); + pingWriter.releaseLock(); + + // Read PONG response + const reader = stream.readable.getReader(); + const { value, done } = await reader.read(); + + if (done) { + throw new Error("Gateway closed before receiving PONG"); + } + + const response = Buffer.from(value).toString(); + + if (response !== "PONG\n" && response !== "PONG") { + throw new Error(`Failed to Ping. Unexpected response: ${response}`); + } + + reader.releaseLock(); + return; + } catch (err) { + lastError = err as Error; + + if (attempt < maxRetries) { + await new Promise((resolve) => { + setTimeout(resolve, DEFAULT_RETRY_DELAY); + }); + } + } finally { + await quicClient.destroy(); + } + } + + logger.error(lastError); + throw new BadRequestError({ + message: `Failed to ping gateway after ${maxRetries} attempts. Last error: ${lastError?.message}` + }); +}; + +const setupProxyServer = async ({ + targetPort, + targetHost, + tlsOptions, + relayHost, + relayPort, + identityId, + orgId, + protocol = GatewayProxyProtocol.Tcp, + httpsAgent +}: { + targetHost?: string; + targetPort?: number; + relayPort: number; + relayHost: string; + tlsOptions: TGatewayTlsOptions; + identityId: string; + orgId: string; + protocol?: GatewayProxyProtocol; + httpsAgent?: https.Agent; +}): Promise => { + const quicClient = await createQuicConnection(relayHost, relayPort, tlsOptions, identityId, orgId).catch((err) => { + throw new BadRequestError({ + error: err as Error + }); + }); + const proxyErrorMsg = [""]; + + return new Promise((resolve, reject) => { + const server = net.createServer(); + + let streamClosed = false; + + // eslint-disable-next-line @typescript-eslint/no-misused-promises + server.on("connection", async (clientConn) => { + try { + clientConn.setKeepAlive(true, 30000); // 30 seconds + clientConn.setNoDelay(true); + + const stream = quicClient.connection.newStream("bidi"); + + const forwardWriter = stream.writable.getWriter(); + let command: string; + + if (protocol === GatewayProxyProtocol.Http) { + if (!targetHost && !targetPort) { + command = `FORWARD-HTTP`; + logger.debug(`Using HTTP proxy mode, no target URL provided [command=${command.trim()}]`); + } else { + if (!targetHost || targetPort === undefined) { + throw new BadRequestError({ + message: `Target host and port are required for HTTP proxy mode with custom target` + }); + } + + const targetUrl = `${targetHost}:${targetPort}`; // note(daniel): targetHost MUST include the scheme (https|http) + command = `FORWARD-HTTP ${targetUrl}`; + logger.debug(`Using HTTP proxy mode, custom target URL provided [command=${command.trim()}]`); + + // extract ca certificate from httpsAgent if present + if (httpsAgent && targetHost.startsWith("https://")) { + const agentOptions = httpsAgent.options; + if (agentOptions && agentOptions.ca) { + const caCert = Array.isArray(agentOptions.ca) ? agentOptions.ca.join("\n") : agentOptions.ca; + const caB64 = Buffer.from(caCert as string).toString("base64"); + command += ` ca=${caB64}`; + + const rejectUnauthorized = agentOptions.rejectUnauthorized !== false; + command += ` verify=${rejectUnauthorized}`; + + logger.debug(`Using HTTP proxy mode, custom target URL provided [command=${command.trim()}]`); + } + } + } + + command += "\n"; + } else if (protocol === GatewayProxyProtocol.Tcp) { + if (!targetHost || !targetPort) { + throw new BadRequestError({ + message: `Target host and port are required for TCP proxy mode` + }); + } + + // For TCP mode, send FORWARD-TCP with host:port + command = `FORWARD-TCP ${targetHost}:${targetPort}\n`; + logger.debug(`Using TCP proxy mode: ${command.trim()}`); + } else { + throw new BadRequestError({ + message: `Invalid protocol: ${protocol as string}` + }); + } + + await forwardWriter.write(Buffer.from(command)); + forwardWriter.releaseLock(); + + // Set up bidirectional copy + const setupCopy = () => { + // Client to QUIC + // eslint-disable-next-line + (async () => { + const writer = stream.writable.getWriter(); + + // Create a handler for client data + clientConn.on("data", (chunk) => { + writer.write(chunk).catch((err) => { + proxyErrorMsg.push((err as Error)?.message); + }); + }); + + // Handle client connection close + clientConn.on("end", () => { + if (!streamClosed) { + try { + writer.close().catch((err) => { + logger.debug(err, "Error closing writer (already closed)"); + }); + } catch (error) { + logger.debug(error, "Error in writer close"); + } + } + }); + + clientConn.on("error", (clientConnErr) => { + writer.abort(clientConnErr?.message).catch((err) => { + proxyErrorMsg.push((err as Error)?.message); + }); + }); + })(); + + // QUIC to Client + void (async () => { + try { + const reader = stream.readable.getReader(); + + let reading = true; + while (reading) { + const { value, done } = await reader.read(); + + if (done) { + reading = false; + clientConn.end(); // Close client connection when QUIC stream ends + break; + } + + // Write data to TCP client + const canContinue = clientConn.write(Buffer.from(value)); + + // Handle backpressure + if (!canContinue) { + await new Promise((res) => { + clientConn.once("drain", res); + }); + } + } + } catch (err) { + proxyErrorMsg.push((err as Error)?.message); + clientConn.destroy(); + } + })(); + }; + + setupCopy(); + // Handle connection closure + clientConn.on("close", () => { + if (!streamClosed) { + streamClosed = true; + stream.destroy().catch((err) => { + logger.debug(err, "Stream already destroyed during close event"); + }); + } + }); + + const cleanup = async () => { + try { + clientConn?.destroy(); + } catch (err) { + logger.debug(err, "Error destroying client connection"); + } + + if (!streamClosed) { + streamClosed = true; + try { + await stream.destroy(); + } catch (err) { + logger.debug(err, "Error destroying stream (might be already closed)"); + } + } + }; + + clientConn.on("error", (clientConnErr) => { + logger.error(clientConnErr, "Client socket error"); + cleanup().catch((err) => { + logger.error(err, "Client conn cleanup"); + }); + }); + + clientConn.on("end", () => { + cleanup().catch((err) => { + logger.error(err, "Client conn end"); + }); + }); + } catch (err) { + logger.error(err, "Failed to establish target connection:"); + clientConn.end(); + reject(err); + } + }); + + server.on("error", (err) => { + reject(err); + }); + + server.on("close", () => { + quicClient?.destroy().catch((err) => { + logger.error(err, "Failed to destroy quic client"); + }); + }); + + server.listen(0, () => { + const address = server.address(); + if (!address || typeof address === "string") { + server.close(); + reject(new Error("Failed to get server port")); + return; + } + + logger.info(`Gateway proxy started on port ${address.port} (${protocol} mode)`); + resolve({ + server, + port: address.port, + cleanup: async () => { + try { + server.close(); + } catch (err) { + logger.debug(err, "Error closing server"); + } + + try { + await quicClient?.destroy(); + } catch (err) { + logger.debug(err, "Error destroying QUIC client"); + } + }, + getProxyError: () => proxyErrorMsg.join(",") + }); + }); + }); +}; + +export const withGatewayProxy = async ( + callback: (port: number, httpsAgent?: https.Agent) => Promise, + options: IGatewayProxyOptions +): Promise => { + const { relayHost, relayPort, targetHost, targetPort, tlsOptions, identityId, orgId, protocol, httpsAgent } = options; + + // Setup the proxy server + const { port, cleanup, getProxyError } = await setupProxyServer({ + targetHost, + targetPort, + relayPort, + relayHost, + tlsOptions, + identityId, + orgId, + protocol, + httpsAgent + }); + + try { + // Execute the callback with the allocated port + return await callback(port, httpsAgent); + } catch (err) { + const proxyErrorMessage = getProxyError(); + if (proxyErrorMessage) { + logger.error(new Error(proxyErrorMessage), "Failed to proxy"); + } + logger.error(err, "Failed to do gateway"); + let errorMessage = proxyErrorMessage || (err as Error)?.message; + if (axios.isAxiosError(err) && (err.response?.data as { message?: string })?.message) { + errorMessage = (err.response?.data as { message: string }).message; + } + + throw new BadRequestError({ message: errorMessage }); + } finally { + // Ensure cleanup happens regardless of success or failure + await cleanup(); + } +}; diff --git a/backend/src/lib/gateway/index.ts b/backend/src/lib/gateway/index.ts index 4d6401eac..9292473e5 100644 --- a/backend/src/lib/gateway/index.ts +++ b/backend/src/lib/gateway/index.ts @@ -1,392 +1,2 @@ -/* eslint-disable no-await-in-loop */ -import crypto from "node:crypto"; -import net from "node:net"; - -import quicDefault, * as quicModule from "@infisical/quic"; -import axios from "axios"; - -import { BadRequestError } from "../errors"; -import { logger } from "../logger"; - -const DEFAULT_MAX_RETRIES = 3; -const DEFAULT_RETRY_DELAY = 1000; // 1 second - -const quic = quicDefault || quicModule; - -const parseSubjectDetails = (data: string) => { - const values: Record = {}; - data.split("\n").forEach((el) => { - const [key, value] = el.split("="); - values[key.trim()] = value.trim(); - }); - return values; -}; - -type TTlsOption = { ca: string; cert: string; key: string }; - -const createQuicConnection = async ( - relayHost: string, - relayPort: number, - tlsOptions: TTlsOption, - identityId: string, - orgId: string -) => { - const client = await quic.QUICClient.createQUICClient({ - host: relayHost, - port: relayPort, - config: { - ca: tlsOptions.ca, - cert: tlsOptions.cert, - key: tlsOptions.key, - applicationProtos: ["infisical-gateway"], - verifyPeer: true, - verifyCallback: async (certs) => { - if (!certs || certs.length === 0) return quic.native.CryptoError.CertificateRequired; - const serverCertificate = new crypto.X509Certificate(Buffer.from(certs[0])); - const caCertificate = new crypto.X509Certificate(tlsOptions.ca); - const isValidServerCertificate = serverCertificate.verify(caCertificate.publicKey); - if (!isValidServerCertificate) return quic.native.CryptoError.BadCertificate; - - const subjectDetails = parseSubjectDetails(serverCertificate.subject); - if (subjectDetails.OU !== "Gateway" || subjectDetails.CN !== identityId || subjectDetails.O !== orgId) { - return quic.native.CryptoError.CertificateUnknown; - } - - if (new Date() > new Date(serverCertificate.validTo) || new Date() < new Date(serverCertificate.validFrom)) { - return quic.native.CryptoError.CertificateExpired; - } - - const formatedRelayHost = - process.env.NODE_ENV === "development" ? relayHost.replace("host.docker.internal", "127.0.0.1") : relayHost; - if (!serverCertificate.checkIP(formatedRelayHost)) return quic.native.CryptoError.BadCertificate; - }, - maxIdleTimeout: 90000, - keepAliveIntervalTime: 30000 - }, - crypto: { - ops: { - randomBytes: async (data) => { - crypto.getRandomValues(new Uint8Array(data)); - } - } - } - }); - return client; -}; - -type TPingGatewayAndVerifyDTO = { - relayHost: string; - relayPort: number; - tlsOptions: TTlsOption; - maxRetries?: number; - identityId: string; - orgId: string; -}; - -export const pingGatewayAndVerify = async ({ - relayHost, - relayPort, - tlsOptions, - maxRetries = DEFAULT_MAX_RETRIES, - identityId, - orgId -}: TPingGatewayAndVerifyDTO) => { - let lastError: Error | null = null; - const quicClient = await createQuicConnection(relayHost, relayPort, tlsOptions, identityId, orgId).catch((err) => { - throw new BadRequestError({ - message: (err as Error)?.message, - error: err as Error - }); - }); - - for (let attempt = 1; attempt <= maxRetries; attempt += 1) { - try { - const stream = quicClient.connection.newStream("bidi"); - const pingWriter = stream.writable.getWriter(); - await pingWriter.write(Buffer.from("PING\n")); - pingWriter.releaseLock(); - - // Read PONG response - const reader = stream.readable.getReader(); - const { value, done } = await reader.read(); - - if (done) { - throw new Error("Gateway closed before receiving PONG"); - } - - const response = Buffer.from(value).toString(); - - if (response !== "PONG\n" && response !== "PONG") { - throw new Error(`Failed to Ping. Unexpected response: ${response}`); - } - - reader.releaseLock(); - return; - } catch (err) { - lastError = err as Error; - - if (attempt < maxRetries) { - await new Promise((resolve) => { - setTimeout(resolve, DEFAULT_RETRY_DELAY); - }); - } - } finally { - await quicClient.destroy(); - } - } - - logger.error(lastError); - throw new BadRequestError({ - message: `Failed to ping gateway after ${maxRetries} attempts. Last error: ${lastError?.message}` - }); -}; - -interface TProxyServer { - server: net.Server; - port: number; - cleanup: () => Promise; - getProxyError: () => string; -} - -const setupProxyServer = async ({ - targetPort, - targetHost, - tlsOptions, - relayHost, - relayPort, - identityId, - orgId -}: { - targetHost: string; - targetPort: number; - relayPort: number; - relayHost: string; - tlsOptions: TTlsOption; - identityId: string; - orgId: string; -}): Promise => { - const quicClient = await createQuicConnection(relayHost, relayPort, tlsOptions, identityId, orgId).catch((err) => { - throw new BadRequestError({ - error: err as Error - }); - }); - const proxyErrorMsg = [""]; - - return new Promise((resolve, reject) => { - const server = net.createServer(); - - let streamClosed = false; - - // eslint-disable-next-line @typescript-eslint/no-misused-promises - server.on("connection", async (clientConn) => { - try { - clientConn.setKeepAlive(true, 30000); // 30 seconds - clientConn.setNoDelay(true); - - const stream = quicClient.connection.newStream("bidi"); - // Send FORWARD-TCP command - const forwardWriter = stream.writable.getWriter(); - await forwardWriter.write(Buffer.from(`FORWARD-TCP ${targetHost}:${targetPort}\n`)); - forwardWriter.releaseLock(); - - // Set up bidirectional copy - const setupCopy = () => { - // Client to QUIC - // eslint-disable-next-line - (async () => { - const writer = stream.writable.getWriter(); - - // Create a handler for client data - clientConn.on("data", (chunk) => { - writer.write(chunk).catch((err) => { - proxyErrorMsg.push((err as Error)?.message); - }); - }); - - // Handle client connection close - clientConn.on("end", () => { - if (!streamClosed) { - try { - writer.close().catch((err) => { - logger.debug(err, "Error closing writer (already closed)"); - }); - } catch (error) { - logger.debug(error, "Error in writer close"); - } - } - }); - - clientConn.on("error", (clientConnErr) => { - writer.abort(clientConnErr?.message).catch((err) => { - proxyErrorMsg.push((err as Error)?.message); - }); - }); - })(); - - // QUIC to Client - void (async () => { - try { - const reader = stream.readable.getReader(); - - let reading = true; - while (reading) { - const { value, done } = await reader.read(); - - if (done) { - reading = false; - clientConn.end(); // Close client connection when QUIC stream ends - break; - } - - // Write data to TCP client - const canContinue = clientConn.write(Buffer.from(value)); - - // Handle backpressure - if (!canContinue) { - await new Promise((res) => { - clientConn.once("drain", res); - }); - } - } - } catch (err) { - proxyErrorMsg.push((err as Error)?.message); - clientConn.destroy(); - } - })(); - }; - - setupCopy(); - // Handle connection closure - clientConn.on("close", () => { - if (!streamClosed) { - streamClosed = true; - stream.destroy().catch((err) => { - logger.debug(err, "Stream already destroyed during close event"); - }); - } - }); - - const cleanup = async () => { - try { - clientConn?.destroy(); - } catch (err) { - logger.debug(err, "Error destroying client connection"); - } - - if (!streamClosed) { - streamClosed = true; - try { - await stream.destroy(); - } catch (err) { - logger.debug(err, "Error destroying stream (might be already closed)"); - } - } - }; - - clientConn.on("error", (clientConnErr) => { - logger.error(clientConnErr, "Client socket error"); - cleanup().catch((err) => { - logger.error(err, "Client conn cleanup"); - }); - }); - - clientConn.on("end", () => { - cleanup().catch((err) => { - logger.error(err, "Client conn end"); - }); - }); - } catch (err) { - logger.error(err, "Failed to establish target connection:"); - clientConn.end(); - reject(err); - } - }); - - server.on("error", (err) => { - reject(err); - }); - - server.on("close", () => { - quicClient?.destroy().catch((err) => { - logger.error(err, "Failed to destroy quic client"); - }); - }); - - server.listen(0, () => { - const address = server.address(); - if (!address || typeof address === "string") { - server.close(); - reject(new Error("Failed to get server port")); - return; - } - - logger.info("Gateway proxy started"); - resolve({ - server, - port: address.port, - cleanup: async () => { - try { - server.close(); - } catch (err) { - logger.debug(err, "Error closing server"); - } - - try { - await quicClient?.destroy(); - } catch (err) { - logger.debug(err, "Error destroying QUIC client"); - } - }, - getProxyError: () => proxyErrorMsg.join(",") - }); - }); - }); -}; - -interface ProxyOptions { - targetHost: string; - targetPort: number; - relayHost: string; - relayPort: number; - tlsOptions: TTlsOption; - identityId: string; - orgId: string; -} - -export const withGatewayProxy = async ( - callback: (port: number) => Promise, - options: ProxyOptions -): Promise => { - const { relayHost, relayPort, targetHost, targetPort, tlsOptions, identityId, orgId } = options; - - // Setup the proxy server - const { port, cleanup, getProxyError } = await setupProxyServer({ - targetHost, - targetPort, - relayPort, - relayHost, - tlsOptions, - identityId, - orgId - }); - - try { - // Execute the callback with the allocated port - return await callback(port); - } catch (err) { - const proxyErrorMessage = getProxyError(); - if (proxyErrorMessage) { - logger.error(new Error(proxyErrorMessage), "Failed to proxy"); - } - logger.error(err, "Failed to do gateway"); - let errorMessage = proxyErrorMessage || (err as Error)?.message; - if (axios.isAxiosError(err) && (err.response?.data as { message?: string })?.message) { - errorMessage = (err.response?.data as { message: string }).message; - } - - throw new BadRequestError({ message: errorMessage }); - } finally { - // Ensure cleanup happens regardless of success or failure - await cleanup(); - } -}; +export { pingGatewayAndVerify, withGatewayProxy } from "./gateway"; +export { GatewayHttpProxyActions, GatewayProxyProtocol } from "./types"; diff --git a/backend/src/lib/gateway/types.ts b/backend/src/lib/gateway/types.ts new file mode 100644 index 000000000..8552fbf54 --- /dev/null +++ b/backend/src/lib/gateway/types.ts @@ -0,0 +1,43 @@ +import net from "node:net"; + +import https from "https"; + +export type TGatewayTlsOptions = { ca: string; cert: string; key: string }; + +export enum GatewayProxyProtocol { + Http = "http", + Tcp = "tcp" +} + +export enum GatewayHttpProxyActions { + InjectGatewayK8sServiceAccountToken = "inject-k8s-sa-auth-token", + UseGatewayK8sServiceAccount = "use-k8s-sa" +} + +export interface IGatewayProxyOptions { + targetHost?: string; + targetPort?: number; + relayHost: string; + relayPort: number; + tlsOptions: TGatewayTlsOptions; + identityId: string; + orgId: string; + protocol: GatewayProxyProtocol; + httpsAgent?: https.Agent; +} + +export type TPingGatewayAndVerifyDTO = { + relayHost: string; + relayPort: number; + tlsOptions: TGatewayTlsOptions; + maxRetries?: number; + identityId: string; + orgId: string; +}; + +export interface IGatewayProxyServer { + server: net.Server; + port: number; + cleanup: () => Promise; + getProxyError: () => string; +} diff --git a/backend/src/lib/template/validate-handlebars.ts b/backend/src/lib/template/validate-handlebars.ts index 08343e962..4aa0d1f63 100644 --- a/backend/src/lib/template/validate-handlebars.ts +++ b/backend/src/lib/template/validate-handlebars.ts @@ -7,13 +7,24 @@ type SanitizationArg = { allowedExpressions?: (arg: string) => boolean; }; +const isValidExpression = (expression: string, dto: SanitizationArg): boolean => { + // Allow helper functions (replace, truncate) + const allowedHelpers = ["replace", "truncate", "random"]; + if (allowedHelpers.includes(expression)) { + return true; + } + + // Check regular allowed expressions + return dto?.allowedExpressions?.(expression) || false; +}; + export const validateHandlebarTemplate = (templateName: string, template: string, dto: SanitizationArg) => { const parsedAst = handlebars.parse(template); parsedAst.body.forEach((el) => { if (el.type === "ContentStatement") return; if (el.type === "MustacheStatement" && "path" in el) { const { path } = el as { type: "MustacheStatement"; path: { type: "PathExpression"; original: string } }; - if (path.type === "PathExpression" && dto?.allowedExpressions?.(path.original)) return; + if (path.type === "PathExpression" && isValidExpression(path.original, dto)) return; } logger.error(el, "Template sanitization failed"); throw new BadRequestError({ message: `Template sanitization failed: ${templateName}` }); @@ -26,7 +37,7 @@ export const isValidHandleBarTemplate = (template: string, dto: SanitizationArg) if (el.type === "ContentStatement") return true; if (el.type === "MustacheStatement" && "path" in el) { const { path } = el as { type: "MustacheStatement"; path: { type: "PathExpression"; original: string } }; - if (path.type === "PathExpression" && dto?.allowedExpressions?.(path.original)) return true; + if (path.type === "PathExpression" && isValidExpression(path.original, dto)) return true; } return false; }); diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index e4d654998..25677841d 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -60,6 +60,7 @@ export enum QueueName { ImportSecretsFromExternalSource = "import-secrets-from-external-source", AppConnectionSecretSync = "app-connection-secret-sync", SecretRotationV2 = "secret-rotation-v2", + FolderTreeCheckpoint = "folder-tree-checkpoint", InvalidateCache = "invalidate-cache", SecretScanningV2 = "secret-scanning-v2" } @@ -94,6 +95,7 @@ export enum QueueJobs { SecretRotationV2QueueRotations = "secret-rotation-v2-queue-rotations", SecretRotationV2RotateSecrets = "secret-rotation-v2-rotate-secrets", SecretRotationV2SendNotification = "secret-rotation-v2-send-notification", + CreateFolderTreeCheckpoint = "create-folder-tree-checkpoint", InvalidateCache = "invalidate-cache", SecretScanningV2FullScan = "secret-scanning-v2-full-scan", SecretScanningV2DiffScan = "secret-scanning-v2-diff-scan", @@ -209,6 +211,12 @@ export type TQueueJobTypes = { name: QueueJobs.ProjectV3Migration; payload: { projectId: string }; }; + [QueueName.FolderTreeCheckpoint]: { + name: QueueJobs.CreateFolderTreeCheckpoint; + payload: { + envId: string; + }; + }; [QueueName.ImportSecretsFromExternalSource]: { name: QueueJobs.ImportSecretsFromExternalSource; payload: { diff --git a/backend/src/server/config/rateLimiter.ts b/backend/src/server/config/rateLimiter.ts index 7b4b9a99b..d3d3d3efd 100644 --- a/backend/src/server/config/rateLimiter.ts +++ b/backend/src/server/config/rateLimiter.ts @@ -11,7 +11,7 @@ export const globalRateLimiterCfg = (): RateLimitPluginOptions => { return { errorResponseBuilder: (_, context) => { throw new RateLimitError({ - message: `Rate limit exceeded. Please try again in ${context.after}` + message: `Rate limit exceeded. Please try again in ${Math.ceil(context.ttl / 1000)} seconds` }); }, timeWindow: 60 * 1000, @@ -113,3 +113,12 @@ export const requestAccessLimit: RateLimitOptions = { max: 10, keyGenerator: (req) => req.realIp }; + +export const smtpRateLimit = ({ + keyGenerator = (req) => req.realIp +}: Pick = {}): RateLimitOptions => ({ + timeWindow: 40 * 1000, + hook: "preValidation", + max: 2, + keyGenerator +}); diff --git a/backend/src/server/plugins/auth/inject-identity.ts b/backend/src/server/plugins/auth/inject-identity.ts index afea5c9f9..f065bfbed 100644 --- a/backend/src/server/plugins/auth/inject-identity.ts +++ b/backend/src/server/plugins/auth/inject-identity.ts @@ -155,6 +155,12 @@ export const injectIdentity = fp(async (server: FastifyZodProvider) => { oidc: token?.identityAuth?.oidc }); } + if (token?.identityAuth?.kubernetes) { + requestContext.set("identityAuthInfo", { + identityId: identity.identityId, + kubernetes: token?.identityAuth?.kubernetes + }); + } break; } case AuthMode.SERVICE_TOKEN: { diff --git a/backend/src/server/plugins/serve-ui.ts b/backend/src/server/plugins/serve-ui.ts index 22c097726..b71451b6e 100644 --- a/backend/src/server/plugins/serve-ui.ts +++ b/backend/src/server/plugins/serve-ui.ts @@ -57,9 +57,12 @@ export const registerServeUI = async ( reply.callNotFound(); return; } - // reference: https://github.com/fastify/fastify-static?tab=readme-ov-file#managing-cache-control-headers - // to avoid ui bundle skew on new deployment - return reply.sendFile("index.html", { maxAge: 0, immutable: false }); + + // This should help avoid caching any chunks (temp fix) + void reply.header("Cache-Control", "no-cache, no-store, must-revalidate, private, max-age=0"); + void reply.header("Pragma", "no-cache"); + void reply.header("Expires", "0"); + return reply.sendFile("index.html"); } }); } diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 509a2ef1f..5513fb49b 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -60,6 +60,7 @@ import { oidcConfigDALFactory } from "@app/ee/services/oidc/oidc-config-dal"; import { oidcConfigServiceFactory } from "@app/ee/services/oidc/oidc-config-service"; import { permissionDALFactory } from "@app/ee/services/permission/permission-dal"; import { permissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { pitServiceFactory } from "@app/ee/services/pit/pit-service"; import { projectTemplateDALFactory } from "@app/ee/services/project-template/project-template-dal"; import { projectTemplateServiceFactory } from "@app/ee/services/project-template/project-template-service"; import { projectUserAdditionalPrivilegeDALFactory } from "@app/ee/services/project-user-additional-privilege/project-user-additional-privilege-dal"; @@ -154,6 +155,14 @@ import { externalGroupOrgRoleMappingDALFactory } from "@app/services/external-gr import { externalGroupOrgRoleMappingServiceFactory } from "@app/services/external-group-org-role-mapping/external-group-org-role-mapping-service"; import { externalMigrationQueueFactory } from "@app/services/external-migration/external-migration-queue"; import { externalMigrationServiceFactory } from "@app/services/external-migration/external-migration-service"; +import { folderCheckpointDALFactory } from "@app/services/folder-checkpoint/folder-checkpoint-dal"; +import { folderCheckpointResourcesDALFactory } from "@app/services/folder-checkpoint-resources/folder-checkpoint-resources-dal"; +import { folderCommitDALFactory } from "@app/services/folder-commit/folder-commit-dal"; +import { folderCommitQueueServiceFactory } from "@app/services/folder-commit/folder-commit-queue"; +import { folderCommitServiceFactory } from "@app/services/folder-commit/folder-commit-service"; +import { folderCommitChangesDALFactory } from "@app/services/folder-commit-changes/folder-commit-changes-dal"; +import { folderTreeCheckpointDALFactory } from "@app/services/folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { folderTreeCheckpointResourcesDALFactory } from "@app/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal"; import { groupProjectDALFactory } from "@app/services/group-project/group-project-dal"; import { groupProjectMembershipRoleDALFactory } from "@app/services/group-project/group-project-membership-role-dal"; import { groupProjectServiceFactory } from "@app/services/group-project/group-project-service"; @@ -583,6 +592,41 @@ export const registerRoutes = async ( projectRoleDAL, permissionService }); + + const folderCommitChangesDAL = folderCommitChangesDALFactory(db); + const folderCheckpointDAL = folderCheckpointDALFactory(db); + const folderCheckpointResourcesDAL = folderCheckpointResourcesDALFactory(db); + const folderTreeCheckpointDAL = folderTreeCheckpointDALFactory(db); + const folderCommitDAL = folderCommitDALFactory(db); + const folderTreeCheckpointResourcesDAL = folderTreeCheckpointResourcesDALFactory(db); + const folderCommitQueueService = folderCommitQueueServiceFactory({ + queueService, + folderTreeCheckpointDAL, + keyStore, + folderTreeCheckpointResourcesDAL, + folderCommitDAL, + folderDAL + }); + const folderCommitService = folderCommitServiceFactory({ + folderCommitDAL, + folderCommitChangesDAL, + folderCheckpointDAL, + folderTreeCheckpointDAL, + userDAL, + identityDAL, + folderDAL, + folderVersionDAL, + secretVersionV2BridgeDAL, + projectDAL, + folderCheckpointResourcesDAL, + secretV2BridgeDAL, + folderTreeCheckpointResourcesDAL, + folderCommitQueueService, + permissionService, + kmsService, + secretTagDAL, + resourceMetadataDAL + }); const scimService = scimServiceFactory({ licenseService, scimDAL, @@ -987,6 +1031,7 @@ export const registerRoutes = async ( projectMembershipDAL, projectBotDAL, secretDAL, + folderCommitService, secretBlindIndexDAL, secretVersionDAL, secretTagDAL, @@ -1034,6 +1079,7 @@ export const registerRoutes = async ( secretReminderRecipientsDAL, orgService, resourceMetadataDAL, + folderCommitService, secretSyncQueue }); @@ -1110,6 +1156,7 @@ export const registerRoutes = async ( snapshotDAL, snapshotFolderDAL, snapshotSecretDAL, + folderCommitService, secretVersionDAL, folderVersionDAL, secretTagDAL, @@ -1136,7 +1183,8 @@ export const registerRoutes = async ( folderVersionDAL, projectEnvDAL, snapshotService, - projectDAL + projectDAL, + folderCommitService }); const secretImportService = secretImportServiceFactory({ @@ -1161,6 +1209,7 @@ export const registerRoutes = async ( const secretV2BridgeService = secretV2BridgeServiceFactory({ folderDAL, secretVersionDAL: secretVersionV2BridgeDAL, + folderCommitService, secretQueueService, secretDAL: secretV2BridgeDAL, permissionService, @@ -1204,7 +1253,8 @@ export const registerRoutes = async ( projectSlackConfigDAL, resourceMetadataDAL, projectMicrosoftTeamsConfigDAL, - microsoftTeamsService + microsoftTeamsService, + folderCommitService }); const secretService = secretServiceFactory({ @@ -1291,7 +1341,8 @@ export const registerRoutes = async ( secretV2BridgeDAL, secretVersionV2TagBridgeDAL: secretVersionTagV2BridgeDAL, secretVersionV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const secretRotationQueue = secretRotationQueueFactory({ @@ -1303,6 +1354,7 @@ export const registerRoutes = async ( projectBotService, secretVersionV2BridgeDAL, secretV2BridgeDAL, + folderCommitService, kmsService }); @@ -1454,6 +1506,15 @@ export const registerRoutes = async ( permissionService }); + const pitService = pitServiceFactory({ + folderCommitService, + secretService, + folderService, + permissionService, + folderDAL, + projectEnvDAL + }); + const identityOidcAuthService = identityOidcAuthServiceFactory({ identityOidcAuthDAL, identityOrgMembershipDAL, @@ -1516,7 +1577,9 @@ export const registerRoutes = async ( dynamicSecretProviders, folderDAL, licenseService, - kmsService + kmsService, + userDAL, + identityDAL }); const dailyResourceCleanUp = dailyResourceCleanUpQueueServiceFactory({ auditLogDAL, @@ -1595,7 +1658,9 @@ export const registerRoutes = async ( secretDAL: secretV2BridgeDAL, queueService, secretV2BridgeService, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService, + folderVersionDAL }); const migrationService = externalMigrationServiceFactory({ @@ -1705,6 +1770,7 @@ export const registerRoutes = async ( auditLogService, secretV2BridgeDAL, secretTagDAL, + folderCommitService, secretVersionTagV2BridgeDAL, secretVersionV2BridgeDAL, keyStore, @@ -1893,6 +1959,7 @@ export const registerRoutes = async ( certificateTemplate: certificateTemplateService, certificateAuthorityCrl: certificateAuthorityCrlService, certificateEst: certificateEstService, + pit: pitService, pkiAlert: pkiAlertService, pkiCollection: pkiCollectionService, pkiSubscriber: pkiSubscriberService, @@ -1927,6 +1994,7 @@ export const registerRoutes = async ( microsoftTeams: microsoftTeamsService, assumePrivileges: assumePrivilegeService, githubOrgSync: githubOrgSyncConfigService, + folderCommit: folderCommitService, secretScanningV2: secretScanningV2Service }); diff --git a/backend/src/server/routes/sanitizedSchemas.ts b/backend/src/server/routes/sanitizedSchemas.ts index a26293ac8..ce51b1079 100644 --- a/backend/src/server/routes/sanitizedSchemas.ts +++ b/backend/src/server/routes/sanitizedSchemas.ts @@ -262,7 +262,8 @@ export const SanitizedProjectSchema = ProjectsSchema.pick({ kmsCertificateKeyId: true, auditLogsRetentionDays: true, hasDeleteProtection: true, - secretSharing: true + secretSharing: true, + showSnapshotsLegacy: true }); export const SanitizedTagSchema = SecretTagsSchema.pick({ diff --git a/backend/src/server/routes/v1/app-connection-routers/gcp-connection-router.ts b/backend/src/server/routes/v1/app-connection-routers/gcp-connection-router.ts index f92d5e668..c88308c64 100644 --- a/backend/src/server/routes/v1/app-connection-routers/gcp-connection-router.ts +++ b/backend/src/server/routes/v1/app-connection-routers/gcp-connection-router.ts @@ -45,4 +45,37 @@ export const registerGcpConnectionRouter = async (server: FastifyZodProvider) => return projects; } }); + + server.route({ + method: "GET", + url: `/:connectionId/secret-manager-project-locations`, + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + connectionId: z.string().uuid() + }), + querystring: z.object({ + projectId: z.string() + }), + response: { + 200: z.object({ displayName: z.string(), locationId: z.string() }).array() + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const { + params: { connectionId }, + query: { projectId } + } = req; + + const locations = await server.services.appConnection.gcp.listSecretManagerProjectLocations( + { connectionId, projectId }, + req.permission + ); + + return locations; + } + }); }; diff --git a/backend/src/server/routes/v1/identity-kubernetes-auth-router.ts b/backend/src/server/routes/v1/identity-kubernetes-auth-router.ts index d9ef62087..879310790 100644 --- a/backend/src/server/routes/v1/identity-kubernetes-auth-router.ts +++ b/backend/src/server/routes/v1/identity-kubernetes-auth-router.ts @@ -8,6 +8,7 @@ import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; import { TIdentityTrustedIp } from "@app/services/identity/identity-types"; +import { IdentityKubernetesAuthTokenReviewMode } from "@app/services/identity-kubernetes-auth/identity-kubernetes-auth-types"; import { isSuperAdmin } from "@app/services/super-admin/super-admin-fns"; const IdentityKubernetesAuthResponseSchema = IdentityKubernetesAuthsSchema.pick({ @@ -18,6 +19,7 @@ const IdentityKubernetesAuthResponseSchema = IdentityKubernetesAuthsSchema.pick( accessTokenTrustedIps: true, createdAt: true, updatedAt: true, + tokenReviewMode: true, identityId: true, kubernetesHost: true, allowedNamespaces: true, @@ -106,17 +108,21 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide .string() .trim() .min(1) + .nullable() .describe(KUBERNETES_AUTH.ATTACH.kubernetesHost) .refine( - (val) => - characterValidator([ + (val) => { + if (val === null) return true; + + return characterValidator([ CharacterType.Alphabets, CharacterType.Numbers, CharacterType.Colon, CharacterType.Period, CharacterType.ForwardSlash, CharacterType.Hyphen - ])(val), + ])(val); + }, { message: "Kubernetes host must only contain alphabets, numbers, colons, periods, hyphen, and forward slashes." @@ -124,6 +130,10 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide ), caCert: z.string().trim().default("").describe(KUBERNETES_AUTH.ATTACH.caCert), tokenReviewerJwt: z.string().trim().optional().describe(KUBERNETES_AUTH.ATTACH.tokenReviewerJwt), + tokenReviewMode: z + .nativeEnum(IdentityKubernetesAuthTokenReviewMode) + .default(IdentityKubernetesAuthTokenReviewMode.Api) + .describe(KUBERNETES_AUTH.ATTACH.tokenReviewMode), allowedNamespaces: z.string().describe(KUBERNETES_AUTH.ATTACH.allowedNamespaces), // TODO: validation allowedNames: z.string().describe(KUBERNETES_AUTH.ATTACH.allowedNames), allowedAudience: z.string().describe(KUBERNETES_AUTH.ATTACH.allowedAudience), @@ -157,10 +167,30 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide .default(0) .describe(KUBERNETES_AUTH.ATTACH.accessTokenNumUsesLimit) }) - .refine( - (val) => val.accessTokenTTL <= val.accessTokenMaxTTL, - "Access Token TTL cannot be greater than Access Token Max TTL." - ), + .superRefine((data, ctx) => { + if (data.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api && !data.kubernetesHost) { + ctx.addIssue({ + path: ["kubernetesHost"], + code: z.ZodIssueCode.custom, + message: "When token review mode is set to API, a Kubernetes host must be provided" + }); + } + if (data.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Gateway && !data.gatewayId) { + ctx.addIssue({ + path: ["gatewayId"], + code: z.ZodIssueCode.custom, + message: "When token review mode is set to Gateway, a gateway must be selected" + }); + } + + if (data.accessTokenTTL > data.accessTokenMaxTTL) { + ctx.addIssue({ + path: ["accessTokenTTL"], + code: z.ZodIssueCode.custom, + message: "Access Token TTL cannot be greater than Access Token Max TTL." + }); + } + }), response: { 200: z.object({ identityKubernetesAuth: IdentityKubernetesAuthResponseSchema @@ -185,7 +215,7 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide type: EventType.ADD_IDENTITY_KUBERNETES_AUTH, metadata: { identityId: identityKubernetesAuth.identityId, - kubernetesHost: identityKubernetesAuth.kubernetesHost, + kubernetesHost: identityKubernetesAuth.kubernetesHost ?? "", allowedNamespaces: identityKubernetesAuth.allowedNamespaces, allowedNames: identityKubernetesAuth.allowedNames, accessTokenTTL: identityKubernetesAuth.accessTokenTTL, @@ -225,6 +255,7 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide .string() .trim() .min(1) + .nullable() .optional() .describe(KUBERNETES_AUTH.UPDATE.kubernetesHost) .refine( @@ -247,6 +278,10 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide ), caCert: z.string().trim().optional().describe(KUBERNETES_AUTH.UPDATE.caCert), tokenReviewerJwt: z.string().trim().nullable().optional().describe(KUBERNETES_AUTH.UPDATE.tokenReviewerJwt), + tokenReviewMode: z + .nativeEnum(IdentityKubernetesAuthTokenReviewMode) + .optional() + .describe(KUBERNETES_AUTH.UPDATE.tokenReviewMode), allowedNamespaces: z.string().optional().describe(KUBERNETES_AUTH.UPDATE.allowedNamespaces), // TODO: validation allowedNames: z.string().optional().describe(KUBERNETES_AUTH.UPDATE.allowedNames), allowedAudience: z.string().optional().describe(KUBERNETES_AUTH.UPDATE.allowedAudience), @@ -280,10 +315,26 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide .optional() .describe(KUBERNETES_AUTH.UPDATE.accessTokenMaxTTL) }) - .refine( - (val) => (val.accessTokenMaxTTL && val.accessTokenTTL ? val.accessTokenTTL <= val.accessTokenMaxTTL : true), - "Access Token TTL cannot be greater than Access Token Max TTL." - ), + .superRefine((data, ctx) => { + if ( + data.tokenReviewMode && + data.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Gateway && + !data.gatewayId + ) { + ctx.addIssue({ + path: ["gatewayId"], + code: z.ZodIssueCode.custom, + message: "When token review mode is set to Gateway, a gateway must be selected" + }); + } + if (data.accessTokenMaxTTL && data.accessTokenTTL ? data.accessTokenTTL > data.accessTokenMaxTTL : false) { + ctx.addIssue({ + path: ["accessTokenTTL"], + code: z.ZodIssueCode.custom, + message: "Access Token TTL cannot be greater than Access Token Max TTL." + }); + } + }), response: { 200: z.object({ identityKubernetesAuth: IdentityKubernetesAuthResponseSchema @@ -307,7 +358,7 @@ export const registerIdentityKubernetesRouter = async (server: FastifyZodProvide type: EventType.UPDATE_IDENTITY_KUBENETES_AUTH, metadata: { identityId: identityKubernetesAuth.identityId, - kubernetesHost: identityKubernetesAuth.kubernetesHost, + kubernetesHost: identityKubernetesAuth.kubernetesHost ?? "", allowedNamespaces: identityKubernetesAuth.allowedNamespaces, allowedNames: identityKubernetesAuth.allowedNames, accessTokenTTL: identityKubernetesAuth.accessTokenTTL, diff --git a/backend/src/server/routes/v1/invite-org-router.ts b/backend/src/server/routes/v1/invite-org-router.ts index 77ae0e627..525d51913 100644 --- a/backend/src/server/routes/v1/invite-org-router.ts +++ b/backend/src/server/routes/v1/invite-org-router.ts @@ -1,7 +1,7 @@ import { z } from "zod"; import { OrgMembershipRole, ProjectMembershipRole, UsersSchema } from "@app/db/schemas"; -import { inviteUserRateLimit } from "@app/server/config/rateLimiter"; +import { inviteUserRateLimit, smtpRateLimit } from "@app/server/config/rateLimiter"; import { getTelemetryDistinctId } from "@app/server/lib/telemetry"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { ActorType, AuthMode } from "@app/services/auth/auth-type"; @@ -11,7 +11,7 @@ export const registerInviteOrgRouter = async (server: FastifyZodProvider) => { server.route({ url: "/signup", config: { - rateLimit: inviteUserRateLimit + rateLimit: smtpRateLimit() }, method: "POST", schema: { @@ -81,7 +81,10 @@ export const registerInviteOrgRouter = async (server: FastifyZodProvider) => { server.route({ url: "/signup-resend", config: { - rateLimit: inviteUserRateLimit + rateLimit: smtpRateLimit({ + keyGenerator: (req) => + (req.body as { membershipId?: string })?.membershipId?.trim().substring(0, 100) ?? req.realIp + }) }, method: "POST", schema: { diff --git a/backend/src/server/routes/v1/org-admin-router.ts b/backend/src/server/routes/v1/org-admin-router.ts index cc0543d4c..d4b1ee188 100644 --- a/backend/src/server/routes/v1/org-admin-router.ts +++ b/backend/src/server/routes/v1/org-admin-router.ts @@ -2,9 +2,9 @@ import { z } from "zod"; import { ProjectMembershipsSchema } from "@app/db/schemas"; import { EventType } from "@app/ee/services/audit-log/audit-log-types"; -import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; +import { readLimit, smtpRateLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; -import { AuthMode } from "@app/services/auth/auth-type"; +import { ActorType, AuthMode } from "@app/services/auth/auth-type"; import { SanitizedProjectSchema } from "../sanitizedSchemas"; @@ -47,7 +47,9 @@ export const registerOrgAdminRouter = async (server: FastifyZodProvider) => { method: "POST", url: "/projects/:projectId/grant-admin-access", config: { - rateLimit: writeLimit + rateLimit: smtpRateLimit({ + keyGenerator: (req) => (req.auth.actor === ActorType.USER ? req.auth.userId : req.realIp) + }) }, schema: { params: z.object({ diff --git a/backend/src/server/routes/v1/password-router.ts b/backend/src/server/routes/v1/password-router.ts index 724468e02..eeb730f29 100644 --- a/backend/src/server/routes/v1/password-router.ts +++ b/backend/src/server/routes/v1/password-router.ts @@ -2,10 +2,10 @@ import { z } from "zod"; import { BackupPrivateKeySchema, UsersSchema } from "@app/db/schemas"; import { getConfig } from "@app/lib/config/env"; -import { authRateLimit } from "@app/server/config/rateLimiter"; +import { authRateLimit, smtpRateLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { validateSignUpAuthorization } from "@app/services/auth/auth-fns"; -import { AuthMode } from "@app/services/auth/auth-type"; +import { ActorType, AuthMode } from "@app/services/auth/auth-type"; import { UserEncryption } from "@app/services/user/user-types"; export const registerPasswordRouter = async (server: FastifyZodProvider) => { @@ -80,7 +80,9 @@ export const registerPasswordRouter = async (server: FastifyZodProvider) => { method: "POST", url: "/email/password-reset", config: { - rateLimit: authRateLimit + rateLimit: smtpRateLimit({ + keyGenerator: (req) => (req.body as { email?: string })?.email?.trim().substring(0, 100) ?? req.realIp + }) }, schema: { body: z.object({ @@ -224,7 +226,9 @@ export const registerPasswordRouter = async (server: FastifyZodProvider) => { method: "POST", url: "/email/password-setup", config: { - rateLimit: authRateLimit + rateLimit: smtpRateLimit({ + keyGenerator: (req) => (req.auth.actor === ActorType.USER ? req.auth.userId : req.realIp) + }) }, schema: { response: { @@ -233,6 +237,7 @@ export const registerPasswordRouter = async (server: FastifyZodProvider) => { }) } }, + onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { await server.services.password.sendPasswordSetupEmail(req.permission); @@ -267,6 +272,7 @@ export const registerPasswordRouter = async (server: FastifyZodProvider) => { }) } }, + onRequest: verifyAuth([AuthMode.JWT]), handler: async (req, res) => { await server.services.password.setupPassword(req.body, req.permission); diff --git a/backend/src/server/routes/v1/project-router.ts b/backend/src/server/routes/v1/project-router.ts index 2a868864e..cc94adede 100644 --- a/backend/src/server/routes/v1/project-router.ts +++ b/backend/src/server/routes/v1/project-router.ts @@ -376,7 +376,8 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { }) .optional() .describe(PROJECTS.UPDATE.slug), - secretSharing: z.boolean().optional().describe(PROJECTS.UPDATE.secretSharing) + secretSharing: z.boolean().optional().describe(PROJECTS.UPDATE.secretSharing), + showSnapshotsLegacy: z.boolean().optional().describe(PROJECTS.UPDATE.showSnapshotsLegacy) }), response: { 200: z.object({ @@ -397,7 +398,8 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { autoCapitalization: req.body.autoCapitalization, hasDeleteProtection: req.body.hasDeleteProtection, slug: req.body.slug, - secretSharing: req.body.secretSharing + secretSharing: req.body.secretSharing, + showSnapshotsLegacy: req.body.showSnapshotsLegacy }, actorAuthMethod: req.permission.authMethod, actorId: req.permission.id, diff --git a/backend/src/server/routes/v2/group-project-router.ts b/backend/src/server/routes/v2/group-project-router.ts index 5a081a3d9..d07e3bd8b 100644 --- a/backend/src/server/routes/v2/group-project-router.ts +++ b/backend/src/server/routes/v2/group-project-router.ts @@ -4,9 +4,11 @@ import { GroupProjectMembershipsSchema, GroupsSchema, ProjectMembershipRole, - ProjectUserMembershipRolesSchema + ProjectUserMembershipRolesSchema, + UsersSchema } from "@app/db/schemas"; -import { ApiDocsTags, PROJECTS } from "@app/lib/api-docs"; +import { EFilterReturnedUsers } from "@app/ee/services/group/group-types"; +import { ApiDocsTags, GROUPS, PROJECTS } from "@app/lib/api-docs"; import { ms } from "@app/lib/ms"; import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; @@ -301,4 +303,61 @@ export const registerGroupProjectRouter = async (server: FastifyZodProvider) => return { groupMembership }; } }); + + server.route({ + method: "GET", + url: "/:projectId/groups/:groupId/users", + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + config: { + rateLimit: readLimit + }, + schema: { + hide: false, + tags: [ApiDocsTags.ProjectGroups], + description: "Return project group users", + params: z.object({ + projectId: z.string().trim().describe(GROUPS.LIST_USERS.projectId), + groupId: z.string().trim().describe(GROUPS.LIST_USERS.id) + }), + querystring: z.object({ + offset: z.coerce.number().min(0).max(100).default(0).describe(GROUPS.LIST_USERS.offset), + limit: z.coerce.number().min(1).max(100).default(10).describe(GROUPS.LIST_USERS.limit), + username: z.string().trim().optional().describe(GROUPS.LIST_USERS.username), + search: z.string().trim().optional().describe(GROUPS.LIST_USERS.search), + filter: z.nativeEnum(EFilterReturnedUsers).optional().describe(GROUPS.LIST_USERS.filterUsers) + }), + response: { + 200: z.object({ + users: UsersSchema.pick({ + email: true, + username: true, + firstName: true, + lastName: true, + id: true + }) + .merge( + z.object({ + isPartOfGroup: z.boolean(), + joinedGroupAt: z.date().nullable() + }) + ) + .array(), + totalCount: z.number() + }) + } + }, + handler: async (req) => { + const { users, totalCount } = await server.services.groupProject.listProjectGroupUsers({ + id: req.params.groupId, + projectId: req.params.projectId, + actor: req.permission.type, + actorId: req.permission.id, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + ...req.query + }); + + return { users, totalCount }; + } + }); }; diff --git a/backend/src/server/routes/v2/user-router.ts b/backend/src/server/routes/v2/user-router.ts index 027f527fc..bbd566334 100644 --- a/backend/src/server/routes/v2/user-router.ts +++ b/backend/src/server/routes/v2/user-router.ts @@ -2,7 +2,7 @@ import { z } from "zod"; import { AuthTokenSessionsSchema, UserEncryptionKeysSchema, UsersSchema } from "@app/db/schemas"; import { ApiKeysSchema } from "@app/db/schemas/api-keys"; -import { authRateLimit, readLimit, writeLimit } from "@app/server/config/rateLimiter"; +import { authRateLimit, readLimit, smtpRateLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMethod, AuthMode, MfaMethod } from "@app/services/auth/auth-type"; import { sanitizedOrganizationSchema } from "@app/services/org/org-schema"; @@ -12,7 +12,9 @@ export const registerUserRouter = async (server: FastifyZodProvider) => { method: "POST", url: "/me/emails/code", config: { - rateLimit: authRateLimit + rateLimit: smtpRateLimit({ + keyGenerator: (req) => (req.body as { username?: string })?.username?.trim().substring(0, 100) ?? req.realIp + }) }, schema: { body: z.object({ diff --git a/backend/src/server/routes/v3/signup-router.ts b/backend/src/server/routes/v3/signup-router.ts index 552253cde..c249e7dbe 100644 --- a/backend/src/server/routes/v3/signup-router.ts +++ b/backend/src/server/routes/v3/signup-router.ts @@ -3,7 +3,7 @@ import { z } from "zod"; import { UsersSchema } from "@app/db/schemas"; import { getConfig } from "@app/lib/config/env"; import { ForbiddenRequestError } from "@app/lib/errors"; -import { authRateLimit } from "@app/server/config/rateLimiter"; +import { authRateLimit, smtpRateLimit } from "@app/server/config/rateLimiter"; import { GenericResourceNameSchema } from "@app/server/lib/schemas"; import { getServerCfg } from "@app/services/super-admin/super-admin-service"; import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types"; @@ -13,7 +13,9 @@ export const registerSignupRouter = async (server: FastifyZodProvider) => { url: "/email/signup", method: "POST", config: { - rateLimit: authRateLimit + rateLimit: smtpRateLimit({ + keyGenerator: (req) => (req.body as { email?: string })?.email?.trim().substring(0, 100) ?? req.realIp + }) }, schema: { body: z.object({ diff --git a/backend/src/services/app-connection/gcp/gcp-connection-fns.ts b/backend/src/services/app-connection/gcp/gcp-connection-fns.ts index 8bde74062..d1533b696 100644 --- a/backend/src/services/app-connection/gcp/gcp-connection-fns.ts +++ b/backend/src/services/app-connection/gcp/gcp-connection-fns.ts @@ -11,8 +11,10 @@ import { AppConnection } from "../app-connection-enums"; import { GcpConnectionMethod } from "./gcp-connection-enums"; import { GCPApp, + GCPGetProjectLocationsRes, GCPGetProjectsRes, GCPGetServiceRes, + GCPLocation, TGcpConnection, TGcpConnectionConfig } from "./gcp-connection-types"; @@ -145,6 +147,45 @@ export const getGcpSecretManagerProjects = async (appConnection: TGcpConnection) return projects; }; +export const getGcpSecretManagerProjectLocations = async (projectId: string, appConnection: TGcpConnection) => { + const accessToken = await getGcpConnectionAuthToken(appConnection); + + let gcpLocations: GCPLocation[] = []; + + const pageSize = 100; + let pageToken: string | undefined; + let hasMorePages = true; + + while (hasMorePages) { + const params = new URLSearchParams({ + pageSize: String(pageSize), + ...(pageToken ? { pageToken } : {}) + }); + + // eslint-disable-next-line no-await-in-loop + const { data } = await request.get( + `${IntegrationUrls.GCP_SECRET_MANAGER_URL}/v1/projects/${projectId}/locations`, + { + params, + headers: { + Authorization: `Bearer ${accessToken}`, + "Accept-Encoding": "application/json" + } + } + ); + + gcpLocations = gcpLocations.concat(data.locations); + + if (!data.nextPageToken) { + hasMorePages = false; + } + + pageToken = data.nextPageToken; + } + + return gcpLocations.sort((a, b) => a.displayName.localeCompare(b.displayName)); +}; + export const validateGcpConnectionCredentials = async (appConnection: TGcpConnectionConfig) => { // Check if provided service account email suffix matches organization ID. // We do this to mitigate confused deputy attacks in multi-tenant instances diff --git a/backend/src/services/app-connection/gcp/gcp-connection-service.ts b/backend/src/services/app-connection/gcp/gcp-connection-service.ts index 96b795a8f..74f2ab2c4 100644 --- a/backend/src/services/app-connection/gcp/gcp-connection-service.ts +++ b/backend/src/services/app-connection/gcp/gcp-connection-service.ts @@ -1,8 +1,8 @@ import { OrgServiceActor } from "@app/lib/types"; import { AppConnection } from "../app-connection-enums"; -import { getGcpSecretManagerProjects } from "./gcp-connection-fns"; -import { TGcpConnection } from "./gcp-connection-types"; +import { getGcpSecretManagerProjectLocations, getGcpSecretManagerProjects } from "./gcp-connection-fns"; +import { TGcpConnection, TGetGCPProjectLocationsDTO } from "./gcp-connection-types"; type TGetAppConnectionFunc = ( app: AppConnection, @@ -23,7 +23,23 @@ export const gcpConnectionService = (getAppConnection: TGetAppConnectionFunc) => } }; + const listSecretManagerProjectLocations = async ( + { connectionId, projectId }: TGetGCPProjectLocationsDTO, + actor: OrgServiceActor + ) => { + const appConnection = await getAppConnection(AppConnection.GCP, connectionId, actor); + + try { + const locations = await getGcpSecretManagerProjectLocations(projectId, appConnection); + + return locations; + } catch (error) { + return []; + } + }; + return { - listSecretManagerProjects + listSecretManagerProjects, + listSecretManagerProjectLocations }; }; diff --git a/backend/src/services/app-connection/gcp/gcp-connection-types.ts b/backend/src/services/app-connection/gcp/gcp-connection-types.ts index 2bb518820..4dc4bd131 100644 --- a/backend/src/services/app-connection/gcp/gcp-connection-types.ts +++ b/backend/src/services/app-connection/gcp/gcp-connection-types.ts @@ -38,6 +38,22 @@ export type GCPGetProjectsRes = { nextPageToken?: string; }; +export type GCPLocation = { + name: string; + locationId: string; + displayName: string; +}; + +export type GCPGetProjectLocationsRes = { + locations: GCPLocation[]; + nextPageToken?: string; +}; + +export type TGetGCPProjectLocationsDTO = { + projectId: string; + connectionId: string; +}; + export type GCPGetServiceRes = { name: string; parent: string; diff --git a/backend/src/services/auth/auth-login-service.ts b/backend/src/services/auth/auth-login-service.ts index bee85b14c..64ba573d5 100644 --- a/backend/src/services/auth/auth-login-service.ts +++ b/backend/src/services/auth/auth-login-service.ts @@ -397,7 +397,7 @@ export const authLoginServiceFactory = ({ // Check if the user actually has access to the specified organization. const userOrgs = await orgDAL.findAllOrgsByUserId(user.id); - const hasOrganizationMembership = userOrgs.some((org) => org.id === organizationId); + const hasOrganizationMembership = userOrgs.some((org) => org.id === organizationId && org.userStatus !== "invited"); const selectedOrg = await orgDAL.findById(organizationId); if (!hasOrganizationMembership) { diff --git a/backend/src/services/external-migration/external-migration-fns.ts b/backend/src/services/external-migration/external-migration-fns.ts index 856b39012..018d7bd43 100644 --- a/backend/src/services/external-migration/external-migration-fns.ts +++ b/backend/src/services/external-migration/external-migration-fns.ts @@ -10,6 +10,7 @@ import { chunkArray } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; +import { CommitType, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { KmsDataKey } from "../kms/kms-types"; import { TProjectDALFactory } from "../project/project-dal"; @@ -18,6 +19,7 @@ import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { fnSecretBulkInsert, getAllSecretReferences } from "../secret-v2-bridge/secret-v2-bridge-fns"; @@ -42,6 +44,8 @@ export type TImportDataIntoInfisicalDTO = { projectService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; + folderCommitService: Pick; + folderVersionDAL: Pick; input: TImportInfisicalDataCreate; }; @@ -507,6 +511,8 @@ export const importDataIntoInfisicalFn = async ({ secretVersionTagDAL, folderDAL, resourceMetadataDAL, + folderVersionDAL, + folderCommitService, input: { data, actor, actorId, actorOrgId, actorAuthMethod } }: TImportDataIntoInfisicalDTO) => { // Import data to infisical @@ -599,6 +605,36 @@ export const importDataIntoInfisicalFn = async ({ tx ); + const newFolderVersion = await folderVersionDAL.create( + { + name: newFolder.name, + envId: newFolder.envId, + version: newFolder.version, + folderId: newFolder.id + }, + tx + ); + + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Changed by external migration", + folderId: parentEnv.rootFolderId, + changes: [ + { + type: CommitType.ADD, + folderVersionId: newFolderVersion.id + } + ] + }, + tx + ); + originalToNewFolderId.set(folder.id, { folderId: newFolder.id, projectId: parentEnv.projectId @@ -772,6 +808,7 @@ export const importDataIntoInfisicalFn = async ({ secretVersionDAL, secretTagDAL, secretVersionTagDAL, + folderCommitService, actor: { type: actor, actorId diff --git a/backend/src/services/external-migration/external-migration-queue.ts b/backend/src/services/external-migration/external-migration-queue.ts index 8aa46b94c..66f2c73e7 100644 --- a/backend/src/services/external-migration/external-migration-queue.ts +++ b/backend/src/services/external-migration/external-migration-queue.ts @@ -3,6 +3,7 @@ import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectServiceFactory } from "../project/project-service"; @@ -10,6 +11,7 @@ import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectEnvServiceFactory } from "../project-env/project-env-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service"; @@ -36,6 +38,8 @@ export type TExternalMigrationQueueFactoryDep = { projectService: Pick; projectEnvService: Pick; secretV2BridgeService: Pick; + folderCommitService: Pick; + folderVersionDAL: Pick; resourceMetadataDAL: Pick; }; @@ -56,6 +60,8 @@ export const externalMigrationQueueFactory = ({ secretTagDAL, secretVersionTagDAL, folderDAL, + folderCommitService, + folderVersionDAL, resourceMetadataDAL }: TExternalMigrationQueueFactoryDep) => { const startImport = async (dto: { @@ -114,6 +120,8 @@ export const externalMigrationQueueFactory = ({ projectService, projectEnvService, secretV2BridgeService, + folderCommitService, + folderVersionDAL, resourceMetadataDAL }); diff --git a/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts new file mode 100644 index 000000000..3e2d09d0b --- /dev/null +++ b/backend/src/services/folder-checkpoint-resources/folder-checkpoint-resources-dal.ts @@ -0,0 +1,118 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { + TableName, + TFolderCheckpointResources, + TFolderCheckpoints, + TSecretFolderVersions, + TSecretVersionsV2 +} from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCheckpointResourcesDALFactory = ReturnType; + +export type ResourceWithCheckpointInfo = TFolderCheckpointResources & { + folderCommitId: string; +}; + +export const folderCheckpointResourcesDALFactory = (db: TDbClient) => { + const folderCheckpointResourcesOrm = ormify(db, TableName.FolderCheckpointResources); + + const findByCheckpointId = async ( + folderCheckpointId: string, + tx?: Knex + ): Promise< + (TFolderCheckpointResources & { + referencedSecretId?: string; + referencedFolderId?: string; + folderName?: string; + folderVersion?: string; + secretKey?: string; + secretVersion?: string; + })[] + > => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCheckpointResources) + .where({ folderCheckpointId }) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.FolderCheckpointResources}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.FolderCheckpointResources}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select(selectAllTableCols(TableName.FolderCheckpointResources)) + .select( + db.ref("secretId").withSchema(TableName.SecretVersionV2).as("referencedSecretId"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("referencedFolderId"), + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"), + db.ref("version").withSchema(TableName.SecretFolderVersion).as("folderVersion"), + db.ref("key").withSchema(TableName.SecretVersionV2).as("secretKey"), + db.ref("version").withSchema(TableName.SecretVersionV2).as("secretVersion") + ); + return docs.map((doc) => ({ + ...doc, + folderVersion: doc.folderVersion?.toString(), + secretVersion: doc.secretVersion?.toString() + })); + } catch (error) { + throw new DatabaseError({ error, name: "FindByCheckpointId" }); + } + }; + + const findBySecretVersionId = async (secretVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCheckpointResources & Pick + >(TableName.FolderCheckpointResources) + .where({ secretVersionId }) + .select(selectAllTableCols(TableName.FolderCheckpointResources)) + .join( + TableName.FolderCheckpoint, + `${TableName.FolderCheckpointResources}.folderCheckpointId`, + `${TableName.FolderCheckpoint}.id` + ) + .select( + db.ref("folderCommitId").withSchema(TableName.FolderCheckpoint), + db.ref("createdAt").withSchema(TableName.FolderCheckpoint) + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindBySecretVersionId" }); + } + }; + + const findByFolderVersionId = async (folderVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCheckpointResources & Pick + >(TableName.FolderCheckpointResources) + .where({ folderVersionId }) + .select(selectAllTableCols(TableName.FolderCheckpointResources)) + .join( + TableName.FolderCheckpoint, + `${TableName.FolderCheckpointResources}.folderCheckpointId`, + `${TableName.FolderCheckpoint}.id` + ) + .select( + db.ref("folderCommitId").withSchema(TableName.FolderCheckpoint), + db.ref("createdAt").withSchema(TableName.FolderCheckpoint) + ); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderVersionId" }); + } + }; + + return { + ...folderCheckpointResourcesOrm, + findByCheckpointId, + findBySecretVersionId, + findByFolderVersionId + }; +}; diff --git a/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts new file mode 100644 index 000000000..51bba9cc0 --- /dev/null +++ b/backend/src/services/folder-checkpoint/folder-checkpoint-dal.ts @@ -0,0 +1,129 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderCheckpoints, TFolderCommits } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCheckpointDALFactory = ReturnType; + +type CheckpointWithCommitInfo = TFolderCheckpoints & { + actorMetadata: unknown; + actorType: string; + message?: string | null; + commitDate: Date; + folderId: string; +}; + +export const folderCheckpointDALFactory = (db: TDbClient) => { + const folderCheckpointOrm = ormify(db, TableName.FolderCheckpoint); + + const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCheckpoint) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderCommitId }, TableName.FolderCheckpoint)) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindByCommitId" }); + } + }; + + const findByFolderId = async (folderId: string, limit?: number, tx?: Knex): Promise => { + try { + let query = (tx || db.replicaNode())(TableName.FolderCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderId }, TableName.FolderCommit)) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .select( + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderCheckpoint}.createdAt`, "desc"); + + if (limit !== undefined) { + query = query.limit(limit); + } + + return await query; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderId" }); + } + }; + + const findLatestByFolderId = async (folderId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderId }, TableName.FolderCommit)) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .select( + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderCheckpoint}.createdAt`, "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestByFolderId" }); + } + }; + + const findNearestCheckpoint = async ( + folderCommitId: bigint, + folderId: string, + tx?: Knex + ): Promise<(CheckpointWithCommitInfo & { commitId: bigint }) | undefined> => { + try { + // Get the checkpoint with the highest commitId that's still less than or equal to our commit + const nearestCheckpoint = await (tx || db.replicaNode())(TableName.FolderCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + .where(`${TableName.FolderCommit}.folderId`, "=", folderId) + .where(`${TableName.FolderCommit}.commitId`, "<=", folderCommitId.toString()) + .select(selectAllTableCols(TableName.FolderCheckpoint)) + .select( + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("commitId").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit).as("commitDate"), + db.ref("folderId").withSchema(TableName.FolderCommit) + ) + .orderBy(`${TableName.FolderCommit}.commitId`, "desc") + .first(); + return nearestCheckpoint; + } catch (error) { + throw new DatabaseError({ error, name: "FindNearestCheckpoint" }); + } + }; + + return { + ...folderCheckpointOrm, + findByCommitId, + findByFolderId, + findLatestByFolderId, + findNearestCheckpoint + }; +}; diff --git a/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts new file mode 100644 index 000000000..2c30c5bcf --- /dev/null +++ b/backend/src/services/folder-commit-changes/folder-commit-changes-dal.ts @@ -0,0 +1,233 @@ +/* eslint-disable @typescript-eslint/no-misused-promises */ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { + TableName, + TFolderCommitChanges, + TFolderCommits, + TProjectEnvironments, + TSecretFolderVersions, + TSecretVersionsV2 +} from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCommitChangesDALFactory = ReturnType; + +// Base type with common fields +type BaseCommitChangeInfo = TFolderCommitChanges & { + actorMetadata: unknown; + actorType: string; + message?: string | null; + folderId: string; + createdAt: Date; +}; + +// Secret-specific change +export type SecretCommitChange = BaseCommitChangeInfo & { + resourceType: "secret"; + secretKey: string; + changeType: string; + secretVersionId?: string | null; + secretVersion: string; + secretId: string; + versions?: { + secretKey: string; + secretComment: string; + skipMultilineEncoding?: boolean | null; + secretReminderRepeatDays?: number | null; + secretReminderNote?: string | null; + metadata?: unknown; + tags?: string[] | null; + secretReminderRecipients?: string[] | null; + secretValue: string; + }[]; +}; + +// Folder-specific change +export type FolderCommitChange = BaseCommitChangeInfo & { + resourceType: "folder"; + folderName: string; + folderVersion: string; + folderChangeId: string; + versions?: { + version: string; + name?: string; + }[]; +}; + +// Discriminated union +export type CommitChangeWithCommitInfo = SecretCommitChange | FolderCommitChange; + +// Type guards +export const isSecretCommitChange = (change: CommitChangeWithCommitInfo): change is SecretCommitChange => + change.resourceType === "secret"; + +export const isFolderCommitChange = (change: CommitChangeWithCommitInfo): change is FolderCommitChange => + change.resourceType === "folder"; + +export const folderCommitChangesDALFactory = (db: TDbClient) => { + const folderCommitChangesOrm = ormify(db, TableName.FolderCommitChanges); + + const findByCommitId = async ( + folderCommitId: string, + projectId: string, + tx?: Knex + ): Promise => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCommitChanges) + .where(buildFindFilter({ folderCommitId }, TableName.FolderCommitChanges)) + .leftJoin( + TableName.FolderCommit, + `${TableName.FolderCommitChanges}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.FolderCommitChanges}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.FolderCommitChanges}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .leftJoin( + TableName.Environment, + `${TableName.FolderCommit}.envId`, + `${TableName.Environment}.id` + ) + .where((qb) => { + if (projectId) { + void qb.where(`${TableName.Environment}.projectId`, "=", projectId); + } + }) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .select( + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderChangeId"), + db.ref("version").withSchema(TableName.SecretFolderVersion).as("folderVersion"), + db.ref("key").withSchema(TableName.SecretVersionV2).as("secretKey"), + db.ref("version").withSchema(TableName.SecretVersionV2).as("secretVersion"), + db.ref("secretId").withSchema(TableName.SecretVersionV2), + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit), + db.ref("folderId").withSchema(TableName.FolderCommit) + ); + + return docs.map((doc) => { + // Determine if this is a secret or folder change based on populated fields + if (doc.secretKey && doc.secretVersion !== null && doc.secretId) { + return { + ...doc, + resourceType: "secret", + secretKey: doc.secretKey, + secretVersion: doc.secretVersion.toString(), + secretId: doc.secretId + } as SecretCommitChange; + } + return { + ...doc, + resourceType: "folder", + folderName: doc.folderName, + folderVersion: doc.folderVersion.toString(), + folderChangeId: doc.folderChangeId + } as FolderCommitChange; + }); + } catch (error) { + throw new DatabaseError({ error, name: "FindByCommitId" }); + } + }; + + const findBySecretVersionId = async (secretVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCommitChanges & + Pick + >(TableName.FolderCommitChanges) + .where(buildFindFilter({ secretVersionId }, TableName.FolderCommitChanges)) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.FolderCommitChanges}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .select( + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit), + db.ref("folderId").withSchema(TableName.FolderCommit), + db.ref("key").withSchema(TableName.SecretVersionV2).as("secretKey"), + db.ref("version").withSchema(TableName.SecretVersionV2).as("secretVersion"), + db.ref("secretId").withSchema(TableName.SecretVersionV2) + ); + + return docs + .filter((doc) => doc.secretKey && doc.secretVersion !== null && doc.secretId) + .map( + (doc): SecretCommitChange => ({ + ...doc, + resourceType: "secret", + secretKey: doc.secretKey, + secretVersion: doc.secretVersion.toString(), + secretId: doc.secretId + }) + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindBySecretVersionId" }); + } + }; + + const findByFolderVersionId = async (folderVersionId: string, tx?: Knex): Promise => { + try { + const docs = await (tx || db.replicaNode())< + TFolderCommitChanges & + Pick + >(TableName.FolderCommitChanges) + .where(buildFindFilter({ folderVersionId }, TableName.FolderCommitChanges)) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .join(TableName.FolderCommit, `${TableName.FolderCommitChanges}.folderCommitId`, `${TableName.FolderCommit}.id`) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.FolderCommitChanges}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select( + db.ref("actorMetadata").withSchema(TableName.FolderCommit), + db.ref("actorType").withSchema(TableName.FolderCommit), + db.ref("message").withSchema(TableName.FolderCommit), + db.ref("createdAt").withSchema(TableName.FolderCommit), + db.ref("folderId").withSchema(TableName.FolderCommit), + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("folderChangeId"), + db.ref("version").withSchema(TableName.SecretFolderVersion).as("folderVersion") + ); + + return docs + .filter((doc) => doc.folderName && doc.folderVersion !== null && doc.folderChangeId) + .map( + (doc): FolderCommitChange => ({ + ...doc, + resourceType: "folder", + folderName: doc.folderName, + folderVersion: doc.folderVersion!.toString(), + folderChangeId: doc.folderChangeId + }) + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderVersionId" }); + } + }; + + return { + ...folderCommitChangesOrm, + findByCommitId, + findBySecretVersionId, + findByFolderVersionId + }; +}; diff --git a/backend/src/services/folder-commit/folder-commit-dal.ts b/backend/src/services/folder-commit/folder-commit-dal.ts new file mode 100644 index 000000000..e95dbb839 --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-dal.ts @@ -0,0 +1,513 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { + TableName, + TFolderCommitChanges, + TFolderCommits, + TProjectEnvironments, + TSecretFolderVersions, + TSecretVersionsV2 +} from "@app/db/schemas"; +import { DatabaseError, NotFoundError } from "@app/lib/errors"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderCommitDALFactory = ReturnType; + +export const folderCommitDALFactory = (db: TDbClient) => { + const folderCommitOrm = ormify(db, TableName.FolderCommit); + const { delete: deleteOp, deleteById, ...restOfOrm } = folderCommitOrm; + + const findByFolderId = async (folderId: string, tx?: Knex): Promise => { + try { + const trx = tx || db.replicaNode(); + + // First, get all folder commits + const folderCommits = await trx(TableName.FolderCommit) + .where({ folderId }) + .select("*") + .orderBy("createdAt", "desc"); + + if (folderCommits.length === 0) return []; + + // Get all commit IDs + const commitIds = folderCommits.map((commit) => commit.id); + + // Then get all related changes + const changes = await trx(TableName.FolderCommitChanges).whereIn("folderCommitId", commitIds).select("*"); + + const changesMap = changes.reduce( + (acc, change) => { + const { folderCommitId } = change; + if (!acc[folderCommitId]) acc[folderCommitId] = []; + acc[folderCommitId].push(change); + return acc; + }, + {} as Record + ); + + return folderCommits.map((commit) => ({ + ...commit, + changes: changesMap[commit.id] || [] + })); + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderId" }); + } + }; + + const findLatestCommit = async ( + folderId: string, + projectId?: string, + tx?: Knex + ): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .leftJoin(TableName.Environment, `${TableName.FolderCommit}.envId`, `${TableName.Environment}.id`) + .where((qb) => { + if (projectId) { + void qb.where(`${TableName.Environment}.projectId`, "=", projectId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommit" }); + } + }; + + const findLatestCommitByFolderIds = async (folderIds: string[], tx?: Knex): Promise => { + try { + // First get max commitId for each folderId + const maxCommitIdSubquery = (tx || db.replicaNode())(TableName.FolderCommit) + .select("folderId") + .max("commitId as maxCommitId") + .whereIn("folderId", folderIds) + .groupBy("folderId"); + + // Join with main table to get complete records for each max commitId + const docs = await (tx || db.replicaNode())(TableName.FolderCommit) + .select(selectAllTableCols(TableName.FolderCommit)) + // eslint-disable-next-line func-names + .join(maxCommitIdSubquery.as("latest"), function () { + this.on(`${TableName.FolderCommit}.folderId`, "=", "latest.folderId").andOn( + `${TableName.FolderCommit}.commitId`, + "=", + "latest.maxCommitId" + ); + }); + + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommitByFolderIds" }); + } + }; + + const findLatestEnvCommit = async (envId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where(`${TableName.FolderCommit}.envId`, "=", envId) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommit" }); + } + }; + + const findMultipleLatestCommits = async (folderIds: string[], tx?: Knex): Promise => { + try { + const knexInstance = tx || db.replicaNode(); + + // Get the latest commitId for each folderId + const subquery = knexInstance(TableName.FolderCommit) + .whereIn("folderId", folderIds) + .groupBy("folderId") + .select("folderId") + .max("commitId as maxCommitId"); + + // Then fetch the complete rows matching those latest commits + const docs = await knexInstance(TableName.FolderCommit) + // eslint-disable-next-line func-names + .innerJoin(subquery.as("latest"), function () { + this.on(`${TableName.FolderCommit}.folderId`, "=", "latest.folderId").andOn( + `${TableName.FolderCommit}.commitId`, + "=", + "latest.maxCommitId" + ); + }) + .select(selectAllTableCols(TableName.FolderCommit)); + + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindMultipleLatestCommits" }); + } + }; + + const getNumberOfCommitsSince = async (folderId: string, folderCommitId: string, tx?: Knex): Promise => { + try { + const referencedCommit = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ id: folderCommitId }) + .select("commitId") + .first(); + + if (referencedCommit?.commitId) { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .where("commitId", ">", referencedCommit.commitId) + .count(); + return Number(doc?.[0].count); + } + return 0; + } catch (error) { + throw new DatabaseError({ error, name: "getNumberOfCommitsSince" }); + } + }; + + const getEnvNumberOfCommitsSince = async (envId: string, folderCommitId: string, tx?: Knex): Promise => { + try { + const referencedCommit = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ id: folderCommitId }) + .select("commitId") + .first(); + + if (referencedCommit?.commitId) { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where(`${TableName.FolderCommit}.envId`, "=", envId) + .where("commitId", ">", referencedCommit.commitId) + .count(); + return Number(doc?.[0].count); + } + return 0; + } catch (error) { + throw new DatabaseError({ error, name: "getNumberOfCommitsSince" }); + } + }; + + const findCommitsToRecreate = async ( + folderId: string, + targetCommitNumber: bigint, + checkpointCommitNumber: bigint, + tx?: Knex + ): Promise< + (TFolderCommits & { + changes: (TFolderCommitChanges & { + referencedSecretId?: string; + referencedFolderId?: string; + folderName?: string; + folderVersion?: string; + secretKey?: string; + secretVersion?: string; + })[]; + })[] + > => { + try { + // First get all the commits in the range + const commits = await (tx || db.replicaNode())(TableName.FolderCommit) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderId }, TableName.FolderCommit)) + .andWhere(`${TableName.FolderCommit}.commitId`, ">", checkpointCommitNumber.toString()) + .andWhere(`${TableName.FolderCommit}.commitId`, "<=", targetCommitNumber.toString()) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy(`${TableName.FolderCommit}.commitId`, "asc"); + + // If no commits found, return empty array + if (!commits.length) { + return []; + } + + // Get all the commit IDs + const commitIds = commits.map((commit) => commit.id); + + // Get all changes for these commits in a single query + const allChanges = await (tx || db.replicaNode())(TableName.FolderCommitChanges) + .whereIn(`${TableName.FolderCommitChanges}.folderCommitId`, commitIds) + .leftJoin( + TableName.SecretVersionV2, + `${TableName.FolderCommitChanges}.secretVersionId`, + `${TableName.SecretVersionV2}.id` + ) + .leftJoin( + TableName.SecretFolderVersion, + `${TableName.FolderCommitChanges}.folderVersionId`, + `${TableName.SecretFolderVersion}.id` + ) + .select(selectAllTableCols(TableName.FolderCommitChanges)) + .select( + db.ref("secretId").withSchema(TableName.SecretVersionV2).as("referencedSecretId"), + db.ref("folderId").withSchema(TableName.SecretFolderVersion).as("referencedFolderId"), + db.ref("name").withSchema(TableName.SecretFolderVersion).as("folderName"), + db.ref("version").withSchema(TableName.SecretFolderVersion).as("folderVersion"), + db.ref("key").withSchema(TableName.SecretVersionV2).as("secretKey"), + db.ref("version").withSchema(TableName.SecretVersionV2).as("secretVersion") + ); + + // Organize changes by commit ID + const changesByCommitId = allChanges.reduce( + (acc, change) => { + if (!acc[change.folderCommitId]) { + acc[change.folderCommitId] = []; + } + acc[change.folderCommitId].push(change); + return acc; + }, + {} as Record + ); + + // Attach changes to each commit + return commits.map((commit) => ({ + ...commit, + changes: changesByCommitId[commit.id] || [] + })); + } catch (error) { + throw new DatabaseError({ error, name: "FindCommitsToRecreate" }); + } + }; + + const findLatestCommitBetween = async ({ + folderId, + startCommitId, + endCommitId, + tx + }: { + folderId: string; + startCommitId?: string; + endCommitId: string; + tx?: Knex; + }): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where("commitId", "<=", endCommitId) + .where({ folderId }) + .where((qb) => { + if (startCommitId) { + void qb.where("commitId", ">=", startCommitId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommitBetween" }); + } + }; + + const findAllCommitsBetween = async ({ + envId, + startCommitId, + endCommitId, + tx + }: { + envId?: string; + startCommitId?: string; + endCommitId?: string; + tx?: Knex; + }): Promise => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCommit) + .where((qb) => { + if (envId) { + void qb.where(`${TableName.FolderCommit}.envId`, "=", envId); + } + if (startCommitId) { + void qb.where("commitId", ">=", startCommitId); + } + if (endCommitId) { + void qb.where("commitId", "<=", endCommitId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc"); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommitBetween" }); + } + }; + + const findAllFolderCommitsAfter = async ({ + envId, + startCommitId, + tx + }: { + envId?: string; + startCommitId?: string; + tx?: Knex; + }): Promise => { + try { + const docs = await (tx || db.replicaNode())(TableName.FolderCommit) + .where((qb) => { + if (envId) { + void qb.where(`${TableName.FolderCommit}.envId`, "=", envId); + } + if (startCommitId) { + void qb.where("commitId", ">=", startCommitId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc"); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestCommitBetween" }); + } + }; + + const findPreviousCommitTo = async ( + folderId: string, + commitId: string, + tx?: Knex + ): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .where("commitId", "<=", commitId) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindPreviousCommitTo" }); + } + }; + + const findById = async (id: string, tx?: Knex, projectId?: string): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ id }, TableName.FolderCommit)) + .leftJoin( + TableName.Environment, + `${TableName.FolderCommit}.envId`, + `${TableName.Environment}.id` + ) + .where((qb) => { + if (projectId) { + void qb.where(`${TableName.Environment}.projectId`, "=", projectId); + } + }) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + if (!doc) { + throw new NotFoundError({ + message: `Folder commit not found for ID ${id}` + }); + } + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindById" }); + } + }; + + const findByFolderIdPaginated = async ( + folderId: string, + options: { + offset?: number; + limit?: number; + search?: string; + sort?: "asc" | "desc"; + } = {}, + tx?: Knex + ): Promise<{ + commits: TFolderCommits[]; + total: number; + hasMore: boolean; + }> => { + try { + const { offset = 0, limit = 20, search, sort = "desc" } = options; + const trx = tx || db.replicaNode(); + + // Build base query + let baseQuery = trx(TableName.FolderCommit).where({ folderId }); + + // Add search functionality + if (search) { + baseQuery = baseQuery.where((qb) => { + void qb.whereILike("message", `%${search}%`); + }); + } + + // Get total count + const totalResult = await baseQuery.clone().count("*", { as: "count" }).first(); + const total = Number(totalResult?.count || 0); + + // Get paginated commits + const folderCommits = await baseQuery.select("*").orderBy("createdAt", sort).limit(limit).offset(offset); + + if (folderCommits.length === 0) { + return { commits: [], total, hasMore: false }; + } + + // Get all commit IDs for changes + const commitIds = folderCommits.map((commit) => commit.id); + + // Get all related changes + const changes = await trx(TableName.FolderCommitChanges).whereIn("folderCommitId", commitIds).select("*"); + + const changesMap = changes.reduce( + (acc, change) => { + const { folderCommitId } = change; + if (!acc[folderCommitId]) acc[folderCommitId] = []; + acc[folderCommitId].push(change); + return acc; + }, + {} as Record + ); + + const commitsWithChanges = folderCommits.map((commit) => ({ + ...commit, + changes: changesMap[commit.id] || [] + })); + + const hasMore = offset + limit < total; + + return { + commits: commitsWithChanges, + total, + hasMore + }; + } catch (error) { + throw new DatabaseError({ error, name: "FindByFolderIdPaginated" }); + } + }; + + const findCommitBefore = async ( + folderId: string, + commitId: bigint, + tx?: Knex + ): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderCommit) + .where({ folderId }) + .where("commitId", "<", commitId.toString()) + .select(selectAllTableCols(TableName.FolderCommit)) + .orderBy("commitId", "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindCommitBefore" }); + } + }; + + return { + ...restOfOrm, + findByFolderId, + findLatestCommit, + getNumberOfCommitsSince, + findCommitsToRecreate, + findMultipleLatestCommits, + findAllCommitsBetween, + findLatestCommitBetween, + findLatestEnvCommit, + getEnvNumberOfCommitsSince, + findLatestCommitByFolderIds, + findAllFolderCommitsAfter, + findPreviousCommitTo, + findById, + findByFolderIdPaginated, + findCommitBefore + }; +}; diff --git a/backend/src/services/folder-commit/folder-commit-queue.ts b/backend/src/services/folder-commit/folder-commit-queue.ts new file mode 100644 index 000000000..fcb348784 --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-queue.ts @@ -0,0 +1,282 @@ +import { Knex } from "knex"; + +import { TSecretFolders } from "@app/db/schemas"; +import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore"; +import { getConfig } from "@app/lib/config/env"; +import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; + +import { TFolderTreeCheckpointDALFactory } from "../folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { TFolderTreeCheckpointResourcesDALFactory } from "../folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TFolderCommitDALFactory } from "./folder-commit-dal"; + +// Define types for job data +type TCreateFolderTreeCheckpointDTO = { + envId: string; + failedToAcquireLockCount?: number; + folderCommitId?: string; +}; + +type TFolderCommitQueueServiceFactoryDep = { + queueService: TQueueServiceFactory; + keyStore: Pick; + folderTreeCheckpointDAL: Pick< + TFolderTreeCheckpointDALFactory, + "create" | "findLatestByEnvId" | "findNearestCheckpoint" + >; + folderTreeCheckpointResourcesDAL: Pick< + TFolderTreeCheckpointResourcesDALFactory, + "insertMany" | "findByTreeCheckpointId" + >; + folderCommitDAL: Pick< + TFolderCommitDALFactory, + "findLatestEnvCommit" | "getEnvNumberOfCommitsSince" | "findMultipleLatestCommits" | "findById" + >; + folderDAL: Pick; +}; + +export type TFolderCommitQueueServiceFactory = ReturnType; + +export const folderCommitQueueServiceFactory = ({ + queueService, + keyStore, + folderTreeCheckpointDAL, + folderTreeCheckpointResourcesDAL, + folderCommitDAL, + folderDAL +}: TFolderCommitQueueServiceFactoryDep) => { + const appCfg = getConfig(); + + // Helper function to calculate delay for requeuing + const getRequeueDelay = (failureCount?: number) => { + if (!failureCount) return 0; + + const baseDelay = 5000; + const maxDelay = 30000; + + const delay = Math.min(baseDelay * 2 ** failureCount, maxDelay); + const jitter = delay * (0.5 + Math.random() * 0.5); + + return jitter; + }; + + const scheduleTreeCheckpoint = async (payload: TCreateFolderTreeCheckpointDTO) => { + const { envId, failedToAcquireLockCount = 0 } = payload; + + // Create a unique jobId for each retry to prevent conflicts + const jobId = + failedToAcquireLockCount > 0 ? `${envId}-retry-${failedToAcquireLockCount}-${Date.now()}` : `${envId}`; + + await queueService.queue(QueueName.FolderTreeCheckpoint, QueueJobs.CreateFolderTreeCheckpoint, payload, { + jobId, + delay: getRequeueDelay(failedToAcquireLockCount), + backoff: { + type: "exponential", + delay: 3000 + }, + removeOnFail: { + count: 3 + }, + removeOnComplete: true + }); + }; + + // Sort folders by hierarchy (copied from the source code) + const sortFoldersByHierarchy = (folders: TSecretFolders[]) => { + const childrenMap = new Map(); + const allFolderIds = new Set(); + + folders.forEach((folder) => { + if (folder.id) allFolderIds.add(folder.id); + }); + + folders.forEach((folder) => { + if (folder.parentId) { + const children = childrenMap.get(folder.parentId) || []; + children.push(folder); + childrenMap.set(folder.parentId, children); + } + }); + + const rootFolders = folders.filter((folder) => !folder.parentId || !allFolderIds.has(folder.parentId)); + + const result = []; + let currentLevel = rootFolders; + + while (currentLevel.length > 0) { + result.push(...currentLevel); + + const nextLevel = []; + for (const folder of currentLevel) { + if (folder.id) { + const children = childrenMap.get(folder.id) || []; + nextLevel.push(...children); + } + } + + currentLevel = nextLevel; + } + + return result; + }; + + const createFolderTreeCheckpoint = async (jobData: TCreateFolderTreeCheckpointDTO, tx?: Knex) => { + const { envId, folderCommitId, failedToAcquireLockCount = 0 } = jobData; + + logger.info(`Folder tree checkpoint creation started [envId=${envId}] [attempt=${failedToAcquireLockCount + 1}]`); + + // First, try to clear any stale locks before attempting to acquire + if (failedToAcquireLockCount > 1) { + try { + await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId)); + logger.info(`Cleared potential stale lock for envId ${envId} before attempt ${failedToAcquireLockCount + 1}`); + } catch (error) { + // This is fine if it fails, we'll still try to acquire the lock + logger.info(`No stale lock found for envId ${envId}`); + } + } + + let lock: Awaited> | undefined; + + try { + // Attempt to acquire the lock with a shorter timeout for first attempts + const timeout = failedToAcquireLockCount > 3 ? 60 * 1000 : 15 * 1000; + + logger.info(`Attempting to acquire lock for envId=${envId} with timeout ${timeout}ms`); + + lock = await keyStore.acquireLock([KeyStorePrefixes.FolderTreeCheckpoint(envId)], timeout); + + logger.info(`Successfully acquired lock for envId=${envId}`); + } catch (e) { + logger.info( + `Failed to acquire lock for folder tree checkpoint [envId=${envId}] [attempt=${failedToAcquireLockCount + 1}]` + ); + + // Requeue with incremented failure count if under max attempts + if (failedToAcquireLockCount < 10) { + // Force a delay between retries + const nextRetryCount = failedToAcquireLockCount + 1; + + logger.info(`Scheduling retry #${nextRetryCount} for folder tree checkpoint [envId=${envId}]`); + + // Create a new job with incremented counter + await scheduleTreeCheckpoint({ + envId, + folderCommitId, + failedToAcquireLockCount: nextRetryCount + }); + } else { + // Max retries reached + logger.error(`Maximum lock acquisition attempts (10) reached for envId ${envId}. Giving up.`); + // Try to force-clear the lock for next time + try { + await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId)); + } catch (clearError) { + logger.error(clearError, `Failed to clear lock after maximum retries for envId=${envId}`); + } + } + return; + } + + if (!lock) { + logger.error(`Lock is undefined after acquisition for envId=${envId}. This should never happen.`); + return; + } + + try { + logger.info(`Processing tree checkpoint data for envId=${envId}`); + + const latestTreeCheckpoint = await folderTreeCheckpointDAL.findLatestByEnvId(envId, tx); + + let latestCommit; + if (folderCommitId) { + latestCommit = await folderCommitDAL.findById(folderCommitId, tx); + } else { + latestCommit = await folderCommitDAL.findLatestEnvCommit(envId, tx); + } + if (!latestCommit) { + logger.info(`Latest commit ID not found for envId ${envId}`); + return; + } + const latestCommitId = latestCommit.id; + + if (latestTreeCheckpoint) { + const commitsSinceLastCheckpoint = await folderCommitDAL.getEnvNumberOfCommitsSince( + envId, + latestTreeCheckpoint.folderCommitId, + tx + ); + if (commitsSinceLastCheckpoint < Number(appCfg.PIT_TREE_CHECKPOINT_WINDOW)) { + logger.info( + `Commits since last checkpoint ${commitsSinceLastCheckpoint} is less than ${appCfg.PIT_TREE_CHECKPOINT_WINDOW}` + ); + return; + } + } + + const folders = await folderDAL.findByEnvId(envId, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + const filteredFoldersIds = sortedFolders.filter((folder) => !folder.isReserved).map((folder) => folder.id); + + const folderCommits = await folderCommitDAL.findMultipleLatestCommits(filteredFoldersIds, tx); + const folderTreeCheckpoint = await folderTreeCheckpointDAL.create( + { + folderCommitId: latestCommitId + }, + tx + ); + + await folderTreeCheckpointResourcesDAL.insertMany( + folderCommits.map((folderCommit) => ({ + folderTreeCheckpointId: folderTreeCheckpoint.id, + folderId: folderCommit.folderId, + folderCommitId: folderCommit.id + })), + tx + ); + + logger.info(`Folder tree checkpoint created successfully: ${folderTreeCheckpoint.id}`); + } catch (error) { + logger.error(error, `Error processing folder tree checkpoint [envId=${envId}]`); + throw error; + } finally { + // Always release the lock + try { + if (lock) { + await lock.release(); + logger.info(`Released lock for folder tree checkpoint [envId=${envId}]`); + } else { + logger.error(`No lock to release for envId=${envId}. This should never happen.`); + } + } catch (releaseError) { + logger.error(releaseError, `Error releasing lock for folder tree checkpoint [envId=${envId}]`); + // Try to force delete the lock if release fails + try { + await keyStore.deleteItem(KeyStorePrefixes.FolderTreeCheckpoint(envId)); + logger.info(`Force deleted lock after release failure for envId=${envId}`); + } catch (deleteError) { + logger.error(deleteError, `Failed to force delete lock after release failure for envId=${envId}`); + } + } + } + }; + + queueService.start(QueueName.FolderTreeCheckpoint, async (job) => { + try { + if (job.name === QueueJobs.CreateFolderTreeCheckpoint) { + const jobData = job.data as TCreateFolderTreeCheckpointDTO; + await createFolderTreeCheckpoint(jobData); + } + } catch (error) { + logger.error(error, "Error creating folder tree checkpoint:"); + throw error; + } + }); + + return { + scheduleTreeCheckpoint: (envId: string) => scheduleTreeCheckpoint({ envId }), + createFolderTreeCheckpoint: (envId: string, folderCommitId?: string, tx?: Knex) => + createFolderTreeCheckpoint({ envId, folderCommitId }, tx) + }; +}; diff --git a/backend/src/services/folder-commit/folder-commit-schemas.ts b/backend/src/services/folder-commit/folder-commit-schemas.ts new file mode 100644 index 000000000..9f99bd2cc --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-schemas.ts @@ -0,0 +1,143 @@ +import { z } from "zod"; + +// Base schema shared by both secret and folder changes +const baseChangeSchema = z.object({ + id: z.string(), + folderCommitId: z.string(), + changeType: z.string(), + isUpdate: z.boolean().optional(), + createdAt: z.union([z.string(), z.date()]), + updatedAt: z.union([z.string(), z.date()]), + actorMetadata: z + .union([ + z.object({ + id: z.string().optional(), + name: z.string().optional() + }), + z.unknown() + ]) + .optional(), + actorType: z.string(), + message: z.string().nullable().optional(), + folderId: z.string() +}); + +// Secret-specific versions schema +const secretVersionSchema = z.object({ + secretKey: z.string(), + secretComment: z.string(), + skipMultilineEncoding: z.boolean().nullable().optional(), + tags: z.array(z.string()).nullable().optional(), + metadata: z.unknown().nullable().optional(), + secretValue: z.string() +}); + +// Folder-specific versions schema +const folderVersionSchema = z.object({ + version: z.string().optional(), + name: z.string().optional(), + description: z.string().optional().nullable() +}); + +// Secret commit change schema +const secretCommitChangeSchema = baseChangeSchema.extend({ + resourceType: z.literal("secret"), + secretVersionId: z.string().optional().nullable(), + secretKey: z.string(), + secretVersion: z.union([z.string(), z.number()]), + secretId: z.string(), + versions: z.array(secretVersionSchema).optional() +}); + +// Folder commit change schema +const folderCommitChangeSchema = baseChangeSchema.extend({ + resourceType: z.literal("folder"), + folderVersionId: z.string().optional().nullable(), + folderName: z.string(), + folderChangeId: z.string(), + folderVersion: z.union([z.string(), z.number()]), + versions: z.array(folderVersionSchema).optional() +}); + +// Discriminated union for commit changes +export const commitChangeSchema = z.discriminatedUnion("resourceType", [ + secretCommitChangeSchema, + folderCommitChangeSchema +]); + +// Commit schema +const commitSchema = z.object({ + id: z.string(), + commitId: z.string(), + actorMetadata: z + .union([ + z.object({ + id: z.string().optional(), + name: z.string().optional() + }), + z.unknown() + ]) + .optional(), + actorType: z.string(), + message: z.string().nullable().optional(), + folderId: z.string(), + envId: z.string(), + createdAt: z.union([z.string(), z.date()]), + updatedAt: z.union([z.string(), z.date()]), + isLatest: z.boolean().default(false), + changes: z.array(commitChangeSchema).optional() +}); + +// Response schema +export const commitChangesResponseSchema = z.object({ + changes: commitSchema +}); + +// Base resource change schema for comparison results +const baseResourceChangeSchema = z.object({ + id: z.string(), + versionId: z.string(), + oldVersionId: z.string().optional(), + changeType: z.enum(["add", "delete", "update", "create"]), + commitId: z.union([z.string(), z.bigint()]), + createdAt: z.union([z.string(), z.date()]).optional(), + parentId: z.string().optional(), + isUpdate: z.boolean().optional(), + fromVersion: z.union([z.string(), z.number()]).optional() +}); + +// Secret resource change schema +const secretResourceChangeSchema = baseResourceChangeSchema.extend({ + type: z.literal("secret"), + secretKey: z.string(), + secretVersion: z.union([z.string(), z.number()]), + secretId: z.string(), + versions: z + .array( + z.object({ + secretKey: z.string().optional(), + secretComment: z.string().optional(), + skipMultilineEncoding: z.boolean().nullable().optional(), + secretReminderRepeatDays: z.number().nullable().optional(), + tags: z.array(z.string()).nullable().optional(), + metadata: z.unknown().nullable().optional(), + secretReminderNote: z.string().nullable().optional(), + secretValue: z.string().optional() + }) + ) + .optional() +}); + +// Folder resource change schema +const folderResourceChangeSchema = baseResourceChangeSchema.extend({ + type: z.literal("folder"), + folderName: z.string(), + folderVersion: z.union([z.string(), z.number()]), + versions: z.array(folderVersionSchema).optional() +}); + +// Discriminated union for resource changes +export const resourceChangeSchema = z.discriminatedUnion("type", [ + secretResourceChangeSchema, + folderResourceChangeSchema +]); diff --git a/backend/src/services/folder-commit/folder-commit-service.test.ts b/backend/src/services/folder-commit/folder-commit-service.test.ts new file mode 100644 index 000000000..1879cf493 --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-service.test.ts @@ -0,0 +1,671 @@ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/return-await */ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +import { Knex } from "knex"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { ProjectType, TSecretFolderVersions, TSecretVersionsV2 } from "@app/db/schemas"; +import { BadRequestError, NotFoundError } from "@app/lib/errors"; + +import { ActorType } from "../auth/auth-type"; +import { + ChangeType, + CommitType, + folderCommitServiceFactory, + ResourceChange, + TFolderCommitServiceFactory +} from "./folder-commit-service"; + +// Mock config +vi.mock("@app/lib/config/env", () => ({ + getConfig: () => ({ + PIT_CHECKPOINT_WINDOW: 5, + PIT_TREE_CHECKPOINT_WINDOW: 10 + }) +})); + +// Mock logger +vi.mock("@app/lib/logger", () => ({ + logger: { + info: vi.fn(), + error: vi.fn() + } +})); + +describe("folderCommitServiceFactory", () => { + // Properly type the mock functions + type TransactionCallback = (trx: Knex) => Promise; + + // Mock dependencies + const mockFolderCommitDAL = { + create: vi.fn().mockResolvedValue({}), + findById: vi.fn().mockResolvedValue({}), + findByFolderId: vi.fn().mockResolvedValue([]), + findLatestCommit: vi.fn().mockResolvedValue({}), + transaction: vi.fn().mockImplementation((callback: TransactionCallback) => callback({} as Knex)), + getNumberOfCommitsSince: vi.fn().mockResolvedValue(0), + getEnvNumberOfCommitsSince: vi.fn().mockResolvedValue(0), + findCommitsToRecreate: vi.fn().mockResolvedValue([]), + findMultipleLatestCommits: vi.fn().mockResolvedValue([]), + findLatestCommitBetween: vi.fn().mockResolvedValue({}), + findAllCommitsBetween: vi.fn().mockResolvedValue([]), + findLatestEnvCommit: vi.fn().mockResolvedValue({}), + findLatestCommitByFolderIds: vi.fn().mockResolvedValue({}) + }; + + const mockKmsService = { + createCipherPairWithDataKey: vi.fn().mockResolvedValue({}) + }; + + const mockFolderCommitChangesDAL = { + create: vi.fn().mockResolvedValue({}), + findByCommitId: vi.fn().mockResolvedValue([]), + insertMany: vi.fn().mockResolvedValue([]) + }; + + const mockFolderCheckpointDAL = { + create: vi.fn().mockResolvedValue({}), + findByFolderId: vi.fn().mockResolvedValue([]), + findLatestByFolderId: vi.fn().mockResolvedValue(null), + findNearestCheckpoint: vi.fn().mockResolvedValue({}) + }; + + const mockFolderCheckpointResourcesDAL = { + insertMany: vi.fn().mockResolvedValue([]), + findByCheckpointId: vi.fn().mockResolvedValue([]) + }; + + const mockFolderTreeCheckpointDAL = { + create: vi.fn().mockResolvedValue({}), + findByProjectId: vi.fn().mockResolvedValue([]), + findLatestByProjectId: vi.fn().mockResolvedValue({}), + findNearestCheckpoint: vi.fn().mockResolvedValue({}), + findLatestByEnvId: vi.fn().mockResolvedValue({}) + }; + + const mockFolderTreeCheckpointResourcesDAL = { + insertMany: vi.fn().mockResolvedValue([]), + findByTreeCheckpointId: vi.fn().mockResolvedValue([]) + }; + + const mockUserDAL = { + findById: vi.fn().mockResolvedValue({}) + }; + + const mockIdentityDAL = { + findById: vi.fn().mockResolvedValue({}) + }; + + const mockFolderDAL = { + findByParentId: vi.fn().mockResolvedValue([]), + findByProjectId: vi.fn().mockResolvedValue([]), + deleteById: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + updateById: vi.fn().mockResolvedValue({}), + update: vi.fn().mockResolvedValue({}), + find: vi.fn().mockResolvedValue([]), + findById: vi.fn().mockResolvedValue({}), + findByEnvId: vi.fn().mockResolvedValue([]), + findFoldersByRootAndIds: vi.fn().mockResolvedValue([]) + }; + + const mockFolderVersionDAL = { + findLatestFolderVersions: vi.fn().mockResolvedValue({}), + findById: vi.fn().mockResolvedValue({}), + deleteById: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + updateById: vi.fn().mockResolvedValue({}), + find: vi.fn().mockResolvedValue({}), // Changed from [] to {} to match Object.values() expectation + findByIdsWithLatestVersion: vi.fn().mockResolvedValue({}) + }; + + const mockSecretVersionV2BridgeDAL = { + findLatestVersionByFolderId: vi.fn().mockResolvedValue([]), + findById: vi.fn().mockResolvedValue({}), + deleteById: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + updateById: vi.fn().mockResolvedValue({}), + find: vi.fn().mockResolvedValue([]), + findByIdsWithLatestVersion: vi.fn().mockResolvedValue({}), + findLatestVersionMany: vi.fn().mockResolvedValue({}) + }; + + const mockSecretV2BridgeDAL = { + deleteById: vi.fn().mockResolvedValue({}), + create: vi.fn().mockResolvedValue({}), + updateById: vi.fn().mockResolvedValue({}), + update: vi.fn().mockResolvedValue({}), + insertMany: vi.fn().mockResolvedValue([]), + invalidateSecretCacheByProjectId: vi.fn().mockResolvedValue({}) + }; + + const mockProjectDAL = { + findById: vi.fn().mockResolvedValue({}), + findProjectByEnvId: vi.fn().mockResolvedValue({}) + }; + + const mockFolderCommitQueueService = { + scheduleTreeCheckpoint: vi.fn().mockResolvedValue({}), + createFolderTreeCheckpoint: vi.fn().mockResolvedValue({}) + }; + + const mockPermissionService = { + getProjectPermission: vi.fn().mockResolvedValue({}) + }; + + const mockSecretTagDAL = { + findSecretTagsByVersionId: vi.fn().mockResolvedValue([]), + saveTagsToSecretV2: vi.fn().mockResolvedValue([]), + findSecretTagsBySecretId: vi.fn().mockResolvedValue([]), + deleteTagsToSecretV2: vi.fn().mockResolvedValue([]), + saveTagsToSecretVersionV2: vi.fn().mockResolvedValue([]) + }; + + const mockResourceMetadataDAL = { + find: vi.fn().mockResolvedValue([]), + insertMany: vi.fn().mockResolvedValue([]), + delete: vi.fn().mockResolvedValue([]) + }; + + let folderCommitService: TFolderCommitServiceFactory; + + beforeEach(() => { + vi.clearAllMocks(); + + folderCommitService = folderCommitServiceFactory({ + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderCommitDAL: mockFolderCommitDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderCommitChangesDAL: mockFolderCommitChangesDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderCheckpointDAL: mockFolderCheckpointDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderCheckpointResourcesDAL: mockFolderCheckpointResourcesDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderTreeCheckpointDAL: mockFolderTreeCheckpointDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderTreeCheckpointResourcesDAL: mockFolderTreeCheckpointResourcesDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + userDAL: mockUserDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + identityDAL: mockIdentityDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderDAL: mockFolderDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + folderVersionDAL: mockFolderVersionDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + secretVersionV2BridgeDAL: mockSecretVersionV2BridgeDAL, + projectDAL: mockProjectDAL, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + secretV2BridgeDAL: mockSecretV2BridgeDAL, + folderCommitQueueService: mockFolderCommitQueueService, + // @ts-expect-error - Mock implementation doesn't need all interface methods for testing + permissionService: mockPermissionService, + kmsService: mockKmsService, + secretTagDAL: mockSecretTagDAL, + resourceMetadataDAL: mockResourceMetadataDAL + }); + }); + + afterEach(() => { + vi.resetAllMocks(); + }); + + describe("createCommit", () => { + it("should successfully create a commit with user actor", async () => { + // Arrange + const userData = { id: "user-id", username: "testuser" }; + const folderData = { id: "folder-id", envId: "env-id" }; + const commitData = { id: "commit-id", folderId: "folder-id" }; + + mockUserDAL.findById.mockResolvedValue(userData); + mockFolderDAL.findById.mockResolvedValue(folderData); + mockFolderCommitDAL.create.mockResolvedValue(commitData); + mockFolderCheckpointDAL.findLatestByFolderId.mockResolvedValue(null); + mockFolderCommitDAL.findLatestCommit.mockResolvedValue({ id: "latest-commit-id" }); + mockFolderDAL.findByParentId.mockResolvedValue([]); + mockSecretVersionV2BridgeDAL.findLatestVersionByFolderId.mockResolvedValue([]); + + const data = { + actor: { + type: ActorType.USER, + metadata: { id: userData.id } + }, + message: "Test commit", + folderId: folderData.id, + changes: [ + { + type: CommitType.ADD, + secretVersionId: "secret-version-1" + } + ] + }; + + // Act + const result = await folderCommitService.createCommit(data); + + // Assert + expect(mockUserDAL.findById).toHaveBeenCalledWith(userData.id, undefined); + expect(mockFolderDAL.findById).toHaveBeenCalledWith(folderData.id, undefined); + expect(mockFolderCommitDAL.create).toHaveBeenCalledWith( + expect.objectContaining({ + actorType: ActorType.USER, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + actorMetadata: expect.objectContaining({ name: userData.username }), + message: data.message, + folderId: data.folderId, + envId: folderData.envId + }), + undefined + ); + expect(mockFolderCommitChangesDAL.insertMany).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ + folderCommitId: commitData.id, + changeType: data.changes[0].type, + secretVersionId: data.changes[0].secretVersionId + }) + ]), + undefined + ); + expect(mockFolderCommitQueueService.scheduleTreeCheckpoint).toHaveBeenCalledWith(folderData.envId); + expect(result).toEqual(commitData); + }); + + it("should successfully create a commit with identity actor", async () => { + // Arrange + const identityData = { id: "identity-id", name: "testidentity" }; + const folderData = { id: "folder-id", envId: "env-id" }; + const commitData = { id: "commit-id", folderId: "folder-id" }; + + mockIdentityDAL.findById.mockResolvedValue(identityData); + mockFolderDAL.findById.mockResolvedValue(folderData); + mockFolderCommitDAL.create.mockResolvedValue(commitData); + mockFolderCheckpointDAL.findLatestByFolderId.mockResolvedValue(null); + mockFolderCommitDAL.findLatestCommit.mockResolvedValue({ id: "latest-commit-id" }); + mockFolderDAL.findByParentId.mockResolvedValue([]); + mockSecretVersionV2BridgeDAL.findLatestVersionByFolderId.mockResolvedValue([]); + + // Mock folderVersionDAL.find to return an object with folder version data + mockFolderVersionDAL.find.mockResolvedValue({ + "folder-version-1": { + id: "folder-version-1", + folderId: "sub-folder-id", + envId: "env-id", + name: "Test Folder", + version: 1 + } + }); + + const data = { + actor: { + type: ActorType.IDENTITY, + metadata: { id: identityData.id } + }, + message: "Test commit", + folderId: folderData.id, + changes: [ + { + type: CommitType.ADD, + folderVersionId: "folder-version-1" + } + ], + omitIgnoreFilter: true + }; + + // Act + const result = await folderCommitService.createCommit(data); + + // Assert + expect(mockIdentityDAL.findById).toHaveBeenCalledWith(identityData.id, undefined); + expect(mockFolderDAL.findById).toHaveBeenCalledWith(folderData.id, undefined); + expect(mockFolderCommitDAL.create).toHaveBeenCalledWith( + expect.objectContaining({ + actorType: ActorType.IDENTITY, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + actorMetadata: expect.objectContaining({ name: identityData.name }), + message: data.message, + folderId: data.folderId, + envId: folderData.envId + }), + undefined + ); + expect(mockFolderCommitChangesDAL.insertMany).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ + folderCommitId: commitData.id, + changeType: data.changes[0].type, + folderVersionId: data.changes[0].folderVersionId + }) + ]), + undefined + ); + expect(mockFolderCommitQueueService.scheduleTreeCheckpoint).toHaveBeenCalledWith(folderData.envId); + expect(result).toEqual(commitData); + }); + + it("should throw NotFoundError when folder does not exist", async () => { + // Arrange + mockFolderDAL.findById.mockResolvedValue(null); + + const data = { + actor: { + type: ActorType.PLATFORM + }, + message: "Test commit", + folderId: "non-existent-folder", + changes: [] + }; + + // Act & Assert + await expect(folderCommitService.createCommit(data)).rejects.toThrow(NotFoundError); + expect(mockFolderDAL.findById).toHaveBeenCalledWith("non-existent-folder", undefined); + }); + }); + + describe("addCommitChange", () => { + it("should successfully add a change to an existing commit", async () => { + // Arrange + const commitData = { id: "commit-id", folderId: "folder-id" }; + const changeData = { id: "change-id", folderCommitId: "commit-id" }; + + mockFolderCommitDAL.findById.mockResolvedValue(commitData); + mockFolderCommitChangesDAL.create.mockResolvedValue(changeData); + + const data = { + folderCommitId: commitData.id, + changeType: CommitType.ADD, + secretVersionId: "secret-version-1" + }; + + // Act + const result = await folderCommitService.addCommitChange(data); + + // Assert + expect(mockFolderCommitDAL.findById).toHaveBeenCalledWith(commitData.id, undefined); + expect(mockFolderCommitChangesDAL.create).toHaveBeenCalledWith(data, undefined); + expect(result).toEqual(changeData); + }); + + it("should throw BadRequestError when neither secretVersionId nor folderVersionId is provided", async () => { + // Arrange + const data = { + folderCommitId: "commit-id", + changeType: CommitType.ADD + }; + + // Act & Assert + await expect(folderCommitService.addCommitChange(data)).rejects.toThrow(BadRequestError); + }); + + it("should throw NotFoundError when commit does not exist", async () => { + // Arrange + mockFolderCommitDAL.findById.mockResolvedValue(null); + + const data = { + folderCommitId: "non-existent-commit", + changeType: CommitType.ADD, + secretVersionId: "secret-version-1" + }; + + // Act & Assert + await expect(folderCommitService.addCommitChange(data)).rejects.toThrow(NotFoundError); + expect(mockFolderCommitDAL.findById).toHaveBeenCalledWith("non-existent-commit", undefined); + }); + }); + + // Note: reconstructFolderState is an internal function not exposed in the public API + // We'll test it indirectly through compareFolderStates + + describe("compareFolderStates", () => { + it("should mark all resources as creates when currentCommitId is not provided", async () => { + // Arrange + const targetCommitId = "target-commit-id"; + const targetCommit = { id: targetCommitId, commitId: 1, folderId: "folder-id" }; + + mockFolderCommitDAL.findById.mockResolvedValue(targetCommit); + // Mock how compareFolderStates would process the results internally + mockFolderCheckpointDAL.findNearestCheckpoint.mockResolvedValue({ id: "checkpoint-id", commitId: "hash-0" }); + mockFolderCheckpointResourcesDAL.findByCheckpointId.mockResolvedValue([ + { secretVersionId: "secret-version-1", referencedSecretId: "secret-1" }, + { folderVersionId: "folder-version-1", referencedFolderId: "folder-1" } + ]); + mockFolderCommitDAL.findCommitsToRecreate.mockResolvedValue([]); + mockProjectDAL.findProjectByEnvId.mockResolvedValue({ + id: "project-id", + name: "test-project", + type: ProjectType.SecretManager + }); + + // Act + const result = await folderCommitService.compareFolderStates({ + targetCommitId + }); + + // Assert + expect(mockFolderCommitDAL.findById).toHaveBeenCalledWith(targetCommitId, undefined); + + // Verify we get resources marked as create + expect(result).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + changeType: "create", + commitId: targetCommit.commitId + }) + ]) + ); + }); + }); + + describe("createFolderCheckpoint", () => { + it("should successfully create a checkpoint when force is true", async () => { + // Arrange + const folderCommitId = "commit-id"; + const folderId = "folder-id"; + const checkpointData = { id: "checkpoint-id", folderCommitId }; + + mockFolderDAL.findByParentId.mockResolvedValue([{ id: "subfolder-id" }]); + mockFolderVersionDAL.findLatestFolderVersions.mockResolvedValue({ "subfolder-id": { id: "folder-version-1" } }); + mockSecretVersionV2BridgeDAL.findLatestVersionByFolderId.mockResolvedValue([{ id: "secret-version-1" }]); + mockFolderCheckpointDAL.create.mockResolvedValue(checkpointData); + + // Act + const result = await folderCommitService.createFolderCheckpoint({ + folderId, + folderCommitId, + force: true + }); + + // Assert + expect(mockFolderCheckpointDAL.create).toHaveBeenCalledWith({ folderCommitId }, undefined); + expect(mockFolderCheckpointResourcesDAL.insertMany).toHaveBeenCalled(); + expect(result).toBe(folderCommitId); + }); + }); + + describe("deepRollbackFolder", () => { + it("should throw NotFoundError when commit doesn't exist", async () => { + // Arrange + const targetCommitId = "non-existent-commit"; + const envId = "env-id"; + const actorId = "user-id"; + const actorType = ActorType.USER; + const projectId = "project-id"; + + // Mock the transaction to properly handle the error + mockFolderCommitDAL.transaction.mockImplementation(async (callback) => { + return await callback({} as Knex); + }); + + // Mock findById to return null inside the transaction + mockFolderCommitDAL.findById.mockResolvedValue(null); + + // Act & Assert + await expect( + folderCommitService.deepRollbackFolder(targetCommitId, envId, actorId, actorType, projectId) + ).rejects.toThrow(NotFoundError); + }); + }); + + describe("createFolderTreeCheckpoint", () => { + it("should create a tree checkpoint when checkpoint window is exceeded", async () => { + // Arrange + const envId = "env-id"; + const folderCommitId = "commit-id"; + const latestCommit = { id: folderCommitId }; + const latestTreeCheckpoint = { id: "tree-checkpoint-id", folderCommitId: "old-commit-id" }; + const folders = [ + { id: "folder-1", isReserved: false }, + { id: "folder-2", isReserved: false }, + { id: "folder-3", isReserved: true } // Reserved folders should be filtered out + ]; + const folderCommits = [ + { folderId: "folder-1", id: "commit-1" }, + { folderId: "folder-2", id: "commit-2" } + ]; + const treeCheckpoint = { id: "new-tree-checkpoint-id" }; + + mockFolderCommitDAL.findLatestEnvCommit.mockResolvedValue(latestCommit); + mockFolderTreeCheckpointDAL.findLatestByEnvId.mockResolvedValue(latestTreeCheckpoint); + mockFolderCommitDAL.getEnvNumberOfCommitsSince.mockResolvedValue(15); // More than PIT_TREE_CHECKPOINT_WINDOW (10) + mockFolderDAL.findByEnvId.mockResolvedValue(folders); + mockFolderCommitDAL.findMultipleLatestCommits.mockResolvedValue(folderCommits); + mockFolderTreeCheckpointDAL.create.mockResolvedValue(treeCheckpoint); + + // Act + await folderCommitService.createFolderTreeCheckpoint(envId); + + // Assert + expect(mockFolderCommitDAL.findLatestEnvCommit).toHaveBeenCalledWith(envId, undefined); + expect(mockFolderTreeCheckpointDAL.create).toHaveBeenCalledWith({ folderCommitId }, undefined); + }); + }); + + describe("applyFolderStateDifferences", () => { + it("should process changes correctly", async () => { + // Arrange + const folderId = "folder-id"; + const projectId = "project-id"; + const actorId = "user-id"; + const actorType = ActorType.USER; + + const differences = [ + { + id: "secret-1", + versionId: "v1", + changeType: ChangeType.CREATE, + commitId: BigInt(1) + } as ResourceChange, + { + id: "folder-1", + versionId: "v2", + changeType: ChangeType.UPDATE, + commitId: BigInt(1), + folderName: "Test Folder", + folderVersion: "v2" + } as ResourceChange + ]; + + const secretVersions = { + "secret-1": { + id: "secret-version-1", + createdAt: new Date(), + updatedAt: new Date(), + type: "shared", + folderId: "folder-1", + secretId: "secret-1", + version: 1, + key: "SECRET_KEY", + encryptedValue: Buffer.from("encrypted"), + encryptedComment: Buffer.from("comment"), + skipMultilineEncoding: false, + userId: "user-1", + envId: "env-1", + metadata: {} + } as TSecretVersionsV2 + }; + + const folderVersions = { + "folder-1": { + folderId: "folder-1", + version: 1, + name: "Test Folder", + envId: "env-1" + } as TSecretFolderVersions + }; + + // Mock folder lookup for the folder being processed + mockFolderDAL.findById.mockImplementation((id) => { + if (id === folderId) { + return Promise.resolve({ id: folderId, envId: "env-1" }); + } + return Promise.resolve(null); + }); + + // Mock latest commit lookup + mockFolderCommitDAL.findLatestCommit.mockImplementation((id) => { + if (id === folderId) { + return Promise.resolve({ id: "latest-commit-id", folderId }); + } + return Promise.resolve(null); + }); + + // Make sure findByParentId returns an array, not undefined + mockFolderDAL.findByParentId.mockResolvedValue([]); + + // Make sure other required functions return appropriate values + mockFolderCheckpointDAL.findLatestByFolderId.mockResolvedValue(null); + mockSecretVersionV2BridgeDAL.findLatestVersionByFolderId.mockResolvedValue([]); + + // These mocks need to return objects with an id field + mockSecretVersionV2BridgeDAL.findByIdsWithLatestVersion.mockResolvedValue(Object.values(secretVersions)); + mockFolderVersionDAL.findByIdsWithLatestVersion.mockResolvedValue(Object.values(folderVersions)); + mockSecretV2BridgeDAL.insertMany.mockResolvedValue([{ id: "new-secret-1" }]); + mockSecretVersionV2BridgeDAL.create.mockResolvedValue({ id: "new-secret-version-1" }); + mockFolderDAL.updateById.mockResolvedValue({ id: "updated-folder-1" }); + mockFolderVersionDAL.create.mockResolvedValue({ id: "new-folder-version-1" }); + mockFolderCommitDAL.create.mockResolvedValue({ id: "new-commit-id" }); + mockSecretVersionV2BridgeDAL.findLatestVersionMany.mockResolvedValue([ + { + id: "secret-version-1", + createdAt: new Date(), + updatedAt: new Date(), + type: "shared", + folderId: "folder-1", + secretId: "secret-1", + version: 1, + key: "SECRET_KEY", + encryptedValue: Buffer.from("encrypted"), + encryptedComment: Buffer.from("comment"), + skipMultilineEncoding: false, + userId: "user-1", + envId: "env-1", + metadata: {} + } + ]); + + // Mock transaction + mockFolderCommitDAL.transaction.mockImplementation((callback: TransactionCallback) => callback({} as Knex)); + + // Act + const result = await folderCommitService.applyFolderStateDifferences({ + differences, + actorInfo: { + actorType, + actorId, + message: "Applying changes" + }, + folderId, + projectId, + reconstructNewFolders: false + }); + + // Assert + expect(mockFolderCommitDAL.create).toHaveBeenCalled(); + expect(mockSecretV2BridgeDAL.invalidateSecretCacheByProjectId).toHaveBeenCalledWith(projectId); + + // Check that we got the right counts + expect(result.totalChanges).toEqual(2); + }); + }); +}); diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts new file mode 100644 index 000000000..8c0bc8ebf --- /dev/null +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -0,0 +1,2173 @@ +/* eslint-disable no-await-in-loop */ +import { ForbiddenError } from "@casl/ability"; +import { Knex } from "knex"; + +import { + ActionProjectType, + TSecretFolders, + TSecretFolderVersions, + TSecretV2TagJunctionInsert, + TSecretVersionsV2 +} from "@app/db/schemas"; +import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { ProjectPermissionCommitsActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { getConfig } from "@app/lib/config/env"; +import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; +import { chunkArray } from "@app/lib/fn"; +import { logger } from "@app/lib/logger"; + +import { ActorAuthMethod, ActorType } from "../auth/auth-type"; +import { TFolderCheckpointDALFactory } from "../folder-checkpoint/folder-checkpoint-dal"; +import { TFolderCheckpointResourcesDALFactory } from "../folder-checkpoint-resources/folder-checkpoint-resources-dal"; +import { TFolderCommitChangesDALFactory } from "../folder-commit-changes/folder-commit-changes-dal"; +import { TFolderTreeCheckpointDALFactory } from "../folder-tree-checkpoint/folder-tree-checkpoint-dal"; +import { TFolderTreeCheckpointResourcesDALFactory } from "../folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal"; +import { TIdentityDALFactory } from "../identity/identity-dal"; +import { TKmsServiceFactory } from "../kms/kms-service"; +import { KmsDataKey } from "../kms/kms-types"; +import { TProjectDALFactory } from "../project/project-dal"; +import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; +import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; +import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal"; +import { TUserDALFactory } from "../user/user-dal"; +import { TFolderCommitDALFactory } from "./folder-commit-dal"; +import { TFolderCommitQueueServiceFactory } from "./folder-commit-queue"; + +export enum ChangeType { + ADD = "add", + DELETE = "delete", + UPDATE = "update", + CREATE = "create" +} + +export enum CommitType { + ADD = "add", + DELETE = "delete" +} + +export enum ResourceType { + SECRET = "secret", + FOLDER = "folder" +} + +type TCreateCommitDTO = { + actor: { + type: string; + metadata?: { + name?: string; + id?: string; + }; + }; + message?: string; + folderId: string; + changes: { + type: string; + secretVersionId?: string; + folderVersionId?: string; + isUpdate?: boolean; + folderId?: string; + }[]; + omitIgnoreFilter?: boolean; +}; + +type TCommitChangeDTO = { + folderCommitId: string; + changeType: string; + secretVersionId?: string; + folderVersionId?: string; +}; + +type BaseChange = { + id: string; + versionId: string; + oldVersionId?: string; + changeType: ChangeType; + commitId: bigint; + createdAt?: Date; + parentId?: string; + isUpdate?: boolean; + fromVersion?: string; +}; + +type SecretChange = BaseChange & { + type: ResourceType.SECRET; + secretKey: string; + secretVersion: string; + secretId: string; + versions?: { + secretKey?: string; + secretComment?: string; + skipMultilineEncoding?: boolean | null; + metadata?: unknown; + tags?: string[] | null; + secretValue?: string; + }[]; +}; + +type FolderChange = BaseChange & { + type: ResourceType.FOLDER; + folderName: string; + folderVersion: string; + versions?: { + name: string; + description?: string | null; + }[]; +}; + +type SecretTargetChange = { + type: ResourceType.SECRET; + id: string; + versionId: string; + secretKey: string; + secretVersion: string; + fromVersion?: string; +}; + +type FolderTargetChange = { + type: ResourceType.FOLDER; + id: string; + versionId: string; + folderName: string; + folderVersion: string; + fromVersion?: string; +}; + +export type ResourceChange = SecretChange | FolderChange; + +type ActorInfo = { + actorType: string; + actorId?: string; + message?: string; +}; + +type StateChangeResult = { + secretChangesCount: number; + folderChangesCount: number; + totalChanges: number; +}; + +type TFolderCommitServiceFactoryDep = { + folderCommitDAL: TFolderCommitDALFactory; + folderCommitChangesDAL: TFolderCommitChangesDALFactory; + folderCheckpointDAL: TFolderCheckpointDALFactory; + folderCheckpointResourcesDAL: TFolderCheckpointResourcesDALFactory; + folderTreeCheckpointDAL: TFolderTreeCheckpointDALFactory; + folderTreeCheckpointResourcesDAL: TFolderTreeCheckpointResourcesDALFactory; + userDAL: TUserDALFactory; + identityDAL: TIdentityDALFactory; + folderDAL: TSecretFolderDALFactory; + folderVersionDAL: TSecretFolderVersionDALFactory; + secretVersionV2BridgeDAL: TSecretVersionV2DALFactory; + secretV2BridgeDAL: TSecretV2BridgeDALFactory; + projectDAL: Pick; + folderCommitQueueService?: Pick< + TFolderCommitQueueServiceFactory, + "scheduleTreeCheckpoint" | "createFolderTreeCheckpoint" + >; + permissionService?: TPermissionServiceFactory; + kmsService: Pick; + secretTagDAL: Pick< + TSecretTagDALFactory, + | "findSecretTagsByVersionId" + | "saveTagsToSecretV2" + | "findSecretTagsBySecretId" + | "deleteTagsToSecretV2" + | "saveTagsToSecretVersionV2" + >; + resourceMetadataDAL: Pick; +}; + +export const folderCommitServiceFactory = ({ + folderCommitDAL, + folderCommitChangesDAL, + folderCheckpointDAL, + folderTreeCheckpointDAL, + folderCheckpointResourcesDAL, + userDAL, + identityDAL, + folderDAL, + folderVersionDAL, + secretVersionV2BridgeDAL, + projectDAL, + secretV2BridgeDAL, + folderTreeCheckpointResourcesDAL, + folderCommitQueueService, + permissionService, + kmsService, + secretTagDAL, + resourceMetadataDAL +}: TFolderCommitServiceFactoryDep) => { + const appCfg = getConfig(); + + const checkProjectCommitReadPermission = async ({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + }: { + actor: ActorType; + actorId: string; + projectId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + }) => { + if (!permissionService) { + throw new Error("Permission service not initialized"); + } + const { permission } = await permissionService.getProjectPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId, + actionProjectType: ActionProjectType.SecretManager + }); + + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits); + }; + + /** + * Fetches all resources within a folder + */ + const getFolderResources = async (folderId: string, tx?: Knex) => { + const resources = []; + const subFolders = await folderDAL.findByParentId(folderId, tx); + + if (subFolders.length > 0) { + const subFolderIds = subFolders.map((folder) => folder.id); + const folderVersions = await folderVersionDAL.findLatestFolderVersions(subFolderIds, tx); + resources.push( + ...Object.values(folderVersions).map((folderVersion) => ({ + folderVersionId: folderVersion.id, + secretVersionId: undefined + })) + ); + } + + const secretVersions = await secretVersionV2BridgeDAL.findLatestVersionByFolderId(folderId, tx); + if (secretVersions.length > 0) { + resources.push( + ...secretVersions.map((secretVersion) => ({ secretVersionId: secretVersion.id, folderVersionId: undefined })) + ); + } + + return resources; + }; + + /** + * Creates a checkpoint for a folder if necessary + */ + const createFolderCheckpoint = async ({ + folderId, + folderCommitId, + force = false, + tx + }: { + folderId: string; + folderCommitId?: string; + force?: boolean; + tx?: Knex; + }) => { + let latestCommitId = folderCommitId; + const latestCheckpoint = await folderCheckpointDAL.findLatestByFolderId(folderId, tx); + + if (!latestCommitId) { + const latestCommit = await folderCommitDAL.findLatestCommit(folderId, undefined, tx); + if (!latestCommit) { + throw new BadRequestError({ message: "Latest commit ID not found" }); + } + latestCommitId = latestCommit.id; + } + + if (!force && latestCheckpoint) { + const commitsSinceLastCheckpoint = await folderCommitDAL.getNumberOfCommitsSince( + folderId, + latestCheckpoint.folderCommitId, + tx + ); + if (commitsSinceLastCheckpoint < Number(appCfg.PIT_CHECKPOINT_WINDOW)) { + return; + } + } + + const checkpointResources = await getFolderResources(folderId, tx); + + const newCheckpoint = await folderCheckpointDAL.create( + { + folderCommitId: latestCommitId + }, + tx + ); + const batchSize = 500; + const chunks = chunkArray(checkpointResources, batchSize); + + await Promise.all( + chunks.map(async (chunk) => { + await folderCheckpointResourcesDAL.insertMany( + chunk.map((resource) => ({ folderCheckpointId: newCheckpoint.id, ...resource })), + tx + ); + }) + ); + + return latestCommitId; + }; + + /** + * Reconstructs the state of a folder at a specific commit + */ + const reconstructFolderState = async ( + folderCommitId: string, + tx?: Knex + ): Promise<(SecretTargetChange | FolderTargetChange)[]> => { + const targetCommit = await folderCommitDAL.findById(folderCommitId, tx); + if (!targetCommit) { + throw new NotFoundError({ message: `Commit with ID ${folderCommitId} not found` }); + } + + const nearestCheckpoint = await folderCheckpointDAL.findNearestCheckpoint( + targetCommit.commitId, + targetCommit.folderId, + tx + ); + if (!nearestCheckpoint) { + throw new NotFoundError({ message: `Nearest checkpoint not found for commit ${folderCommitId}` }); + } + + const checkpointResources = await folderCheckpointResourcesDAL.findByCheckpointId(nearestCheckpoint.id, tx); + + const folderState: Record = {}; + + // Add all checkpoint resources to initial state + checkpointResources.forEach((resource) => { + if (resource.secretVersionId && resource.referencedSecretId) { + folderState[`secret-${resource.referencedSecretId}`] = { + type: ResourceType.SECRET, + id: resource.referencedSecretId, + versionId: resource.secretVersionId, + secretKey: resource.secretKey, + secretVersion: resource.secretVersion + } as SecretTargetChange; + } else if (resource.folderVersionId && resource.referencedFolderId) { + folderState[`folder-${resource.referencedFolderId}`] = { + type: ResourceType.FOLDER, + id: resource.referencedFolderId, + versionId: resource.folderVersionId, + folderName: resource.folderName, + folderVersion: resource.folderVersion + } as FolderTargetChange; + } + }); + + const commitsToRecreate = await folderCommitDAL.findCommitsToRecreate( + targetCommit.folderId, + targetCommit.commitId, + nearestCheckpoint.commitId, + tx + ); + + // Process commits to recreate final state + for (const commit of commitsToRecreate) { + // eslint-disable-next-line no-continue + if (!commit.changes) continue; + + for (const change of commit.changes) { + if (change.secretVersionId && change.referencedSecretId) { + const key = `secret-${change.referencedSecretId}`; + + if (change.changeType.toLowerCase() === "add") { + folderState[key] = { + type: ResourceType.SECRET, + id: change.referencedSecretId, + versionId: change.secretVersionId, + secretKey: change.secretKey, + secretVersion: change.secretVersion + } as SecretTargetChange; + } else if (change.changeType.toLowerCase() === "delete") { + delete folderState[key]; + } + } else if (change.folderVersionId && change.referencedFolderId) { + const key = `folder-${change.referencedFolderId}`; + + if (change.changeType.toLowerCase() === "add") { + folderState[key] = { + type: ResourceType.FOLDER, + id: change.referencedFolderId, + versionId: change.folderVersionId, + folderName: change.folderName, + folderVersion: change.folderVersion + } as FolderTargetChange; + } else if (change.changeType.toLowerCase() === "delete") { + delete folderState[key]; + } + } + } + } + return Object.values(folderState); + }; + + /** + * Compares folder states between two commits and returns the differences + */ + const compareFolderStates = async ({ + currentCommitId, + targetCommitId, + defaultOperation = "create", + tx + }: { + currentCommitId?: string; + targetCommitId: string; + defaultOperation?: "create" | "update" | "delete"; + tx?: Knex; + }): Promise => { + const targetCommit = await folderCommitDAL.findById(targetCommitId, tx); + if (!targetCommit) { + throw new NotFoundError({ message: `Commit with ID ${targetCommitId} not found` }); + } + + const project = await projectDAL.findProjectByEnvId(targetCommit.envId, tx); + + if (!project) { + throw new NotFoundError({ message: `No project found for envId ${targetCommit.envId}` }); + } + + // If currentCommitId is not provided, mark all resources in target as creates + if (!currentCommitId) { + const targetState = await reconstructFolderState(targetCommitId, tx); + + return targetState + .map((resource): ResourceChange | null => { + if (resource.type === ResourceType.SECRET) { + return { + type: ResourceType.SECRET, + id: resource.id, + versionId: resource.versionId, + changeType: defaultOperation as ChangeType, + commitId: targetCommit.commitId, + secretKey: resource.secretKey, + secretVersion: resource.secretVersion, + secretId: resource.id + }; + } + if (resource.type === ResourceType.FOLDER) { + return { + type: ResourceType.FOLDER, + id: resource.id, + versionId: resource.versionId, + changeType: defaultOperation as ChangeType, + commitId: targetCommit.commitId, + folderName: resource.folderName, + folderVersion: resource.folderVersion + }; + } + return null; + }) + .filter((change): change is ResourceChange => !!change); + } + + // Original logic for when currentCommitId is provided + const currentState = await reconstructFolderState(currentCommitId, tx); + const targetState = await reconstructFolderState(targetCommitId, tx); + + // Create lookup maps for easier comparison + const currentMap: Record = {}; + const targetMap: Record< + string, + { + type: string; + id: string; + versionId: string; + secretKey?: string; + secretVersion?: string; + folderName?: string; + folderVersion?: string; + fromVersion?: string; + } + > = {}; + + // Build lookup maps + currentState.forEach((resource) => { + const key = `${resource.type}-${resource.id}`; + currentMap[key] = resource; + }); + + targetState.forEach((resource) => { + const key = `${resource.type}-${resource.id}`; + targetMap[key] = resource; + }); + + // Track differences + const differences: ResourceChange[] = []; + + // Find deletes and updates + Object.keys(currentMap).forEach((key) => { + const currentResource = currentMap[key]; + const targetResource = targetMap[key]; + + if (!targetResource) { + // Resource was deleted + if (currentResource.type === ResourceType.SECRET) { + differences.push({ + type: ResourceType.SECRET, + id: currentResource.id, + versionId: currentResource.versionId, + changeType: ChangeType.DELETE, + commitId: targetCommit.commitId, + secretKey: currentResource.secretKey, + secretVersion: currentResource.secretVersion, + secretId: currentResource.id, + fromVersion: currentResource.versionId + }); + } else if (currentResource.type === ResourceType.FOLDER) { + differences.push({ + type: ResourceType.FOLDER, + id: currentResource.id, + versionId: currentResource.versionId, + changeType: ChangeType.DELETE, + commitId: targetCommit.commitId, + folderName: currentResource.folderName, + folderVersion: currentResource.folderVersion, + fromVersion: currentResource.versionId + }); + } + } else if (currentResource.versionId !== targetResource.versionId) { + // Resource was updated + if (targetResource.type === ResourceType.SECRET) { + const secretCurrentResource = currentResource as SecretTargetChange; + const secretTargetResource = targetResource as SecretTargetChange; + differences.push({ + type: ResourceType.SECRET, + id: secretTargetResource.id, + versionId: secretTargetResource.versionId, + changeType: ChangeType.UPDATE, + commitId: targetCommit.commitId, + secretKey: secretTargetResource.secretKey, + secretVersion: secretTargetResource.secretVersion, + secretId: secretTargetResource.id, + fromVersion: secretCurrentResource.secretVersion + }); + } else if (targetResource.type === ResourceType.FOLDER) { + const folderCurrentResource = currentResource as FolderTargetChange; + const folderTargetResource = targetResource as FolderTargetChange; + + differences.push({ + type: ResourceType.FOLDER, + id: folderTargetResource.id, + versionId: folderTargetResource.versionId, + changeType: ChangeType.UPDATE, + commitId: targetCommit.commitId, + folderName: folderTargetResource.folderName, + folderVersion: folderTargetResource.folderVersion, + fromVersion: folderCurrentResource.folderVersion + }); + } + } + }); + + // Find new resources + Object.keys(targetMap).forEach((key) => { + if (!currentMap[key]) { + const targetResource = targetMap[key]; + if (targetResource.type === ResourceType.SECRET) { + const secretTargetResource = targetResource as SecretTargetChange; + differences.push({ + type: ResourceType.SECRET, + id: secretTargetResource.id, + versionId: secretTargetResource.versionId, + changeType: ChangeType.CREATE, + commitId: targetCommit.commitId, + createdAt: targetCommit.createdAt, + secretKey: secretTargetResource.secretKey, + secretVersion: secretTargetResource.secretVersion, + secretId: secretTargetResource.id + }); + } else if (targetResource.type === ResourceType.FOLDER) { + const folderTargetResource = targetResource as FolderTargetChange; + differences.push({ + type: ResourceType.FOLDER, + id: folderTargetResource.id, + versionId: folderTargetResource.versionId, + changeType: ChangeType.CREATE, + commitId: targetCommit.commitId, + createdAt: targetCommit.createdAt, + folderName: folderTargetResource.folderName, + folderVersion: folderTargetResource.folderVersion + }); + } + } + }); + + const removeNoChangeUpdate: string[] = []; + + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId: project.id + }); + + await Promise.all( + differences.map(async (change) => { + if (change.changeType === ChangeType.UPDATE) { + if (change.type === ResourceType.FOLDER && change.folderVersion && change.fromVersion) { + const versions = await folderVersionDAL.find({ + folderId: change.id, + $in: { + version: [Number(change.folderVersion), Number(change.fromVersion)] + } + }); + const versionsShaped = versions.map((version) => ({ + name: version.name, + description: version.description + })); + const uniqueVersions = versionsShaped.filter( + (item, index, arr) => + arr.findIndex((other) => + Object.entries(item).every( + ([key, value]) => JSON.stringify(value) === JSON.stringify(other[key as keyof typeof other]) + ) + ) === index + ); + if (uniqueVersions.length === 1) { + removeNoChangeUpdate.push(change.id); + } + } else if (change.type === ResourceType.SECRET && change.secretVersion && change.fromVersion) { + const versions = await secretVersionV2BridgeDAL.findVersionsBySecretIdWithActors({ + secretId: change.id, + projectId: project.id, + secretVersions: [change.secretVersion, change.fromVersion] + }); + const versionsShaped = versions.map((el) => ({ + secretKey: el.key, + secretComment: el.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() + : "", + skipMultilineEncoding: el.skipMultilineEncoding, + secretReminderRepeatDays: el.reminderRepeatDays, + tags: el.tags, + metadata: el.metadata, + secretReminderNote: el.reminderNote, + secretValue: el.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() + : "" + })); + const uniqueVersions = versionsShaped.filter( + (item, index, arr) => + arr.findIndex((other) => + Object.entries(item).every( + ([key, value]) => JSON.stringify(value) === JSON.stringify(other[key as keyof typeof other]) + ) + ) === index + ); + if (uniqueVersions.length === 1) { + removeNoChangeUpdate.push(change.id); + } + } + } + }) + ); + return differences.filter((change) => !removeNoChangeUpdate.includes(change.id)); + }; + + /** + * Adds a change to an existing commit + */ + const addCommitChange = async (data: TCommitChangeDTO, tx?: Knex) => { + try { + if (!data.secretVersionId && !data.folderVersionId) { + throw new BadRequestError({ message: "Either secretVersionId or folderVersionId must be provided" }); + } + + const commit = await folderCommitDAL.findById(data.folderCommitId, tx); + if (!commit) { + throw new NotFoundError({ message: `Commit with ID ${data.folderCommitId} not found` }); + } + + return await folderCommitChangesDAL.create(data, tx); + } catch (error) { + if (error instanceof NotFoundError || error instanceof BadRequestError) { + throw error; + } + throw new DatabaseError({ error, name: "AddCommitChange" }); + } + }; + + const createDeleteCommitForNestedFolders = async ({ + folderId, + actorMetadata, + actorType, + envId, + parentFolderName, + step = 1, + tx + }: { + folderId: string; + actorMetadata: Record; + actorType: string; + envId: string; + parentFolderName: string; + step?: number; + tx?: Knex; + }) => { + if (step > 20) { + logger.info(`createDeleteCommitForNestedFolders - Max step reached for folder ${folderId}`); + return; + } + logger.info(`Creating delete commit for nested folders ${folderId}`); + const folderVersion = await folderVersionDAL.findLatestVersion(folderId, tx); + if (!folderVersion) { + logger.info(`No folder version found for ${folderId}`); + return; + } + const lastFolderCommit = await folderCommitDAL.findLatestCommit(folderId, undefined, tx); + if (!lastFolderCommit) { + logger.info(`No commit found for folder ${folderId}`); + return; + } + const folderState = await reconstructFolderState(lastFolderCommit.id, tx); + const changes = folderState.map((resource) => ({ + type: ChangeType.DELETE, + folderId: resource.id, + folderName: resource.type === ResourceType.FOLDER ? resource.folderName : undefined, + secretVersionId: resource.type === ResourceType.SECRET ? resource.versionId : undefined, + folderVersionId: resource.type === ResourceType.FOLDER ? resource.versionId : undefined, + secretKey: resource.type === ResourceType.SECRET ? resource.secretKey : undefined + })); + logger.info(`Found ${changes.length} changes for ${folderId}`); + + const newCommit = await folderCommitDAL.create( + { + actorMetadata, + actorType, + message: `Parent folder ${parentFolderName} deleted`, + folderId, + envId + }, + tx + ); + + const batchSize = 500; + const chunks = chunkArray(changes, batchSize); + + await Promise.all( + chunks.map(async (chunk) => { + await folderCommitChangesDAL.insertMany( + chunk.map((change) => ({ + folderCommitId: newCommit.id, + changeType: CommitType.DELETE, + secretVersionId: change.secretVersionId, + folderVersionId: change.folderVersionId, + isUpdate: false + })), + tx + ); + }) + ); + + await Promise.all( + changes + .filter((change) => change.type === ChangeType.DELETE && change.folderVersionId) + .map(async (change) => { + await createDeleteCommitForNestedFolders({ + folderId: change.folderId, + actorMetadata, + actorType, + envId, + parentFolderName: folderVersion.name, + step: step + 1, + tx + }); + }) + ); + }; + + const compareSecretVersions = async ( + version1: TSecretVersionsV2 & { tags: { id: string }[] }, + version2: TSecretVersionsV2 & { tags: { id: string }[] }, + projectId: string + ) => { + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + const objectsEqual = (o1: unknown, o2: unknown): boolean => { + if (typeof o1 !== "object" || o1 === null || typeof o2 !== "object" || o2 === null) { + return o1 === o2; + } + + const obj1 = o1 as Record; + const obj2 = o2 as Record; + return ( + Object.keys(obj1).length === Object.keys(obj2).length && Object.keys(obj1).every((p) => obj1[p] === obj2[p]) + ); + }; + + const arraysEqual = (a1: unknown[], a2: unknown[]) => + a1.length === a2.length && a1.every((obj1) => a2.some((obj2) => objectsEqual(obj1, obj2))); + + const version1Reshaped = { + ...version1, + encryptedValue: version1.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: version1.encryptedValue }).toString() + : "", + encryptedComment: version1.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: version1.encryptedComment }).toString() + : "", + metadata: Array.isArray(version1.metadata) ? (version1.metadata as { key: string; value: string }[]) : [], + tags: version1.tags.map((tag) => tag.id) + }; + const version2Reshaped = { + ...version2, + encryptedValue: version2.encryptedValue + ? secretManagerDecryptor({ cipherTextBlob: version2.encryptedValue }).toString() + : "", + encryptedComment: version2.encryptedComment + ? secretManagerDecryptor({ cipherTextBlob: version2.encryptedComment }).toString() + : "", + metadata: Array.isArray(version2.metadata) ? (version2.metadata as { key: string; value: string }[]) : [], + tags: version2.tags.map((tag) => tag.id) + }; + return ( + version1Reshaped.key === version2Reshaped.key && + version1Reshaped.encryptedValue === version2Reshaped.encryptedValue && + version1Reshaped.encryptedComment === version2Reshaped.encryptedComment && + version1Reshaped.skipMultilineEncoding === version2Reshaped.skipMultilineEncoding && + arraysEqual(version1Reshaped.metadata, version2Reshaped.metadata) && + version1Reshaped.tags.length === version2Reshaped.tags.length && + version1Reshaped.tags.every((tag) => version2Reshaped.tags.includes(tag)) + ); + }; + + const filterIgnoredChanges = async ( + changes: { + type: string; + secretVersionId?: string; + folderVersionId?: string; + isUpdate?: boolean; + folderId?: string; + }[], + projectId: string, + tx?: Knex + ) => { + let filteredChanges = [...changes]; + for (const change of changes) { + if (change.type === ChangeType.ADD && change.isUpdate && change.secretVersionId) { + const secretVersions = await secretVersionV2BridgeDAL.findByIdAndPreviousVersion(change.secretVersionId, tx); + const comparison = await compareSecretVersions(secretVersions[0], secretVersions[1], projectId); + if (comparison) { + filteredChanges = filteredChanges.filter( + (filteredChange) => filteredChange.secretVersionId !== change.secretVersionId + ); + } + } + } + return filteredChanges; + }; + + /** + * Creates a new commit with the provided changes + */ + const createCommit = async (data: TCreateCommitDTO, tx?: Knex) => { + try { + const metadata = { ...data.actor.metadata } || {}; + + if (data.actor.type === ActorType.USER && data.actor.metadata?.id) { + const user = await userDAL.findById(data.actor.metadata?.id, tx); + metadata.name = user?.username; + } + + if (data.actor.type === ActorType.IDENTITY && data.actor.metadata?.id) { + const identity = await identityDAL.findById(data.actor.metadata?.id, tx); + metadata.name = identity?.name; + } + + const folder = await folderDAL.findById(data.folderId, tx); + if (!folder) { + throw new NotFoundError({ message: `Folder with ID ${data.folderId} not found` }); + } + + let { changes } = data; + if (!data.omitIgnoreFilter) { + const project = await projectDAL.findProjectByEnvId(folder.envId, tx); + + if (!project) { + return; + } + + changes = await filterIgnoredChanges(data.changes, project.id, tx); + if (changes.length === 0) { + return; + } + } + + const newCommit = await folderCommitDAL.create( + { + actorMetadata: metadata, + actorType: data.actor.type, + message: data.message, + folderId: data.folderId, + envId: folder.envId + }, + tx + ); + + const batchSize = 500; + const chunks = chunkArray(changes, batchSize); + + await Promise.all( + chunks.map(async (chunk) => { + await folderCommitChangesDAL.insertMany( + chunk.map((change) => ({ + folderCommitId: newCommit.id, + changeType: change.type, + secretVersionId: change.secretVersionId, + folderVersionId: change.folderVersionId, + isUpdate: change.isUpdate || false + })), + tx + ); + }) + ); + + await Promise.all( + changes.map(async (change) => { + if (change.type === ChangeType.DELETE && change.folderId) { + await createDeleteCommitForNestedFolders({ + folderId: change.folderId, + actorMetadata: metadata, + actorType: data.actor.type, + envId: folder.envId, + parentFolderName: folder.name, + tx + }); + } + }) + ); + + await createFolderCheckpoint({ folderId: data.folderId, folderCommitId: newCommit.id, tx }); + if (folderCommitQueueService) { + if (!folder.parentId) { + const previousTreeCommit = await folderTreeCheckpointDAL.findLatestByEnvId(folder.envId); + if (!previousTreeCommit) { + await folderCommitQueueService.createFolderTreeCheckpoint(folder.envId, newCommit.id, tx); + } + } + await folderCommitQueueService.scheduleTreeCheckpoint(folder.envId); + } + return newCommit; + } catch (error) { + if (error instanceof NotFoundError || error instanceof BadRequestError) { + throw error; + } + throw new DatabaseError({ error, name: "CreateCommit" }); + } + }; + + /** + * Process secret changes when applying folder state differences + */ + const processSecretChanges = async ( + changes: ResourceChange[], + secretVersions: Record, + actorInfo: ActorInfo, + folderId: string, + tx?: Knex + ) => { + const commitChanges = []; + const folder = await folderDAL.findById(folderId, tx); + if (!folder) { + return []; + } + const project = await projectDAL.findById(folder.projectId, tx); + + // Filter only secret changes using discriminated union + const secretChanges = changes.filter( + (change): change is ResourceChange & SecretChange => change.type === ResourceType.SECRET + ); + + // Collect all secretIds for batch lookup + const secretIds = secretChanges.map((change) => secretVersions[change.id]?.secretId).filter(Boolean); + + // Fetch all latest versions in one call + const latestVersionsMap = await secretVersionV2BridgeDAL.findLatestVersionMany(folderId, secretIds, tx); + + for (const change of secretChanges) { + const secretVersion = secretVersions[change.id]; + // eslint-disable-next-line no-continue + if (!secretVersion) continue; + + // Get the latest version from our batch result + const latestVersion = latestVersionsMap[secretVersion.secretId]; + const nextVersion = latestVersion ? latestVersion.version + 1 : 1; + + switch (change.changeType) { + case "create": + { + const newSecret = [ + { + id: change.id, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + version: nextVersion, + type: secretVersion.type, + key: secretVersion.key, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + encryptedValue: secretVersion.encryptedValue, + encryptedComment: secretVersion.encryptedComment, + userId: secretVersion.userId, + folderId + } + ]; + await secretV2BridgeDAL.insertMany(newSecret, tx); + + const metadata: { key: string; value: string }[] = + (secretVersion.metadata as { key: string; value: string }[]) || []; + if (metadata.length > 0) { + await resourceMetadataDAL.insertMany( + metadata.map(({ key, value }) => ({ + key, + value, + secretId: change.id, + orgId: project.orgId + })), + tx + ); + } + + const newVersion = await secretVersionV2BridgeDAL.create( + { + folderId, + secretId: secretVersion.secretId, + version: nextVersion, + encryptedValue: secretVersion.encryptedValue, + key: secretVersion.key, + encryptedComment: secretVersion.encryptedComment, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + userId: secretVersion.userId, + actorType: actorInfo.actorType, + envId: secretVersion.envId, + metadata: JSON.stringify(metadata), + ...(actorInfo.actorType === ActorType.IDENTITY && { identityActorId: actorInfo.actorId }), + ...(actorInfo.actorType === ActorType.USER && { userActorId: actorInfo.actorId }) + }, + tx + ); + + const secretTagsToBeInsert: TSecretV2TagJunctionInsert[] = []; + const secretTags = await secretTagDAL.findSecretTagsByVersionId(secretVersion.id, tx); + secretTags.forEach((tag) => { + secretTagsToBeInsert.push({ secrets_v2Id: change.id, secret_tagsId: tag.secret_tagsId }); + }); + await secretTagDAL.saveTagsToSecretV2(secretTagsToBeInsert, tx); + await secretTagDAL.saveTagsToSecretVersionV2( + secretTagsToBeInsert.map((tag) => ({ + secret_tagsId: tag.secret_tagsId, + secret_versions_v2Id: newVersion.id + })), + tx + ); + + commitChanges.push({ + type: ChangeType.ADD, + secretVersionId: newVersion.id + }); + } + break; + + case "update": + { + await secretV2BridgeDAL.updateById( + change.id, + { + skipMultilineEncoding: secretVersion?.skipMultilineEncoding, + version: nextVersion, + type: secretVersion?.type, + key: secretVersion?.key, + reminderNote: secretVersion?.reminderNote, + reminderRepeatDays: secretVersion?.reminderRepeatDays, + encryptedValue: secretVersion?.encryptedValue, + encryptedComment: secretVersion?.encryptedComment, + userId: secretVersion?.userId + }, + tx + ); + + const metadata: { key: string; value: string }[] = + (secretVersion.metadata as { key: string; value: string }[]) || []; + await resourceMetadataDAL.delete({ secretId: change.id }, tx); + if (metadata.length > 0) { + await resourceMetadataDAL.insertMany( + metadata.map(({ key, value }) => ({ + key, + value, + secretId: change.id, + orgId: project.orgId + })), + tx + ); + } + + const newVersion = await secretVersionV2BridgeDAL.create( + { + version: nextVersion, + encryptedValue: secretVersion.encryptedValue, + key: secretVersion.key, + encryptedComment: secretVersion.encryptedComment, + skipMultilineEncoding: secretVersion.skipMultilineEncoding, + reminderNote: secretVersion.reminderNote, + reminderRepeatDays: secretVersion.reminderRepeatDays, + userId: secretVersion.userId, + metadata: JSON.stringify(metadata), + actorType: actorInfo.actorType, + envId: secretVersion.envId, + folderId, + secretId: secretVersion.secretId, + ...(actorInfo.actorType === ActorType.IDENTITY && { identityActorId: actorInfo.actorId }), + ...(actorInfo.actorType === ActorType.USER && { userActorId: actorInfo.actorId }) + }, + tx + ); + + let secretTagsToBeInsert: TSecretV2TagJunctionInsert[] = []; + const secretTagsToBeDelete: string[] = []; + const secretTags = await secretTagDAL.findSecretTagsByVersionId(secretVersion.id, tx); + secretTags.forEach((tag) => { + secretTagsToBeInsert.push({ secrets_v2Id: change.id, secret_tagsId: tag.secret_tagsId }); + }); + const currentTags = await secretTagDAL.findSecretTagsBySecretId(change.id, tx); + currentTags.forEach((tag) => { + if (!secretTagsToBeInsert.find((t) => t.secret_tagsId === tag.secret_tagsId)) { + secretTagsToBeDelete.push(tag.secret_tagsId); + secretTagsToBeInsert = secretTagsToBeInsert.filter((t) => t.secret_tagsId !== tag.secret_tagsId); + } + }); + await secretTagDAL.saveTagsToSecretV2(secretTagsToBeInsert, tx); + await secretTagDAL.saveTagsToSecretVersionV2( + secretTagsToBeInsert.map((tag) => ({ + secret_tagsId: tag.secret_tagsId, + secret_versions_v2Id: newVersion.id + })), + tx + ); + await secretTagDAL.deleteTagsToSecretV2( + { $in: { secret_tagsId: secretTagsToBeDelete }, secrets_v2Id: change.id }, + tx + ); + + commitChanges.push({ + type: ChangeType.ADD, + isUpdate: true, + secretVersionId: newVersion.id + }); + } + break; + + // Delete case remains unchanged + case "delete": + await secretV2BridgeDAL.deleteById(change.id, tx); + commitChanges.push({ + type: ChangeType.DELETE, + secretVersionId: change.versionId + }); + break; + + default: + throw new BadRequestError({ message: `Unknown change type: ${change.changeType}` }); + } + } + + return commitChanges; + }; + + /** + * Core function to apply folder state differences + */ + const applyFolderStateDifferencesFn = async ({ + differences, + actorInfo, + folderId, + projectId, + reconstructNewFolders, + reconstructUpToCommit, + step = 0, + tx + }: { + differences: ResourceChange[]; + actorInfo: ActorInfo; + folderId: string; + projectId: string; + reconstructNewFolders: boolean; + reconstructUpToCommit?: string; + step: number; + tx?: Knex; + }): Promise => { + /** + * Process folder changes when applying folder state differences + */ + const processFolderChanges = async ( + changes: ResourceChange[], + folderVersions: Record + ) => { + const commitChanges = []; + + // Filter only folder changes using discriminated union + const folderChanges = changes.filter( + (change): change is ResourceChange & FolderChange => change.type === ResourceType.FOLDER + ); + + for (const change of folderChanges) { + const folderVersion = folderVersions[change.id]; + + switch (change.changeType) { + case "create": + if (folderVersion) { + const newFolder = { + id: change.id, + parentId: folderId, + envId: folderVersion.envId, + version: (folderVersion.version || 1) + 1, + name: folderVersion.name, + description: folderVersion.description + }; + await folderDAL.create(newFolder, tx); + + const newFolderVersion = await folderVersionDAL.create( + { + folderId: change.id, + version: (folderVersion.version || 1) + 1, + name: folderVersion.name, + description: folderVersion.description, + envId: folderVersion.envId + }, + tx + ); + + if (reconstructNewFolders && reconstructUpToCommit && step < 20) { + const subFolderLatestCommit = await folderCommitDAL.findLatestCommitBetween({ + folderId: change.id, + endCommitId: reconstructUpToCommit, + tx + }); + if (subFolderLatestCommit) { + const subFolderDiff = await compareFolderStates({ + targetCommitId: subFolderLatestCommit.id, + tx + }); + if (subFolderDiff?.length > 0) { + await applyFolderStateDifferencesFn({ + differences: subFolderDiff, + actorInfo, + folderId: change.id, + projectId, + reconstructNewFolders, + reconstructUpToCommit, + step: step + 1, + tx + }); + } + } + } + + commitChanges.push({ + type: ChangeType.ADD, + folderVersionId: newFolderVersion.id + }); + } + break; + + case "update": + if (change.versionId) { + const latestVersionDetails = await folderVersionDAL.findByIdsWithLatestVersion( + [change.id], + [change.versionId], + tx + ); + if (latestVersionDetails && Object.keys(latestVersionDetails).length > 0) { + const versionDetails = Object.values(latestVersionDetails)[0]; + await folderDAL.updateById( + change.id, + { + parentId: folderId, + envId: versionDetails.envId, + version: (versionDetails.version || 1) + 1, + name: versionDetails.name, + description: versionDetails.description + }, + tx + ); + + const newFolderVersion = await folderVersionDAL.create( + { + folderId: change.id, + version: (versionDetails.version || 1) + 1, + name: versionDetails.name, + description: versionDetails.description, + envId: versionDetails.envId + }, + tx + ); + + commitChanges.push({ + type: ChangeType.ADD, + isUpdate: true, + folderVersionId: newFolderVersion.id + }); + } + } + break; + + case "delete": + await folderDAL.deleteById(change.id, tx); + + commitChanges.push({ + type: ChangeType.DELETE, + folderVersionId: change.versionId, + folderId: change.id + }); + break; + + default: + throw new BadRequestError({ message: `Unknown change type: ${change.changeType}` }); + } + } + return commitChanges; + }; + + // Group differences by type for more efficient processing using discriminated unions + const secretChanges = differences.filter( + (diff): diff is ResourceChange & SecretChange => diff.type === ResourceType.SECRET + ); + const folderChanges = differences.filter( + (diff): diff is ResourceChange & FolderChange => diff.type === ResourceType.FOLDER + ); + + // Batch fetch necessary data + const secretVersions = await secretVersionV2BridgeDAL.findByIdsWithLatestVersion( + folderId, + secretChanges.map((diff) => diff.id), + secretChanges.map((diff) => diff.versionId), + tx + ); + + const folderVersions = await folderVersionDAL.findByIdsWithLatestVersion( + folderChanges.map((diff) => diff.id), + folderChanges.map((diff) => diff.versionId), + tx + ); + + // Process changes in parallel + const [secretCommitChanges, folderCommitChanges] = await Promise.all([ + processSecretChanges(differences, secretVersions, actorInfo, folderId, tx), + processFolderChanges(differences, folderVersions) + ]); + + // Combine all changes + const allCommitChanges = [...secretCommitChanges, ...folderCommitChanges]; + + // Create a commit with all the changes + await createCommit( + { + actor: { + type: actorInfo.actorType, + metadata: { id: actorInfo.actorId } + }, + message: actorInfo.message || "Rolled back folder state", + folderId, + changes: allCommitChanges, + omitIgnoreFilter: true + }, + tx + ); + + // Invalidate cache to reflect the changes + await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId); + + return { + secretChangesCount: secretChanges.length, + folderChangesCount: folderChanges.length, + totalChanges: differences.length + }; + }; + + /** + * Apply folder state differences with transaction handling + */ + const applyFolderStateDifferences = async (params: { + differences: ResourceChange[]; + actorInfo: ActorInfo; + folderId: string; + projectId: string; + reconstructNewFolders: boolean; + reconstructUpToCommit?: string; + tx?: Knex; + }): Promise => { + // If a transaction was provided, use it directly + if (params.tx) { + return applyFolderStateDifferencesFn({ ...params, step: 0 }); + } + + // Otherwise, start a new transaction + return folderCommitDAL.transaction((newTx) => applyFolderStateDifferencesFn({ ...params, tx: newTx, step: 0 })); + }; + + /** + * Retrieve a commit by ID + */ + const getCommitById = async ({ + commitId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + tx + }: { + commitId: string; + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + tx?: Knex; + }) => { + await checkProjectCommitReadPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + return folderCommitDAL.findById(commitId, tx, projectId); + }; + + /** + * Get all commits for a folder + */ + const getCommitsByFolderId = async (folderId: string, tx?: Knex) => { + return folderCommitDAL.findByFolderId(folderId, tx); + }; + + const getCommitsForFolder = async ({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + environment, + path, + offset = 0, + limit = 20, + search, + sort = "desc" + }: { + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + environment: string; + path: string; + offset: number; + limit: number; + search?: string; + sort: "asc" | "desc"; + }) => { + await checkProjectCommitReadPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + const folder = await folderDAL.findBySecretPath(projectId, environment, path); + if (!folder) { + throw new NotFoundError({ + message: `Folder not found for project ID ${projectId}, environment ${environment}, path ${path}` + }); + } + const folderCommits = await folderCommitDAL.findByFolderIdPaginated(folder.id, { + offset, + limit, + search, + sort + }); + return folderCommits; + }; + + const getCommitsCount = async ({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + environment, + path + }: { + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + environment: string; + path: string; + }) => { + await checkProjectCommitReadPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + + const folder = await folderDAL.findBySecretPath(projectId, environment, path); + if (!folder) { + throw new NotFoundError({ + message: `Folder not found for project ID ${projectId}, environment ${environment}, path ${path}` + }); + } + const folderCommits = await folderCommitDAL.findByFolderId(folder.id); + return { count: folderCommits.length, folderId: folder.id }; + }; + + /** + * Get changes for a commit + */ + const getCommitChanges = async ({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + commitId + }: { + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + commitId: string; + }) => { + await checkProjectCommitReadPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + const changes = await folderCommitChangesDAL.findByCommitId(commitId, projectId); + const commit = await folderCommitDAL.findById(commitId, undefined, projectId); + const latestCommit = await folderCommitDAL.findLatestCommit(commit.folderId, projectId); + return { ...commit, changes, isLatest: commit.id === latestCommit?.id }; + }; + + /** + * Get checkpoints for a folder + */ + const getCheckpointsByFolderId = async (folderId: string, limit?: number, tx?: Knex) => { + return folderCheckpointDAL.findByFolderId(folderId, limit, tx); + }; + + /** + * Get the latest checkpoint for a folder + */ + const getLatestCheckpoint = async (folderId: string, tx?: Knex) => { + return folderCheckpointDAL.findLatestByFolderId(folderId, tx); + }; + + /** + * Initialize a folder with its current state + */ + const getFolderInitialChanges = async (folderId: string, envId: string, tx?: Knex) => { + const folderResources = await getFolderResources(folderId, tx); + const changes = folderResources.map((resource) => ({ type: ChangeType.ADD, ...resource })); + + if (changes.length > 0) { + return { + commit: { + actorMetadata: {}, + actorType: ActorType.PLATFORM, + message: "Initialized folder", + folderId, + envId + }, + changes: changes.map((change) => ({ + folderId, + changeType: change.type, + secretVersionId: change.secretVersionId, + folderVersionId: change.folderVersionId, + isUpdate: false + })) + }; + } + return {}; + }; + + /** + * Sort folders by hierarchy (parents before children) + */ + const sortFoldersByHierarchy = (folders: TSecretFolders[]) => { + // Create a map for quick lookup of children by parent ID + const childrenMap = new Map(); + + // Set of all folder IDs + const allFolderIds = new Set(); + + // Build the set of all folder IDs + folders.forEach((folder) => { + if (folder.id) { + allFolderIds.add(folder.id); + } + }); + + // Group folders by their parentId + folders.forEach((folder) => { + if (folder.parentId) { + const children = childrenMap.get(folder.parentId) || []; + children.push(folder); + childrenMap.set(folder.parentId, children); + } + }); + + // Find root folders - those with no parentId or with a parentId that doesn't exist + const rootFolders = folders.filter((folder) => !folder.parentId || !allFolderIds.has(folder.parentId)); + + // Process each level of the hierarchy + const result = []; + let currentLevel = rootFolders; + + while (currentLevel.length > 0) { + result.push(...currentLevel); + + const nextLevel = []; + for (const folder of currentLevel) { + if (folder.id) { + const children = childrenMap.get(folder.id) || []; + nextLevel.push(...children); + } + } + + currentLevel = nextLevel; + } + + return result; + }; + + /** + * Create a checkpoint for a folder tree + */ + const createFolderTreeCheckpoint = async (envId: string, folderCommitId?: string, tx?: Knex) => { + let latestCommitId = folderCommitId; + const latestTreeCheckpoint = await folderTreeCheckpointDAL.findLatestByEnvId(envId, tx); + + if (!latestCommitId) { + const latestCommit = await folderCommitDAL.findLatestEnvCommit(envId, tx); + if (!latestCommit) { + logger.info(`createFolderTreeCheckpoint - Latest commit ID not found for envId ${envId}`); + return; + } + latestCommitId = latestCommit.id; + } + + if (latestTreeCheckpoint) { + const commitsSinceLastCheckpoint = await folderCommitDAL.getEnvNumberOfCommitsSince( + envId, + latestTreeCheckpoint.folderCommitId, + tx + ); + if (commitsSinceLastCheckpoint < Number(appCfg.PIT_TREE_CHECKPOINT_WINDOW)) { + logger.info( + `createFolderTreeCheckpoint - Commits since last checkpoint ${commitsSinceLastCheckpoint} is less than ${appCfg.PIT_TREE_CHECKPOINT_WINDOW}` + ); + return; + } + } + + const folders = await folderDAL.findByEnvId(envId, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + const filteredFoldersIds = sortedFolders.filter((folder) => !folder.isReserved).map((folder) => folder.id); + const folderCommits = await folderCommitDAL.findMultipleLatestCommits(filteredFoldersIds, tx); + const folderTreeCheckpoint = await folderTreeCheckpointDAL.create( + { + folderCommitId: latestCommitId + }, + tx + ); + await folderTreeCheckpointResourcesDAL.insertMany( + folderCommits.map((folderCommit) => ({ + folderTreeCheckpointId: folderTreeCheckpoint.id, + folderId: folderCommit.folderId, + folderCommitId: folderCommit.id + })), + tx + ); + }; + + const addNestedFolderChanges = async ({ + changes, + beforeCommit, + folderId, + folderName, + folderPath, + step = 1, + tx + }: { + changes: { + folderId: string; + folderName: string; + changes: ResourceChange[]; + folderPath?: string; + }[]; + beforeCommit: bigint; + folderId: string; + folderName?: string; + folderPath?: string; + step?: number; + tx?: Knex; + }) => { + if (step > 20) { + return; + } + const latestFolderCommit = await folderCommitDAL.findCommitBefore(folderId, beforeCommit, tx); + if (!latestFolderCommit) { + return; + } + const diff = await compareFolderStates({ + targetCommitId: latestFolderCommit.id, + tx + }); + changes.push({ + folderId, + folderName: folderName || "", + changes: diff, + folderPath: folderPath || "" + }); + await Promise.all( + diff.map(async (change) => { + if (change.type === ResourceType.FOLDER && change.changeType === ChangeType.CREATE) { + await addNestedFolderChanges({ + changes, + beforeCommit, + folderId: change.id, + folderName: change.folderName, + folderPath: `${folderPath}/${change.folderName}`, + step: step + 1, + tx + }); + } + }) + ); + }; + + const deepCompareFolder = async ({ + targetCommitId, + envId, + projectId, + tx + }: { + targetCommitId: string; + envId: string; + projectId: string; + tx?: Knex; + }) => { + const targetCommit = await folderCommitDAL.findById(targetCommitId, tx); + if (!targetCommit) { + throw new NotFoundError({ message: `No commit found for commit ID ${targetCommitId}` }); + } + + const checkpoint = await folderTreeCheckpointDAL.findNearestCheckpoint(targetCommit.commitId, envId, tx); + if (!checkpoint) { + throw new NotFoundError({ message: `No checkpoint found for commit ID ${targetCommitId}` }); + } + + const folderCheckpointCommits = await folderTreeCheckpointResourcesDAL.findByTreeCheckpointId(checkpoint.id, tx); + const folderCommits = await folderCommitDAL.findAllCommitsBetween({ + envId, + startCommitId: checkpoint.commitId.toString(), + tx + }); + + // Group commits by folderId and keep only the latest + const folderGroups = new Map(); + + if (folderCheckpointCommits && folderCheckpointCommits.length > 0) { + for (const commit of folderCheckpointCommits) { + if (commit.commitId > targetCommit.commitId) { + folderGroups.set(commit.folderId, { + commitId: commit.commitId, + id: commit.folderCommitId + }); + } + } + } + + if (folderCommits && folderCommits.length > 0) { + for (const commit of folderCommits) { + const { folderId, commitId, id } = commit; + const existingCommit = folderGroups.get(folderId); + + if ((!existingCommit || commitId > existingCommit.commitId) && commitId > targetCommit.commitId) { + folderGroups.set(folderId, { commitId, id }); + } + } + } + + const folderDiffs = new Map(); + + // Process each folder to determine differences + await Promise.all( + Array.from(folderGroups.entries()).map(async ([folderId, commit]) => { + const previousCommit = await folderCommitDAL.findPreviousCommitTo( + folderId, + targetCommit.commitId.toString(), + tx + ); + let diff = []; + if (previousCommit && previousCommit.id !== commit.id) { + diff = await compareFolderStates({ + currentCommitId: commit.id, + targetCommitId: previousCommit.id, + tx + }); + } else { + diff = await compareFolderStates({ + targetCommitId: commit.id, + defaultOperation: "delete", + tx + }); + } + if (diff?.length > 0) { + folderDiffs.set(folderId, diff); + } + }) + ); + + // Apply changes in hierarchical order + const folderIds = Array.from(folderDiffs.keys()); + const folders = await folderDAL.findFoldersByRootAndIds({ rootId: targetCommit.folderId, folderIds }, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + + const response: { + folderId: string; + folderName: string; + changes: ResourceChange[]; + folderPath?: string; + }[] = []; + for (const folder of sortedFolders) { + const diff = folderDiffs.get(folder.id); + if (diff) { + const folderPath = await folderDAL.findSecretPathByFolderIds(projectId, [folder.id]); + response.push({ + folderId: folder.id, + folderName: folder.name, + changes: diff, + folderPath: folderPath?.[0]?.path + }); + const recreatedFolders = diff + .filter( + (change): change is FolderChange => + change.type === ResourceType.FOLDER && change.changeType === ChangeType.CREATE + ) + .map((change) => ({ + id: change.id, + folderName: change.folderName, + folderPath: folderPath?.[0]?.path + })); + await Promise.all( + recreatedFolders.map(async (change) => { + const nestedFolderPath = folderPath?.[0]?.path; + await addNestedFolderChanges({ + changes: response, + beforeCommit: targetCommit.commitId, + folderId: change.id, + folderName: change.folderName, + folderPath: `${nestedFolderPath !== "/" ? nestedFolderPath : ""}/${change.folderName}`, + tx + }); + }) + ); + } + } + return response; + }; + + /** + * Roll back a folder tree to a specific commit + */ + const deepRollbackFolder = async ( + targetCommitId: string, + envId: string, + actorId: string, + actorType: ActorType, + projectId: string, + message?: string + ) => { + await folderCommitDAL.transaction(async (tx) => { + const targetCommit = await folderCommitDAL.findById(targetCommitId, tx); + if (!targetCommit) { + throw new NotFoundError({ message: `No commit found for commit ID ${targetCommitId}` }); + } + + const checkpoint = await folderTreeCheckpointDAL.findNearestCheckpoint(targetCommit.commitId, envId, tx); + if (!checkpoint) { + throw new NotFoundError({ message: `No checkpoint found for commit ID ${targetCommitId}` }); + } + + const folderCheckpointCommits = await folderTreeCheckpointResourcesDAL.findByTreeCheckpointId(checkpoint.id, tx); + const folderCommits = await folderCommitDAL.findAllCommitsBetween({ + envId, + startCommitId: checkpoint.commitId.toString(), + tx + }); + + // Group commits by folderId and keep only the latest + const folderGroups = new Map(); + + if (folderCheckpointCommits && folderCheckpointCommits.length > 0) { + for (const commit of folderCheckpointCommits) { + if (commit.commitId > targetCommit.commitId) { + folderGroups.set(commit.folderId, { + commitId: commit.commitId, + id: commit.folderCommitId + }); + } + } + } + + if (folderCommits && folderCommits.length > 0) { + for (const commit of folderCommits) { + const { folderId, commitId, id } = commit; + const existingCommit = folderGroups.get(folderId); + + if ((!existingCommit || commitId > existingCommit.commitId) && commitId > targetCommit.commitId) { + folderGroups.set(folderId, { commitId, id }); + } + } + } + + const folderDiffs = new Map(); + + // Process each folder to determine differences + await Promise.all( + Array.from(folderGroups.entries()).map(async ([folderId, { id }]) => { + const previousCommit = await folderCommitDAL.findPreviousCommitTo( + folderId, + targetCommit.commitId.toString(), + tx + ); + if (previousCommit && previousCommit.id !== id) { + const diff = await compareFolderStates({ + currentCommitId: id, + targetCommitId: previousCommit.id, + tx + }); + if (diff?.length > 0) { + folderDiffs.set(folderId, diff); + } + } + }) + ); + + const foldersToDelete = new Set(); + + // Process all DELETE operations to build a complete set of folders to be deleted + for (const changes of folderDiffs.values()) { + for (const change of changes) { + if (change.changeType === ChangeType.DELETE && change.type === ResourceType.FOLDER) { + foldersToDelete.add(change.id); + } + } + } + + // Now, remove any folder that is being deleted from the folderDiffs map + // before applying any changes + for (const folderId of foldersToDelete) { + folderDiffs.delete(folderId); + } + + // Apply changes in hierarchical order + const folderIds = Array.from(folderDiffs.keys()); + const folders = await folderDAL.findFoldersByRootAndIds({ rootId: targetCommit.folderId, folderIds }, tx); + const sortedFolders = sortFoldersByHierarchy(folders); + + for (const folder of sortedFolders) { + const diff = folderDiffs.get(folder.id); + if (diff) { + await applyFolderStateDifferences({ + differences: diff, + actorInfo: { + actorType, + actorId, + message: message || "Deep rollback" + }, + folderId: folder.id, + projectId, + reconstructNewFolders: true, + reconstructUpToCommit: targetCommit.commitId.toString(), + tx + }); + } + } + }); + }; + + const getLatestCommit = async ({ + folderId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }: { + folderId: string; + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + }) => { + await checkProjectCommitReadPermission({ + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId + }); + return folderCommitDAL.findLatestCommit(folderId, projectId); + }; + + /** + * Revert changes made in a specific commit + */ + const revertCommitChanges = async ({ + commitId, + actor, + actorId, + actorAuthMethod, + actorOrgId, + projectId, + message = "Revert commit changes" + }: { + commitId: string; + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string; + projectId: string; + message?: string; + }) => { + if (!permissionService) { + throw new Error("Permission service not initialized"); + } + const { permission } = await permissionService.getProjectPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId, + actionProjectType: ActionProjectType.SecretManager + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionCommitsActions.PerformRollback, + ProjectPermissionSub.Commits + ); + // Check permissions first + await checkProjectCommitReadPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + }); + + // Get the commit to revert + const commitToRevert = await folderCommitDAL.findById(commitId, undefined, projectId); + if (!commitToRevert) { + throw new NotFoundError({ message: `Commit with ID ${commitId} not found` }); + } + + const previousCommit = await folderCommitDAL.findCommitBefore(commitToRevert.folderId, commitToRevert.commitId); + + if (!previousCommit) { + throw new BadRequestError({ message: "Cannot revert the first commit" }); + } + + // Calculate the changes needed to go from current commit back to the previous one + const inverseChanges = await compareFolderStates({ + currentCommitId: commitToRevert.id, + targetCommitId: previousCommit.id + }); + + const latestCommit = await folderCommitDAL.findLatestCommit(commitToRevert.folderId); + if (!latestCommit) { + throw new NotFoundError({ message: `Latest commit not found for folder ${commitToRevert.folderId}` }); + } + const currentState = await reconstructFolderState(latestCommit.id); + + const filteredChanges = inverseChanges.filter( + (change) => + ((change.changeType === ChangeType.DELETE || change.changeType === ChangeType.UPDATE) && + (currentState.some((c) => c.id === change.id) || currentState.some((c) => c.id === change.id))) || + (change.changeType === ChangeType.CREATE && + (currentState.every((c) => c.id !== change.id) || currentState.every((c) => c.id !== change.id))) + ); + + if (!filteredChanges || filteredChanges.length === 0) { + return { + success: true, + message: "No changes to revert", + originalCommitId: commitId + }; + } + + // Apply the changes to revert the commit + const revertResult = await applyFolderStateDifferences({ + differences: filteredChanges, + actorInfo: { + actorType: actor, + actorId, + message: message || `Reverted changes from commit ${commitId}` + }, + folderId: commitToRevert.folderId, + projectId, + reconstructNewFolders: true, + reconstructUpToCommit: commitToRevert.commitId.toString() + }); + + return { + success: true, + message: "Changes reverted successfully", + originalCommitId: commitId, + revertCommitId: latestCommit?.id, + changesReverted: revertResult.totalChanges + }; + }; + + return { + createCommit, + addCommitChange, + getCommitById, + getCommitsByFolderId, + getCommitChanges, + getCheckpointsByFolderId, + getLatestCheckpoint, + getFolderInitialChanges, + createFolderCheckpoint, + compareFolderStates, + applyFolderStateDifferences, + createFolderTreeCheckpoint, + deepRollbackFolder, + getCommitsCount, + getLatestCommit, + deepCompareFolder, + reconstructFolderState, + getCommitsForFolder, + revertCommitChanges + }; +}; + +export type TFolderCommitServiceFactory = ReturnType; diff --git a/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts new file mode 100644 index 000000000..58651a48a --- /dev/null +++ b/backend/src/services/folder-tree-checkpoint-resources/folder-tree-checkpoint-resources-dal.ts @@ -0,0 +1,44 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderTreeCheckpointResources } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderTreeCheckpointResourcesDALFactory = ReturnType; + +type TFolderTreeCheckpointResourcesWithCommitId = TFolderTreeCheckpointResources & { + commitId: bigint; +}; + +export const folderTreeCheckpointResourcesDALFactory = (db: TDbClient) => { + const folderTreeCheckpointResourcesOrm = ormify(db, TableName.FolderTreeCheckpointResources); + + const findByTreeCheckpointId = async ( + folderTreeCheckpointId: string, + tx?: Knex + ): Promise => { + try { + const docs = await (tx || db.replicaNode())( + TableName.FolderTreeCheckpointResources + ) + .join( + TableName.FolderCommit, + `${TableName.FolderTreeCheckpointResources}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderTreeCheckpointId }, TableName.FolderTreeCheckpointResources)) + .select(selectAllTableCols(TableName.FolderTreeCheckpointResources)) + .select(db.ref("commitId").withSchema(TableName.FolderCommit).as("commitId")); + return docs; + } catch (error) { + throw new DatabaseError({ error, name: "FindByTreeCheckpointId" }); + } + }; + + return { + ...folderTreeCheckpointResourcesOrm, + findByTreeCheckpointId + }; +}; diff --git a/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts new file mode 100644 index 000000000..cc0634e74 --- /dev/null +++ b/backend/src/services/folder-tree-checkpoint/folder-tree-checkpoint-dal.ts @@ -0,0 +1,79 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName, TFolderCommits, TFolderTreeCheckpoints } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; + +export type TFolderTreeCheckpointDALFactory = ReturnType; + +type TreeCheckpointWithCommitInfo = TFolderTreeCheckpoints & { + commitId: bigint; +}; + +export const folderTreeCheckpointDALFactory = (db: TDbClient) => { + const folderTreeCheckpointOrm = ormify(db, TableName.FolderTreeCheckpoint); + + const findByCommitId = async (folderCommitId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderTreeCheckpoint) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ folderCommitId }, TableName.FolderTreeCheckpoint)) + .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindByCommitId" }); + } + }; + + const findNearestCheckpoint = async ( + folderCommitId: bigint, + envId: string, + tx?: Knex + ): Promise => { + try { + const nearestCheckpoint = await (tx || db.replicaNode())(TableName.FolderTreeCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderTreeCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(`${TableName.FolderCommit}.envId`, "=", envId) + .andWhere(`${TableName.FolderCommit}.commitId`, "<=", folderCommitId.toString()) + .select(selectAllTableCols(TableName.FolderTreeCheckpoint)) + .select(db.ref("commitId").withSchema(TableName.FolderCommit)) + .orderBy(`${TableName.FolderCommit}.commitId`, "desc") + .first(); + + return nearestCheckpoint; + } catch (error) { + throw new DatabaseError({ error, name: "FindNearestCheckpoint" }); + } + }; + + const findLatestByEnvId = async (envId: string, tx?: Knex): Promise => { + try { + const doc = await (tx || db.replicaNode())(TableName.FolderTreeCheckpoint) + .join( + TableName.FolderCommit, + `${TableName.FolderTreeCheckpoint}.folderCommitId`, + `${TableName.FolderCommit}.id` + ) + .where(`${TableName.FolderCommit}.envId`, "=", envId) + .orderBy(`${TableName.FolderTreeCheckpoint}.createdAt`, "desc") + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "FindLatestByEnvId" }); + } + }; + + return { + ...folderTreeCheckpointOrm, + findByCommitId, + findNearestCheckpoint, + findLatestByEnvId + }; +}; diff --git a/backend/src/services/group-project/group-project-service.ts b/backend/src/services/group-project/group-project-service.ts index 1ff2c78d2..a793ecfab 100644 --- a/backend/src/services/group-project/group-project-service.ts +++ b/backend/src/services/group-project/group-project-service.ts @@ -1,6 +1,7 @@ import { ForbiddenError } from "@casl/ability"; import { ActionProjectType, ProjectMembershipRole, SecretKeyEncoding, TGroups } from "@app/db/schemas"; +import { TListProjectGroupUsersDTO } from "@app/ee/services/group/group-types"; import { constructPermissionErrorMessage, validatePrivilegeChangeOperation @@ -42,7 +43,7 @@ type TGroupProjectServiceFactoryDep = { projectKeyDAL: Pick; projectRoleDAL: Pick; projectBotDAL: TProjectBotDALFactory; - groupDAL: Pick; + groupDAL: Pick; permissionService: Pick; }; @@ -471,11 +472,54 @@ export const groupProjectServiceFactory = ({ return groupMembership; }; + const listProjectGroupUsers = async ({ + id, + projectId, + offset, + limit, + username, + actor, + actorId, + actorAuthMethod, + actorOrgId, + search, + filter + }: TListProjectGroupUsersDTO) => { + const project = await projectDAL.findById(projectId); + + if (!project) { + throw new NotFoundError({ message: `Failed to find project with ID ${projectId}` }); + } + + const { permission } = await permissionService.getProjectPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId, + actionProjectType: ActionProjectType.Any + }); + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionGroupActions.Read, ProjectPermissionSub.Groups); + + const { members, totalCount } = await groupDAL.findAllGroupPossibleMembers({ + orgId: project.orgId, + groupId: id, + offset, + limit, + username, + search, + filter + }); + + return { users: members, totalCount }; + }; + return { addGroupToProject, updateGroupInProject, removeGroupFromProject, listGroupsInProject, - getGroupInProject + getGroupInProject, + listProjectGroupUsers }; }; diff --git a/backend/src/services/identity-access-token/identity-access-token-types.ts b/backend/src/services/identity-access-token/identity-access-token-types.ts index c97d2f40a..87adfa5dc 100644 --- a/backend/src/services/identity-access-token/identity-access-token-types.ts +++ b/backend/src/services/identity-access-token/identity-access-token-types.ts @@ -11,5 +11,9 @@ export type TIdentityAccessTokenJwtPayload = { oidc?: { claims: Record; }; + kubernetes?: { + namespace: string; + name: string; + }; }; }; diff --git a/backend/src/services/identity-kubernetes-auth/identity-kubernetes-auth-service.ts b/backend/src/services/identity-kubernetes-auth/identity-kubernetes-auth-service.ts index 34a28c6a6..1f89745a9 100644 --- a/backend/src/services/identity-kubernetes-auth/identity-kubernetes-auth-service.ts +++ b/backend/src/services/identity-kubernetes-auth/identity-kubernetes-auth-service.ts @@ -20,8 +20,9 @@ import { import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { getConfig } from "@app/lib/config/env"; import { BadRequestError, NotFoundError, PermissionBoundaryError, UnauthorizedError } from "@app/lib/errors"; -import { withGatewayProxy } from "@app/lib/gateway"; +import { GatewayHttpProxyActions, GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway"; import { extractIPDetails, isValidIpOrCidr } from "@app/lib/ip"; +import { logger } from "@app/lib/logger"; import { ActorType, AuthTokenType } from "../auth/auth-type"; import { TIdentityOrgDALFactory } from "../identity/identity-org-dal"; @@ -33,6 +34,7 @@ import { validateIdentityUpdateForSuperAdminPrivileges } from "../super-admin/su import { TIdentityKubernetesAuthDALFactory } from "./identity-kubernetes-auth-dal"; import { extractK8sUsername } from "./identity-kubernetes-auth-fns"; import { + IdentityKubernetesAuthTokenReviewMode, TAttachKubernetesAuthDTO, TCreateTokenReviewResponse, TGetKubernetesAuthDTO, @@ -70,21 +72,27 @@ export const identityKubernetesAuthServiceFactory = ({ const $gatewayProxyWrapper = async ( inputs: { gatewayId: string; - targetHost: string; - targetPort: number; + targetHost?: string; + targetPort?: number; + caCert?: string; + reviewTokenThroughGateway: boolean; }, - gatewayCallback: (host: string, port: number) => Promise + gatewayCallback: (host: string, port: number, httpsAgent?: https.Agent) => Promise ): Promise => { const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(inputs.gatewayId); const [relayHost, relayPort] = relayDetails.relayAddress.split(":"); const callbackResult = await withGatewayProxy( - async (port) => { - // Needs to be https protocol or the kubernetes API server will fail with "Client sent an HTTP request to an HTTPS server" - const res = await gatewayCallback("https://localhost", port); + async (port, httpsAgent) => { + const res = await gatewayCallback( + inputs.reviewTokenThroughGateway ? "http://localhost" : "https://localhost", + port, + httpsAgent + ); return res; }, { + protocol: inputs.reviewTokenThroughGateway ? GatewayProxyProtocol.Http : GatewayProxyProtocol.Tcp, targetHost: inputs.targetHost, targetPort: inputs.targetPort, relayHost, @@ -95,7 +103,16 @@ export const identityKubernetesAuthServiceFactory = ({ ca: relayDetails.certChain, cert: relayDetails.certificate, key: relayDetails.privateKey.toString() - } + }, + // only needed for TCP protocol, because the gateway as reviewer will use the pod's CA cert for auth directly + ...(!inputs.reviewTokenThroughGateway + ? { + httpsAgent: new https.Agent({ + ca: inputs.caCert, + rejectUnauthorized: Boolean(inputs.caCert) + }) + } + : {}) } ); @@ -129,22 +146,36 @@ export const identityKubernetesAuthServiceFactory = ({ caCert = decryptor({ cipherTextBlob: identityKubernetesAuth.encryptedKubernetesCaCertificate }).toString(); } - let tokenReviewerJwt = ""; - if (identityKubernetesAuth.encryptedKubernetesTokenReviewerJwt) { - tokenReviewerJwt = decryptor({ - cipherTextBlob: identityKubernetesAuth.encryptedKubernetesTokenReviewerJwt - }).toString(); - } else { - // if no token reviewer is provided means the incoming token has to act as reviewer - tokenReviewerJwt = serviceAccountJwt; - } + const tokenReviewCallbackRaw = async (host = identityKubernetesAuth.kubernetesHost, port?: number) => { + logger.info({ host, port }, "tokenReviewCallbackRaw: Processing kubernetes token review using raw API"); - let { kubernetesHost } = identityKubernetesAuth; - if (kubernetesHost.startsWith("https://") || kubernetesHost.startsWith("http://")) { - kubernetesHost = new RE2("^https?:\\/\\/").replace(kubernetesHost, ""); - } + if (!host || !identityKubernetesAuth.kubernetesHost) { + throw new BadRequestError({ + message: "Kubernetes host is required when token review mode is set to API" + }); + } + + let tokenReviewerJwt = ""; + if (identityKubernetesAuth.encryptedKubernetesTokenReviewerJwt) { + tokenReviewerJwt = decryptor({ + cipherTextBlob: identityKubernetesAuth.encryptedKubernetesTokenReviewerJwt + }).toString(); + } else { + // if no token reviewer is provided means the incoming token has to act as reviewer + tokenReviewerJwt = serviceAccountJwt; + } + + let servername = identityKubernetesAuth.kubernetesHost; + if (servername.startsWith("https://") || servername.startsWith("http://")) { + servername = new RE2("^https?:\\/\\/").replace(servername, ""); + } + + // get the last colon index, if it has a port, remove it, including the colon + const lastColonIndex = servername.lastIndexOf(":"); + if (lastColonIndex !== -1) { + servername = servername.substring(0, lastColonIndex); + } - const tokenReviewCallback = async (host: string = identityKubernetesAuth.kubernetesHost, port?: number) => { const baseUrl = port ? `${host}:${port}` : host; const res = await axios @@ -165,11 +196,10 @@ export const identityKubernetesAuthServiceFactory = ({ }, signal: AbortSignal.timeout(10000), timeout: 10000, - // if ca cert, rejectUnauthorized: true httpsAgent: new https.Agent({ ca: caCert, rejectUnauthorized: Boolean(caCert), - servername: kubernetesHost + servername }) } ) @@ -192,18 +222,110 @@ export const identityKubernetesAuthServiceFactory = ({ return res.data; }; - const [k8sHost, k8sPort] = kubernetesHost.split(":"); + const tokenReviewCallbackThroughGateway = async (host: string, port?: number) => { + logger.info( + { + host, + port + }, + "tokenReviewCallbackThroughGateway: Processing kubernetes token review using gateway" + ); - const data = identityKubernetesAuth.gatewayId - ? await $gatewayProxyWrapper( + const res = await axios + .post( + `${host}:${port}/apis/authentication.k8s.io/v1/tokenreviews`, { - gatewayId: identityKubernetesAuth.gatewayId, - targetHost: k8sHost, - targetPort: k8sPort ? Number(k8sPort) : 443 + apiVersion: "authentication.k8s.io/v1", + kind: "TokenReview", + spec: { + token: serviceAccountJwt, + ...(identityKubernetesAuth.allowedAudience ? { audiences: [identityKubernetesAuth.allowedAudience] } : {}) + } }, - tokenReviewCallback + { + headers: { + "Content-Type": "application/json", + "x-infisical-action": GatewayHttpProxyActions.UseGatewayK8sServiceAccount + }, + signal: AbortSignal.timeout(10000), + timeout: 10000 + } ) - : await tokenReviewCallback(); + .catch((err) => { + if (err instanceof AxiosError) { + if (err.response) { + let { message } = err?.response?.data as unknown as { message?: string }; + + if (!message && typeof err.response.data === "string") { + message = err.response.data; + } + + if (message) { + throw new UnauthorizedError({ + message, + name: "KubernetesTokenReviewRequestError" + }); + } + } + } + throw err; + }); + + return res.data; + }; + + let data: TCreateTokenReviewResponse | undefined; + + if (identityKubernetesAuth.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Gateway) { + if (!identityKubernetesAuth.gatewayId) { + throw new BadRequestError({ + message: "Gateway ID is required when token review mode is set to Gateway" + }); + } + + data = await $gatewayProxyWrapper( + { + gatewayId: identityKubernetesAuth.gatewayId, + reviewTokenThroughGateway: true + }, + tokenReviewCallbackThroughGateway + ); + } else if (identityKubernetesAuth.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api) { + if (!identityKubernetesAuth.kubernetesHost) { + throw new BadRequestError({ + message: "Kubernetes host is required when token review mode is set to API" + }); + } + + let { kubernetesHost } = identityKubernetesAuth; + if (kubernetesHost.startsWith("https://") || kubernetesHost.startsWith("http://")) { + kubernetesHost = new RE2("^https?:\\/\\/").replace(kubernetesHost, ""); + } + + const [k8sHost, k8sPort] = kubernetesHost.split(":"); + + data = identityKubernetesAuth.gatewayId + ? await $gatewayProxyWrapper( + { + gatewayId: identityKubernetesAuth.gatewayId, + targetHost: k8sHost, + targetPort: k8sPort ? Number(k8sPort) : 443, + reviewTokenThroughGateway: false + }, + tokenReviewCallbackRaw + ) + : await tokenReviewCallbackRaw(); + } else { + throw new BadRequestError({ + message: `Invalid token review mode: ${identityKubernetesAuth.tokenReviewMode}` + }); + } + + if (!data) { + throw new BadRequestError({ + message: "Failed to review token" + }); + } if ("error" in data.status) throw new UnauthorizedError({ message: data.status.error, name: "KubernetesTokenReviewError" }); @@ -278,7 +400,13 @@ export const identityKubernetesAuthServiceFactory = ({ { identityId: identityKubernetesAuth.identityId, identityAccessTokenId: identityAccessToken.id, - authTokenType: AuthTokenType.IDENTITY_ACCESS_TOKEN + authTokenType: AuthTokenType.IDENTITY_ACCESS_TOKEN, + identityAuth: { + kubernetes: { + namespace: targetNamespace, + name: targetName + } + } } as TIdentityAccessTokenJwtPayload, appCfg.AUTH_SECRET, // akhilmhdh: for non-expiry tokens you should not even set the value, including undefined. Even for undefined jsonwebtoken throws error @@ -298,6 +426,7 @@ export const identityKubernetesAuthServiceFactory = ({ kubernetesHost, caCert, tokenReviewerJwt, + tokenReviewMode, allowedNamespaces, allowedNames, allowedAudience, @@ -384,6 +513,7 @@ export const identityKubernetesAuthServiceFactory = ({ { identityId: identityMembershipOrg.identityId, kubernetesHost, + tokenReviewMode, allowedNamespaces, allowedNames, allowedAudience, @@ -410,6 +540,7 @@ export const identityKubernetesAuthServiceFactory = ({ kubernetesHost, caCert, tokenReviewerJwt, + tokenReviewMode, allowedNamespaces, allowedNames, allowedAudience, @@ -492,6 +623,7 @@ export const identityKubernetesAuthServiceFactory = ({ const updateQuery: TIdentityKubernetesAuthsUpdate = { kubernetesHost, + tokenReviewMode, allowedNamespaces, allowedNames, allowedAudience, diff --git a/backend/src/services/identity-kubernetes-auth/identity-kubernetes-auth-types.ts b/backend/src/services/identity-kubernetes-auth/identity-kubernetes-auth-types.ts index 12edd266f..269fa19e0 100644 --- a/backend/src/services/identity-kubernetes-auth/identity-kubernetes-auth-types.ts +++ b/backend/src/services/identity-kubernetes-auth/identity-kubernetes-auth-types.ts @@ -5,11 +5,17 @@ export type TLoginKubernetesAuthDTO = { jwt: string; }; +export enum IdentityKubernetesAuthTokenReviewMode { + Api = "api", + Gateway = "gateway" +} + export type TAttachKubernetesAuthDTO = { identityId: string; - kubernetesHost: string; + kubernetesHost: string | null; caCert: string; tokenReviewerJwt?: string; + tokenReviewMode: IdentityKubernetesAuthTokenReviewMode; allowedNamespaces: string; allowedNames: string; allowedAudience: string; @@ -23,9 +29,10 @@ export type TAttachKubernetesAuthDTO = { export type TUpdateKubernetesAuthDTO = { identityId: string; - kubernetesHost?: string; + kubernetesHost?: string | null; caCert?: string; tokenReviewerJwt?: string | null; + tokenReviewMode?: IdentityKubernetesAuthTokenReviewMode; allowedNamespaces?: string; allowedNames?: string; allowedAudience?: string; diff --git a/backend/src/services/org/org-dal.ts b/backend/src/services/org/org-dal.ts index c4f0856a1..5730c0d92 100644 --- a/backend/src/services/org/org-dal.ts +++ b/backend/src/services/org/org-dal.ts @@ -212,7 +212,7 @@ export const orgDALFactory = (db: TDbClient) => { // special query const findAllOrgsByUserId = async ( userId: string - ): Promise<(TOrganizations & { orgAuthMethod: string; userRole: string })[]> => { + ): Promise<(TOrganizations & { orgAuthMethod: string; userRole: string; userStatus: string })[]> => { try { const org = (await db .replicaNode()(TableName.OrgMembership) @@ -234,6 +234,7 @@ export const orgDALFactory = (db: TDbClient) => { }) .select(selectAllTableCols(TableName.Organization)) .select(db.ref("role").withSchema(TableName.OrgMembership).as("userRole")) + .select(db.ref("status").withSchema(TableName.OrgMembership).as("userStatus")) .select( db.raw(` CASE @@ -242,7 +243,7 @@ export const orgDALFactory = (db: TDbClient) => { ELSE '' END as "orgAuthMethod" `) - )) as (TOrganizations & { orgAuthMethod: string; userRole: string })[]; + )) as (TOrganizations & { orgAuthMethod: string; userRole: string; userStatus: string })[]; return org; } catch (error) { diff --git a/backend/src/services/org/org-service.ts b/backend/src/services/org/org-service.ts index bfd24e639..63cb8935d 100644 --- a/backend/src/services/org/org-service.ts +++ b/backend/src/services/org/org-service.ts @@ -183,7 +183,9 @@ export const orgServiceFactory = ({ * */ const findAllOrganizationOfUser = async (userId: string) => { const orgs = await orgDAL.findAllOrgsByUserId(userId); - return orgs; + + // Filter out orgs where the membership object is an invitation + return orgs.filter((org) => org.userStatus !== "invited"); }; /* * Get all workspace members @@ -835,16 +837,22 @@ export const orgServiceFactory = ({ // if the user doesn't exist we create the user with the email if (!inviteeUser) { - inviteeUser = await userDAL.create( - { - isAccepted: false, - email: inviteeEmail, - username: inviteeEmail, - authMethods: [AuthMethod.EMAIL], - isGhost: false - }, - tx - ); + // TODO(carlos): will be removed once the function receives usernames instead of emails + const usersByEmail = await userDAL.findUserByEmail(inviteeEmail, tx); + if (usersByEmail?.length === 1) { + [inviteeUser] = usersByEmail; + } else { + inviteeUser = await userDAL.create( + { + isAccepted: false, + email: inviteeEmail, + username: inviteeEmail, + authMethods: [AuthMethod.EMAIL], + isGhost: false + }, + tx + ); + } } const inviteeUserId = inviteeUser?.id; diff --git a/backend/src/services/project/project-dal.ts b/backend/src/services/project/project-dal.ts index 54bef02d1..7f733503c 100644 --- a/backend/src/services/project/project-dal.ts +++ b/backend/src/services/project/project-dal.ts @@ -12,7 +12,7 @@ import { TProjectsUpdate } from "@app/db/schemas"; import { BadRequestError, DatabaseError, NotFoundError, UnauthorizedError } from "@app/lib/errors"; -import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; +import { buildFindFilter, ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; import { ActorType } from "../auth/auth-type"; import { Filter, ProjectFilterType, SearchProjectSortBy } from "./project-types"; @@ -475,6 +475,16 @@ export const projectDALFactory = (db: TDbClient) => { return { docs, totalCount: Number(docs?.[0]?.count ?? 0) }; }; + const findProjectByEnvId = async (envId: string, tx?: Knex) => { + const project = await (tx || db.replicaNode())(TableName.Project) + .leftJoin(TableName.Environment, `${TableName.Environment}.projectId`, `${TableName.Project}.id`) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ id: envId }, TableName.Environment)) + .select(selectAllTableCols(TableName.Project)) + .first(); + return project; + }; + const countOfOrgProjects = async (orgId: string | null, tx?: Knex) => { try { const doc = await (tx || db.replicaNode())(TableName.Project) @@ -504,6 +514,7 @@ export const projectDALFactory = (db: TDbClient) => { checkProjectUpgradeStatus, getProjectFromSplitId, searchProjects, + findProjectByEnvId, countOfOrgProjects }; }; diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index 6b132f654..680b0187c 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -30,7 +30,7 @@ import { TSshCertificateDALFactory } from "@app/ee/services/ssh-certificate/ssh- import { TSshCertificateTemplateDALFactory } from "@app/ee/services/ssh-certificate-template/ssh-certificate-template-dal"; import { TSshHostDALFactory } from "@app/ee/services/ssh-host/ssh-host-dal"; import { TSshHostGroupDALFactory } from "@app/ee/services/ssh-host-group/ssh-host-group-dal"; -import { TKeyStoreFactory } from "@app/keystore/keystore"; +import { PgSqlLock, TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig } from "@app/lib/config/env"; import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/errors"; @@ -165,7 +165,7 @@ type TProjectServiceFactoryDep = { sshHostGroupDAL: Pick; permissionService: TPermissionServiceFactory; orgService: Pick; - licenseService: Pick; + licenseService: Pick; queueService: Pick; smtpService: Pick; orgDAL: Pick; @@ -259,16 +259,17 @@ export const projectServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.Workspace); - const plan = await licenseService.getPlan(organization.id); - if (plan.workspaceLimit !== null && plan.workspacesUsed >= plan.workspaceLimit) { - // case: limit imposed on number of workspaces allowed - // case: number of workspaces used exceeds the number of workspaces allowed - throw new BadRequestError({ - message: "Failed to create workspace due to plan limit reached. Upgrade plan to add more workspaces." - }); - } - const results = await (trx || projectDAL).transaction(async (tx) => { + await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.CreateProject(organization.id)]); + + const plan = await licenseService.getPlan(organization.id); + if (plan.workspaceLimit !== null && plan.workspacesUsed >= plan.workspaceLimit) { + // case: limit imposed on number of workspaces allowed + // case: number of workspaces used exceeds the number of workspaces allowed + throw new BadRequestError({ + message: "Failed to create workspace due to plan limit reached. Upgrade plan to add more workspaces." + }); + } const ghostUser = await orgService.addGhostUser(organization.id, tx); if (kmsKeyId) { @@ -493,6 +494,10 @@ export const projectServiceFactory = ({ ); } + // no need to invalidate if there was no limit + if (plan.workspaceLimit) { + await licenseService.invalidateGetPlan(organization.id); + } return { ...project, environments: envs, @@ -666,7 +671,8 @@ export const projectServiceFactory = ({ enforceCapitalization: update.autoCapitalization, hasDeleteProtection: update.hasDeleteProtection, slug: update.slug, - secretSharing: update.secretSharing + secretSharing: update.secretSharing, + showSnapshotsLegacy: update.showSnapshotsLegacy }); return updatedProject; diff --git a/backend/src/services/project/project-types.ts b/backend/src/services/project/project-types.ts index be052f1cb..8ef72492a 100644 --- a/backend/src/services/project/project-types.ts +++ b/backend/src/services/project/project-types.ts @@ -94,6 +94,7 @@ export type TUpdateProjectDTO = { hasDeleteProtection?: boolean; slug?: string; secretSharing?: boolean; + showSnapshotsLegacy?: boolean; }; } & Omit; diff --git a/backend/src/services/secret-folder/secret-folder-dal.ts b/backend/src/services/secret-folder/secret-folder-dal.ts index e136c5a50..7dfeaddcf 100644 --- a/backend/src/services/secret-folder/secret-folder-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-dal.ts @@ -488,6 +488,75 @@ export const secretFolderDALFactory = (db: TDbClient) => { } }; + const findFoldersByRootAndIds = async ({ rootId, folderIds }: { rootId: string; folderIds: string[] }, tx?: Knex) => { + try { + // First, get all descendant folders of rootId + const descendants = await (tx || db.replicaNode()) + .withRecursive("descendants", (qb) => + qb + .select( + selectAllTableCols(TableName.SecretFolder), + db.raw("0 as depth"), + db.raw(`'/' as path`), + db.ref(`${TableName.Environment}.slug`).as("environment") + ) + .from(TableName.SecretFolder) + .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .where(`${TableName.SecretFolder}.id`, rootId) + .union((un) => { + void un + .select( + selectAllTableCols(TableName.SecretFolder), + db.raw("descendants.depth + 1 as depth"), + db.raw( + `CONCAT( + CASE WHEN descendants.path = '/' THEN '' ELSE descendants.path END, + CASE WHEN ${TableName.SecretFolder}."parentId" is NULL THEN '' ELSE CONCAT('/', secret_folders.name) END + )` + ), + db.ref("descendants.environment") + ) + .from(TableName.SecretFolder) + .where(`${TableName.SecretFolder}.isReserved`, false) + .join("descendants", `${TableName.SecretFolder}.parentId`, "descendants.id"); + }) + ) + .select<(TSecretFolders & { path: string; depth: number; environment: string })[]>("*") + .from("descendants") + .whereIn(`id`, folderIds) + .orderBy("depth") + .orderBy(`name`); + + return descendants; + } catch (error) { + throw new DatabaseError({ error, name: "FindFoldersByRootAndIds" }); + } + }; + + const findByParentId = async (parentId: string, tx?: Knex) => { + try { + const folders = await (tx || db.replicaNode())(TableName.SecretFolder) + .where({ parentId }) + .andWhere({ isReserved: false }) + .select(selectAllTableCols(TableName.SecretFolder)); + return folders; + } catch (error) { + throw new DatabaseError({ error, name: "findByParentId" }); + } + }; + + const findByEnvId = async (envId: string, tx?: Knex) => { + try { + const folders = await (tx || db.replicaNode())(TableName.SecretFolder) + .where({ envId }) + .andWhere({ isReserved: false }) + .select(selectAllTableCols(TableName.SecretFolder)); + return folders; + } catch (error) { + throw new DatabaseError({ error, name: "findByEnvId" }); + } + }; + return { ...secretFolderOrm, update, @@ -499,6 +568,9 @@ export const secretFolderDALFactory = (db: TDbClient) => { findClosestFolder, findByProjectId, findByMultiEnv, - findByEnvsDeep + findByEnvsDeep, + findByParentId, + findByEnvId, + findFoldersByRootAndIds }; }; diff --git a/backend/src/services/secret-folder/secret-folder-service.ts b/backend/src/services/secret-folder/secret-folder-service.ts index 842eb2bb7..5722734f8 100644 --- a/backend/src/services/secret-folder/secret-folder-service.ts +++ b/backend/src/services/secret-folder/secret-folder-service.ts @@ -10,6 +10,7 @@ import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { OrderByDirection, OrgServiceActor } from "@app/lib/types"; import { buildFolderPath } from "@app/services/secret-folder/secret-folder-fns"; +import { ChangeType, CommitType, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretFolderDALFactory } from "./secret-folder-dal"; @@ -29,7 +30,8 @@ type TSecretFolderServiceFactoryDep = { snapshotService: Pick; folderDAL: TSecretFolderDALFactory; projectEnvDAL: Pick; - folderVersionDAL: TSecretFolderVersionDALFactory; + folderVersionDAL: Pick; + folderCommitService: Pick; projectDAL: Pick; }; @@ -41,6 +43,7 @@ export const secretFolderServiceFactory = ({ permissionService, projectEnvDAL, folderVersionDAL, + folderCommitService, projectDAL }: TSecretFolderServiceFactoryDep) => { const createFolder = async ({ @@ -111,15 +114,33 @@ export const secretFolderServiceFactory = ({ }); parentFolderId = newFolders.at(-1)?.id as string; const docs = await folderDAL.insertMany(newFolders, tx); - await folderVersionDAL.insertMany( + const folderVersions = await folderVersionDAL.insertMany( docs.map((doc) => ({ name: doc.name, envId: doc.envId, version: doc.version, - folderId: doc.id + folderId: doc.id, + description: doc.description })), tx ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder created", + folderId: parentFolderId, + changes: folderVersions.map((fv) => ({ + type: CommitType.ADD, + folderVersionId: fv.id + })) + }, + tx + ); } } @@ -127,12 +148,32 @@ export const secretFolderServiceFactory = ({ { name, envId: env.id, version: 1, parentId: parentFolderId, description }, tx ); - await folderVersionDAL.create( + const folderVersion = await folderVersionDAL.create( { name: doc.name, envId: doc.envId, version: doc.version, - folderId: doc.id + folderId: doc.id, + description: doc.description + }, + tx + ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder created", + folderId: parentFolderId, + changes: [ + { + type: CommitType.ADD, + folderVersionId: folderVersion.id + } + ] }, tx ); @@ -225,12 +266,33 @@ export const secretFolderServiceFactory = ({ { name, description }, tx ); - await folderVersionDAL.create( + const folderVersion = await folderVersionDAL.create( { name: doc.name, envId: doc.envId, version: doc.version, - folderId: doc.id + folderId: doc.id, + description: doc.description + }, + tx + ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder updated", + folderId: parentFolder.id, + changes: [ + { + type: CommitType.ADD, + isUpdate: true, + folderVersionId: folderVersion.id + } + ] }, tx ); @@ -321,12 +383,33 @@ export const secretFolderServiceFactory = ({ { name, description }, tx ); - await folderVersionDAL.create( + const folderVersion = await folderVersionDAL.create( { name: doc.name, envId: doc.envId, version: doc.version, - folderId: doc.id + folderId: doc.id, + description: doc.description + }, + tx + ); + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder updated", + folderId: parentFolder.id, + changes: [ + { + type: CommitType.ADD, + isUpdate: true, + folderVersionId: folderVersion.id + } + ] }, tx ); @@ -381,7 +464,31 @@ export const secretFolderServiceFactory = ({ }, tx ); + if (!doc) throw new NotFoundError({ message: `Failed to delete folder with ID '${idOrName}', not found` }); + + const folderVersions = await folderVersionDAL.findLatestFolderVersions([doc.id], tx); + + await folderCommitService.createCommit( + { + actor: { + type: actor, + metadata: { + id: actorId + } + }, + message: "Folder deleted", + folderId: parentFolder.id, + changes: [ + { + type: CommitType.DELETE, + folderVersionId: folderVersions[doc.id].id, + folderId: doc.id + } + ] + }, + tx + ); return doc; }); @@ -665,6 +772,45 @@ export const secretFolderServiceFactory = ({ return environmentFolders; }; + const getFolderVersionsByIds = async ({ + folderId, + folderVersions + }: { + folderId: string; + folderVersions: string[]; + }) => { + const versions = await folderVersionDAL.find({ + folderId, + $in: { + version: folderVersions.map((v) => Number.parseInt(v, 10)) + } + }); + return versions; + }; + + const getFolderVersions = async ( + change: { + folderVersion?: string; + isUpdate?: boolean; + changeType?: string; + }, + fromVersion: string, + folderId: string + ) => { + const currentVersion = change.folderVersion || "1"; + // eslint-disable-next-line no-await-in-loop + const versions = await getFolderVersionsByIds({ + folderId, + folderVersions: + change.isUpdate || change.changeType === ChangeType.UPDATE ? [currentVersion, fromVersion] : [currentVersion] + }); + return versions.map((v) => ({ + version: v.version?.toString() || "1", + name: v.name, + description: v.description + })); + }; + return { createFolder, updateFolder, @@ -675,6 +821,8 @@ export const secretFolderServiceFactory = ({ getProjectFolderCount, getFoldersMultiEnv, getFoldersDeepByEnvs, - getProjectEnvironmentsFolders + getProjectEnvironmentsFolders, + getFolderVersionsByIds, + getFolderVersions }; }; diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index 78186333d..46ff49692 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -43,7 +43,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { const docs: Array = await (tx || db.replicaNode())( TableName.SecretFolderVersion ) - .whereIn("folderId", folderIds) + .whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds) .join( (tx || db)(TableName.SecretFolderVersion) .groupBy("folderId") @@ -85,6 +85,8 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) .join("folder_cte", "folder_cte.id", `${TableName.SecretFolderVersion}.id`) .whereRaw(`folder_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + // Projects with version >= 3 will require to have all folder versions for PIT + .andWhere(`${TableName.Project}.version`, "<", 3) .delete(); } catch (error) { throw new DatabaseError({ @@ -95,5 +97,107 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret folder versions completed`); }; - return { ...secretFolderVerOrm, findLatestFolderVersions, findLatestVersionByFolderId, pruneExcessVersions }; + // Get latest versions by folderIds + const getLatestFolderVersions = async (folderIds: string[], tx?: Knex): Promise> => { + if (!folderIds.length) return []; + + const knexInstance = tx || db.replicaNode(); + return knexInstance(TableName.SecretFolderVersion) + .whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds) + .join( + knexInstance(TableName.SecretFolderVersion) + .groupBy("folderId") + .max("version") + .select("folderId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretFolderVersion}.folderId`, "latestVersion.folderId").andOn( + `${TableName.SecretFolderVersion}.version`, + "latestVersion.max" + ); + } + ); + }; + + // Get specific versions and update with max version + const getSpecificFolderVersionsWithLatest = async ( + versionIds: string[], + tx?: Knex + ): Promise> => { + if (!versionIds.length) return []; + + const knexInstance = tx || db.replicaNode(); + + // Get specific versions + const specificVersions = await knexInstance(TableName.SecretFolderVersion).whereIn("id", versionIds); + + // Get folderIds from these versions + const specificFolderIds = [...new Set(specificVersions.map((v) => v.folderId).filter(Boolean))]; + + if (!specificFolderIds.length) return specificVersions; + + // Get max versions for these folderIds + const maxVersionsQuery = await knexInstance(TableName.SecretFolderVersion) + .whereIn("folderId", specificFolderIds) + .groupBy("folderId") + .select("folderId") + .max("version", { as: "maxVersion" }); + + // Create lookup map for max versions + const maxVersionMap = maxVersionsQuery.reduce>((acc, item) => { + if (item.maxVersion) { + acc[item.folderId] = item.maxVersion; + } + return acc; + }, {}); + + // Replace version with max version + return specificVersions.map((version) => ({ + ...version, + version: maxVersionMap[version.folderId] || version.version + })); + }; + + const findByIdsWithLatestVersion = async (folderIds: string[], versionIds?: string[], tx?: Knex) => { + try { + if (!folderIds.length && (!versionIds || !versionIds.length)) return {}; + + // Run both queries in parallel + const [latestVersions, specificVersionsWithLatest] = await Promise.all([ + folderIds.length ? getLatestFolderVersions(folderIds, tx) : [], + versionIds?.length ? getSpecificFolderVersionsWithLatest(versionIds, tx) : [] + ]); + + const allDocs = [...latestVersions, ...specificVersionsWithLatest]; + + // Convert array to record with folderId as key + return allDocs.reduce>( + (prev, curr) => ({ ...prev, [curr.folderId || ""]: curr }), + {} + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindByIdsWithLatestVersion" }); + } + }; + + const findLatestVersion = async (folderId: string, tx?: Knex) => { + try { + const doc = await (tx || db.replicaNode())(TableName.SecretFolderVersion) + .where(`${TableName.SecretFolderVersion}.folderId`, folderId) + .select(selectAllTableCols(TableName.SecretFolderVersion)) + .first(); + return doc; + } catch (error) { + throw new DatabaseError({ error, name: "findLatestVersion" }); + } + }; + + return { + ...secretFolderVerOrm, + findLatestFolderVersions, + findLatestVersionByFolderId, + pruneExcessVersions, + findByIdsWithLatestVersion, + findLatestVersion + }; }; diff --git a/backend/src/services/secret-sync/1password/1password-sync-fns.ts b/backend/src/services/secret-sync/1password/1password-sync-fns.ts index c832fbbdb..9305f2e3c 100644 --- a/backend/src/services/secret-sync/1password/1password-sync-fns.ts +++ b/backend/src/services/secret-sync/1password/1password-sync-fns.ts @@ -127,6 +127,7 @@ export const OnePassSyncFns = { syncSecrets: async (secretSync: TOnePassSyncWithCredentials, secretMap: TSecretMap) => { const { connection, + environment, destinationConfig: { vaultId } } = secretSync; @@ -164,7 +165,7 @@ export const OnePassSyncFns = { for await (const [key, variable] of Object.entries(items)) { // eslint-disable-next-line no-continue - if (!matchesSchema(key, secretSync.syncOptions.keySchema)) continue; + if (!matchesSchema(key, environment?.slug || "", secretSync.syncOptions.keySchema)) continue; if (!(key in secretMap)) { try { diff --git a/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts index a73bc81c9..b687d81dd 100644 --- a/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts +++ b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts @@ -294,7 +294,7 @@ const deleteParametersBatch = async ( export const AwsParameterStoreSyncFns = { syncSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials, secretMap: TSecretMap) => { - const { destinationConfig, syncOptions } = secretSync; + const { destinationConfig, syncOptions, environment } = secretSync; const ssm = await getSSM(secretSync); @@ -391,7 +391,7 @@ export const AwsParameterStoreSyncFns = { const [key, parameter] = entry; // eslint-disable-next-line no-continue - if (!matchesSchema(key, syncOptions.keySchema)) continue; + if (!matchesSchema(key, environment?.slug || "", syncOptions.keySchema)) continue; if (!(key in secretMap) || !secretMap[key].value) { parametersToDelete.push(parameter); diff --git a/backend/src/services/secret-sync/aws-secrets-manager/aws-secrets-manager-sync-fns.ts b/backend/src/services/secret-sync/aws-secrets-manager/aws-secrets-manager-sync-fns.ts index 1b1daf2ac..df73512e5 100644 --- a/backend/src/services/secret-sync/aws-secrets-manager/aws-secrets-manager-sync-fns.ts +++ b/backend/src/services/secret-sync/aws-secrets-manager/aws-secrets-manager-sync-fns.ts @@ -57,7 +57,11 @@ const sleep = async () => setTimeout(resolve, 1000); }); -const getSecretsRecord = async (client: SecretsManagerClient, keySchema?: string): Promise => { +const getSecretsRecord = async ( + client: SecretsManagerClient, + environment: string, + keySchema?: string +): Promise => { const awsSecretsRecord: TAwsSecretsRecord = {}; let hasNext = true; let nextToken: string | undefined; @@ -72,7 +76,7 @@ const getSecretsRecord = async (client: SecretsManagerClient, keySchema?: string if (output.SecretList) { output.SecretList.forEach((secretEntry) => { - if (secretEntry.Name && matchesSchema(secretEntry.Name, keySchema)) { + if (secretEntry.Name && matchesSchema(secretEntry.Name, environment, keySchema)) { awsSecretsRecord[secretEntry.Name] = secretEntry; } }); @@ -307,11 +311,11 @@ const processTags = ({ export const AwsSecretsManagerSyncFns = { syncSecrets: async (secretSync: TAwsSecretsManagerSyncWithCredentials, secretMap: TSecretMap) => { - const { destinationConfig, syncOptions } = secretSync; + const { destinationConfig, syncOptions, environment } = secretSync; const client = await getSecretsManagerClient(secretSync); - const awsSecretsRecord = await getSecretsRecord(client, syncOptions.keySchema); + const awsSecretsRecord = await getSecretsRecord(client, environment?.slug || "", syncOptions.keySchema); const awsValuesRecord = await getSecretValuesRecord(client, awsSecretsRecord); @@ -401,7 +405,7 @@ export const AwsSecretsManagerSyncFns = { for await (const secretKey of Object.keys(awsSecretsRecord)) { // eslint-disable-next-line no-continue - if (!matchesSchema(secretKey, syncOptions.keySchema)) continue; + if (!matchesSchema(secretKey, environment?.slug || "", syncOptions.keySchema)) continue; if (!(secretKey in secretMap) || !secretMap[secretKey].value) { try { @@ -468,7 +472,11 @@ export const AwsSecretsManagerSyncFns = { getSecrets: async (secretSync: TAwsSecretsManagerSyncWithCredentials): Promise => { const client = await getSecretsManagerClient(secretSync); - const awsSecretsRecord = await getSecretsRecord(client, secretSync.syncOptions.keySchema); + const awsSecretsRecord = await getSecretsRecord( + client, + secretSync.environment?.slug || "", + secretSync.syncOptions.keySchema + ); const awsValuesRecord = await getSecretValuesRecord(client, awsSecretsRecord); const { destinationConfig } = secretSync; @@ -503,11 +511,11 @@ export const AwsSecretsManagerSyncFns = { } }, removeSecrets: async (secretSync: TAwsSecretsManagerSyncWithCredentials, secretMap: TSecretMap) => { - const { destinationConfig, syncOptions } = secretSync; + const { destinationConfig, syncOptions, environment } = secretSync; const client = await getSecretsManagerClient(secretSync); - const awsSecretsRecord = await getSecretsRecord(client, syncOptions.keySchema); + const awsSecretsRecord = await getSecretsRecord(client, environment?.slug || "", syncOptions.keySchema); if (destinationConfig.mappingBehavior === AwsSecretsManagerSyncMappingBehavior.OneToOne) { for await (const secretKey of Object.keys(awsSecretsRecord)) { diff --git a/backend/src/services/secret-sync/azure-app-configuration/azure-app-configuration-sync-fns.ts b/backend/src/services/secret-sync/azure-app-configuration/azure-app-configuration-sync-fns.ts index dce509fac..7aa1c16ce 100644 --- a/backend/src/services/secret-sync/azure-app-configuration/azure-app-configuration-sync-fns.ts +++ b/backend/src/services/secret-sync/azure-app-configuration/azure-app-configuration-sync-fns.ts @@ -141,7 +141,7 @@ export const azureAppConfigurationSyncFactory = ({ for await (const key of Object.keys(azureAppConfigSecrets)) { // eslint-disable-next-line no-continue - if (!matchesSchema(key, secretSync.syncOptions.keySchema)) continue; + if (!matchesSchema(key, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema)) continue; const azureSecret = azureAppConfigSecrets[key]; if ( diff --git a/backend/src/services/secret-sync/azure-key-vault/azure-key-vault-sync-fns.ts b/backend/src/services/secret-sync/azure-key-vault/azure-key-vault-sync-fns.ts index fd1e2bd78..edc8af709 100644 --- a/backend/src/services/secret-sync/azure-key-vault/azure-key-vault-sync-fns.ts +++ b/backend/src/services/secret-sync/azure-key-vault/azure-key-vault-sync-fns.ts @@ -194,7 +194,7 @@ export const azureKeyVaultSyncFactory = ({ kmsService, appConnectionDAL }: TAzur for await (const deleteSecretKey of deleteSecrets.filter( (secret) => - matchesSchema(secret, secretSync.syncOptions.keySchema) && + matchesSchema(secret, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema) && !setSecrets.find((setSecret) => setSecret.key === secret) )) { await request.delete(`${secretSync.destinationConfig.vaultBaseUrl}/secrets/${deleteSecretKey}?api-version=7.3`, { diff --git a/backend/src/services/secret-sync/camunda/camunda-sync-fns.ts b/backend/src/services/secret-sync/camunda/camunda-sync-fns.ts index 256ae4644..516efae10 100644 --- a/backend/src/services/secret-sync/camunda/camunda-sync-fns.ts +++ b/backend/src/services/secret-sync/camunda/camunda-sync-fns.ts @@ -118,7 +118,7 @@ export const camundaSyncFactory = ({ kmsService, appConnectionDAL }: TCamundaSec for await (const secret of Object.keys(camundaSecrets)) { // eslint-disable-next-line no-continue - if (!matchesSchema(secret, secretSync.syncOptions.keySchema)) continue; + if (!matchesSchema(secret, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema)) continue; if (!(secret in secretMap) || !secretMap[secret].value) { try { diff --git a/backend/src/services/secret-sync/databricks/databricks-sync-fns.ts b/backend/src/services/secret-sync/databricks/databricks-sync-fns.ts index 11143e24d..175901323 100644 --- a/backend/src/services/secret-sync/databricks/databricks-sync-fns.ts +++ b/backend/src/services/secret-sync/databricks/databricks-sync-fns.ts @@ -117,7 +117,7 @@ export const databricksSyncFactory = ({ kmsService, appConnectionDAL }: TDatabri for await (const secret of databricksSecretKeys) { // eslint-disable-next-line no-continue - if (!matchesSchema(secret.key, secretSync.syncOptions.keySchema)) continue; + if (!matchesSchema(secret.key, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema)) continue; if (!(secret.key in secretMap)) { await deleteDatabricksSecrets({ diff --git a/backend/src/services/secret-sync/gcp/gcp-sync-enums.ts b/backend/src/services/secret-sync/gcp/gcp-sync-enums.ts index 348d2bfa5..389070a9a 100644 --- a/backend/src/services/secret-sync/gcp/gcp-sync-enums.ts +++ b/backend/src/services/secret-sync/gcp/gcp-sync-enums.ts @@ -1,3 +1,63 @@ export enum GcpSyncScope { - Global = "global" + Global = "global", + Region = "region" +} + +export enum GCPSecretManagerLocation { + // Asia Pacific + ASIA_SOUTHEAST3 = "asia-southeast3", // Bangkok + ASIA_SOUTH2 = "asia-south2", // Delhi + ASIA_EAST2 = "asia-east2", // Hong Kong + ASIA_SOUTHEAST2 = "asia-southeast2", // Jakarta + AUSTRALIA_SOUTHEAST2 = "australia-southeast2", // Melbourne + ASIA_SOUTH1 = "asia-south1", // Mumbai + ASIA_NORTHEAST2 = "asia-northeast2", // Osaka + ASIA_NORTHEAST3 = "asia-northeast3", // Seoul + ASIA_SOUTHEAST1 = "asia-southeast1", // Singapore + AUSTRALIA_SOUTHEAST1 = "australia-southeast1", // Sydney + ASIA_EAST1 = "asia-east1", // Taiwan + ASIA_NORTHEAST1 = "asia-northeast1", // Tokyo + + // Europe + EUROPE_WEST1 = "europe-west1", // Belgium + EUROPE_WEST10 = "europe-west10", // Berlin + EUROPE_NORTH1 = "europe-north1", // Finland + EUROPE_NORTH2 = "europe-north2", // Stockholm + EUROPE_WEST3 = "europe-west3", // Frankfurt + EUROPE_WEST2 = "europe-west2", // London + EUROPE_SOUTHWEST1 = "europe-southwest1", // Madrid + EUROPE_WEST8 = "europe-west8", // Milan + EUROPE_WEST4 = "europe-west4", // Netherlands + EUROPE_WEST12 = "europe-west12", // Turin + EUROPE_WEST9 = "europe-west9", // Paris + EUROPE_CENTRAL2 = "europe-central2", // Warsaw + EUROPE_WEST6 = "europe-west6", // Zurich + + // North America + US_CENTRAL1 = "us-central1", // Iowa + US_WEST4 = "us-west4", // Las Vegas + US_WEST2 = "us-west2", // Los Angeles + NORTHAMERICA_SOUTH1 = "northamerica-south1", // Mexico + NORTHAMERICA_NORTHEAST1 = "northamerica-northeast1", // Montréal + US_EAST4 = "us-east4", // Northern Virginia + US_CENTRAL2 = "us-central2", // Oklahoma + US_WEST1 = "us-west1", // Oregon + US_WEST3 = "us-west3", // Salt Lake City + US_EAST1 = "us-east1", // South Carolina + NORTHAMERICA_NORTHEAST2 = "northamerica-northeast2", // Toronto + US_EAST5 = "us-east5", // Columbus + US_SOUTH1 = "us-south1", // Dallas + US_WEST8 = "us-west8", // Phoenix + + // South America + SOUTHAMERICA_EAST1 = "southamerica-east1", // São Paulo + SOUTHAMERICA_WEST1 = "southamerica-west1", // Santiago + + // Middle East + ME_CENTRAL2 = "me-central2", // Dammam + ME_CENTRAL1 = "me-central1", // Doha + ME_WEST1 = "me-west1", // Tel Aviv + + // Africa + AFRICA_SOUTH1 = "africa-south1" // Johannesburg } diff --git a/backend/src/services/secret-sync/gcp/gcp-sync-fns.ts b/backend/src/services/secret-sync/gcp/gcp-sync-fns.ts index 97da66a48..d51383fef 100644 --- a/backend/src/services/secret-sync/gcp/gcp-sync-fns.ts +++ b/backend/src/services/secret-sync/gcp/gcp-sync-fns.ts @@ -4,6 +4,7 @@ import { request } from "@app/lib/config/request"; import { logger } from "@app/lib/logger"; import { getGcpConnectionAuthToken } from "@app/services/app-connection/gcp"; import { IntegrationUrls } from "@app/services/integration-auth/integration-list"; +import { GcpSyncScope } from "@app/services/secret-sync/gcp/gcp-sync-enums"; import { matchesSchema } from "@app/services/secret-sync/secret-sync-fns"; import { SecretSyncError } from "../secret-sync-errors"; @@ -15,9 +16,17 @@ import { TGcpSyncWithCredentials } from "./gcp-sync-types"; -const getGcpSecrets = async (accessToken: string, secretSync: TGcpSyncWithCredentials) => { +const getProjectUrl = (secretSync: TGcpSyncWithCredentials) => { const { destinationConfig } = secretSync; + if (destinationConfig.scope === GcpSyncScope.Global) { + return `${IntegrationUrls.GCP_SECRET_MANAGER_URL}/v1/projects/${destinationConfig.projectId}`; + } + + return `https://secretmanager.${destinationConfig.locationId}.rep.googleapis.com/v1/projects/${destinationConfig.projectId}/locations/${destinationConfig.locationId}`; +}; + +const getGcpSecrets = async (accessToken: string, secretSync: TGcpSyncWithCredentials) => { let gcpSecrets: GCPSecret[] = []; const pageSize = 100; @@ -31,16 +40,13 @@ const getGcpSecrets = async (accessToken: string, secretSync: TGcpSyncWithCreden }); // eslint-disable-next-line no-await-in-loop - const { data: secretsRes } = await request.get( - `${IntegrationUrls.GCP_SECRET_MANAGER_URL}/v1/projects/${secretSync.destinationConfig.projectId}/secrets`, - { - params, - headers: { - Authorization: `Bearer ${accessToken}`, - "Accept-Encoding": "application/json" - } + const { data: secretsRes } = await request.get(`${getProjectUrl(secretSync)}/secrets`, { + params, + headers: { + Authorization: `Bearer ${accessToken}`, + "Accept-Encoding": "application/json" } - ); + }); if (secretsRes.secrets) { gcpSecrets = gcpSecrets.concat(secretsRes.secrets); @@ -61,7 +67,7 @@ const getGcpSecrets = async (accessToken: string, secretSync: TGcpSyncWithCreden try { const { data: secretLatest } = await request.get( - `${IntegrationUrls.GCP_SECRET_MANAGER_URL}/v1/projects/${destinationConfig.projectId}/secrets/${key}/versions/latest:access`, + `${getProjectUrl(secretSync)}/secrets/${key}/versions/latest:access`, { headers: { Authorization: `Bearer ${accessToken}`, @@ -113,11 +119,14 @@ export const GcpSyncFns = { if (!(key in gcpSecrets)) { // case: create secret await request.post( - `${IntegrationUrls.GCP_SECRET_MANAGER_URL}/v1/projects/${destinationConfig.projectId}/secrets`, + `${getProjectUrl(secretSync)}/secrets`, { - replication: { - automatic: {} - } + replication: + destinationConfig.scope === GcpSyncScope.Global + ? { + automatic: {} + } + : undefined }, { params: { @@ -131,7 +140,7 @@ export const GcpSyncFns = { ); await request.post( - `${IntegrationUrls.GCP_SECRET_MANAGER_URL}/v1/projects/${destinationConfig.projectId}/secrets/${key}:addVersion`, + `${getProjectUrl(secretSync)}/secrets/${key}:addVersion`, { payload: { data: Buffer.from(secretMap[key].value).toString("base64") @@ -155,7 +164,7 @@ export const GcpSyncFns = { for await (const key of Object.keys(gcpSecrets)) { // eslint-disable-next-line no-continue - if (!matchesSchema(key, secretSync.syncOptions.keySchema)) continue; + if (!matchesSchema(key, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema)) continue; try { if (!(key in secretMap) || !secretMap[key].value) { @@ -163,15 +172,12 @@ export const GcpSyncFns = { if (secretSync.syncOptions.disableSecretDeletion) continue; // case: delete secret - await request.delete( - `${IntegrationUrls.GCP_SECRET_MANAGER_URL}/v1/projects/${destinationConfig.projectId}/secrets/${key}`, - { - headers: { - Authorization: `Bearer ${accessToken}`, - "Accept-Encoding": "application/json" - } + await request.delete(`${getProjectUrl(secretSync)}/secrets/${key}`, { + headers: { + Authorization: `Bearer ${accessToken}`, + "Accept-Encoding": "application/json" } - ); + }); } else if (secretMap[key].value !== gcpSecrets[key]) { if (!secretMap[key].value) { logger.warn( @@ -180,7 +186,7 @@ export const GcpSyncFns = { } await request.post( - `${IntegrationUrls.GCP_SECRET_MANAGER_URL}/v1/projects/${destinationConfig.projectId}/secrets/${key}:addVersion`, + `${getProjectUrl(secretSync)}/secrets/${key}:addVersion`, { payload: { data: Buffer.from(secretMap[key].value).toString("base64") @@ -212,21 +218,18 @@ export const GcpSyncFns = { }, removeSecrets: async (secretSync: TGcpSyncWithCredentials, secretMap: TSecretMap) => { - const { destinationConfig, connection } = secretSync; + const { connection } = secretSync; const accessToken = await getGcpConnectionAuthToken(connection); const gcpSecrets = await getGcpSecrets(accessToken, secretSync); for await (const [key] of Object.entries(gcpSecrets)) { if (key in secretMap) { - await request.delete( - `${IntegrationUrls.GCP_SECRET_MANAGER_URL}/v1/projects/${destinationConfig.projectId}/secrets/${key}`, - { - headers: { - Authorization: `Bearer ${accessToken}`, - "Accept-Encoding": "application/json" - } + await request.delete(`${getProjectUrl(secretSync)}/secrets/${key}`, { + headers: { + Authorization: `Bearer ${accessToken}`, + "Accept-Encoding": "application/json" } - ); + }); } } } diff --git a/backend/src/services/secret-sync/gcp/gcp-sync-schemas.ts b/backend/src/services/secret-sync/gcp/gcp-sync-schemas.ts index 0643c431a..875ceaf70 100644 --- a/backend/src/services/secret-sync/gcp/gcp-sync-schemas.ts +++ b/backend/src/services/secret-sync/gcp/gcp-sync-schemas.ts @@ -10,14 +10,33 @@ import { import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types"; import { SecretSync } from "../secret-sync-enums"; -import { GcpSyncScope } from "./gcp-sync-enums"; +import { GCPSecretManagerLocation, GcpSyncScope } from "./gcp-sync-enums"; const GcpSyncOptionsConfig: TSyncOptionsConfig = { canImportSecrets: true }; -const GcpSyncDestinationConfigSchema = z.object({ - scope: z.literal(GcpSyncScope.Global).describe(SecretSyncs.DESTINATION_CONFIG.GCP.scope), - projectId: z.string().min(1, "Project ID is required").describe(SecretSyncs.DESTINATION_CONFIG.GCP.projectId) -}); +const GcpSyncDestinationConfigSchema = z.discriminatedUnion("scope", [ + z + .object({ + scope: z.literal(GcpSyncScope.Global).describe(SecretSyncs.DESTINATION_CONFIG.GCP.scope), + projectId: z.string().min(1, "Project ID is required").describe(SecretSyncs.DESTINATION_CONFIG.GCP.projectId) + }) + .describe( + JSON.stringify({ + title: "Global" + }) + ), + z + .object({ + scope: z.literal(GcpSyncScope.Region).describe(SecretSyncs.DESTINATION_CONFIG.GCP.scope), + projectId: z.string().min(1, "Project ID is required").describe(SecretSyncs.DESTINATION_CONFIG.GCP.projectId), + locationId: z.nativeEnum(GCPSecretManagerLocation).describe(SecretSyncs.DESTINATION_CONFIG.GCP.locationId) + }) + .describe( + JSON.stringify({ + title: "Region" + }) + ) +]); export const GcpSyncSchema = BaseSecretSyncSchema(SecretSync.GCPSecretManager, GcpSyncOptionsConfig).extend({ destination: z.literal(SecretSync.GCPSecretManager), diff --git a/backend/src/services/secret-sync/github/github-sync-fns.ts b/backend/src/services/secret-sync/github/github-sync-fns.ts index 952f4b512..f06f0cfc2 100644 --- a/backend/src/services/secret-sync/github/github-sync-fns.ts +++ b/backend/src/services/secret-sync/github/github-sync-fns.ts @@ -223,8 +223,9 @@ export const GithubSyncFns = { if (secretSync.syncOptions.disableSecretDeletion) return; for await (const encryptedSecret of encryptedSecrets) { - // eslint-disable-next-line no-continue - if (!matchesSchema(encryptedSecret.name, secretSync.syncOptions.keySchema)) continue; + if (!matchesSchema(encryptedSecret.name, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema)) + // eslint-disable-next-line no-continue + continue; if (!(encryptedSecret.name in secretMap)) { await deleteSecret(client, secretSync, encryptedSecret); diff --git a/backend/src/services/secret-sync/hc-vault/hc-vault-sync-fns.ts b/backend/src/services/secret-sync/hc-vault/hc-vault-sync-fns.ts index 6331cd91f..724eec7be 100644 --- a/backend/src/services/secret-sync/hc-vault/hc-vault-sync-fns.ts +++ b/backend/src/services/secret-sync/hc-vault/hc-vault-sync-fns.ts @@ -68,6 +68,7 @@ export const HCVaultSyncFns = { syncSecrets: async (secretSync: THCVaultSyncWithCredentials, secretMap: TSecretMap) => { const { connection, + environment, destinationConfig: { mount, path }, syncOptions: { disableSecretDeletion, keySchema } } = secretSync; @@ -97,7 +98,7 @@ export const HCVaultSyncFns = { for await (const [key] of Object.entries(variables)) { // eslint-disable-next-line no-continue - if (!matchesSchema(key, keySchema)) continue; + if (!matchesSchema(key, environment?.slug || "", keySchema)) continue; if (!(key in secretMap)) { delete variables[key]; diff --git a/backend/src/services/secret-sync/humanitec/humanitec-sync-fns.ts b/backend/src/services/secret-sync/humanitec/humanitec-sync-fns.ts index 2fcf488aa..ccb6ac2bc 100644 --- a/backend/src/services/secret-sync/humanitec/humanitec-sync-fns.ts +++ b/backend/src/services/secret-sync/humanitec/humanitec-sync-fns.ts @@ -200,8 +200,9 @@ export const HumanitecSyncFns = { if (secretSync.syncOptions.disableSecretDeletion) return; for await (const humanitecSecret of humanitecSecrets) { - // eslint-disable-next-line no-continue - if (!matchesSchema(humanitecSecret.key, secretSync.syncOptions.keySchema)) continue; + if (!matchesSchema(humanitecSecret.key, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema)) + // eslint-disable-next-line no-continue + continue; if (!secretMap[humanitecSecret.key]) { await deleteSecret(secretSync, humanitecSecret); diff --git a/backend/src/services/secret-sync/secret-sync-fns.ts b/backend/src/services/secret-sync/secret-sync-fns.ts index d7c653613..bcda32967 100644 --- a/backend/src/services/secret-sync/secret-sync-fns.ts +++ b/backend/src/services/secret-sync/secret-sync-fns.ts @@ -1,5 +1,5 @@ import { AxiosError } from "axios"; -import RE2 from "re2"; +import handlebars from "handlebars"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { OCI_VAULT_SYNC_LIST_OPTION, OCIVaultSyncFns } from "@app/ee/services/secret-sync/oci-vault"; @@ -70,13 +70,17 @@ type TSyncSecretDeps = { }; // Add schema to secret keys -const addSchema = (unprocessedSecretMap: TSecretMap, schema?: string): TSecretMap => { +const addSchema = (unprocessedSecretMap: TSecretMap, environment: string, schema?: string): TSecretMap => { if (!schema) return unprocessedSecretMap; const processedSecretMap: TSecretMap = {}; for (const [key, value] of Object.entries(unprocessedSecretMap)) { - const newKey = new RE2("{{secretKey}}").replace(schema, key); + const newKey = handlebars.compile(schema)({ + secretKey: key, + environment + }); + processedSecretMap[newKey] = value; } @@ -84,10 +88,17 @@ const addSchema = (unprocessedSecretMap: TSecretMap, schema?: string): TSecretMa }; // Strip schema from secret keys -const stripSchema = (unprocessedSecretMap: TSecretMap, schema?: string): TSecretMap => { +const stripSchema = (unprocessedSecretMap: TSecretMap, environment: string, schema?: string): TSecretMap => { if (!schema) return unprocessedSecretMap; - const [prefix, suffix] = schema.split("{{secretKey}}"); + const compiledSchemaPattern = handlebars.compile(schema)({ + secretKey: "{{secretKey}}", // Keep secretKey + environment + }); + + const parts = compiledSchemaPattern.split("{{secretKey}}"); + const prefix = parts[0]; + const suffix = parts[parts.length - 1]; const strippedMap: TSecretMap = {}; @@ -105,21 +116,40 @@ const stripSchema = (unprocessedSecretMap: TSecretMap, schema?: string): TSecret }; // Checks if a key matches a schema -export const matchesSchema = (key: string, schema?: string): boolean => { +export const matchesSchema = (key: string, environment: string, schema?: string): boolean => { if (!schema) return true; - const [prefix, suffix] = schema.split("{{secretKey}}"); - if (prefix === undefined || suffix === undefined) return true; + const compiledSchemaPattern = handlebars.compile(schema)({ + secretKey: "{{secretKey}}", // Keep secretKey + environment + }); - return key.startsWith(prefix) && key.endsWith(suffix); + // This edge-case shouldn't be possible + if (!compiledSchemaPattern.includes("{{secretKey}}")) { + return key === compiledSchemaPattern; + } + + const parts = compiledSchemaPattern.split("{{secretKey}}"); + const prefix = parts[0]; + const suffix = parts[parts.length - 1]; + + if (prefix === "" && suffix === "") return true; + + // If prefix is empty, key must end with suffix + if (prefix === "") return key.endsWith(suffix); + + // If suffix is empty, key must start with prefix + if (suffix === "") return key.startsWith(prefix); + + return key.startsWith(prefix) && key.endsWith(suffix) && key.length >= prefix.length + suffix.length; }; // Filter only for secrets with keys that match the schema -const filterForSchema = (secretMap: TSecretMap, schema?: string): TSecretMap => { +const filterForSchema = (secretMap: TSecretMap, environment: string, schema?: string): TSecretMap => { const filteredMap: TSecretMap = {}; for (const [key, value] of Object.entries(secretMap)) { - if (matchesSchema(key, schema)) { + if (matchesSchema(key, environment, schema)) { filteredMap[key] = value; } } @@ -133,7 +163,7 @@ export const SecretSyncFns = { secretMap: TSecretMap, { kmsService, appConnectionDAL }: TSyncSecretDeps ): Promise => { - const schemaSecretMap = addSchema(secretMap, secretSync.syncOptions.keySchema); + const schemaSecretMap = addSchema(secretMap, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema); switch (secretSync.destination) { case SecretSync.AWSParameterStore: @@ -268,14 +298,16 @@ export const SecretSyncFns = { ); } - return stripSchema(filterForSchema(secretMap), secretSync.syncOptions.keySchema); + const filtered = filterForSchema(secretMap, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema); + const stripped = stripSchema(filtered, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema); + return stripped; }, removeSecrets: ( secretSync: TSecretSyncWithCredentials, secretMap: TSecretMap, { kmsService, appConnectionDAL }: TSyncSecretDeps ): Promise => { - const schemaSecretMap = addSchema(secretMap, secretSync.syncOptions.keySchema); + const schemaSecretMap = addSchema(secretMap, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema); switch (secretSync.destination) { case SecretSync.AWSParameterStore: diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.ts index 6f627c24e..66e83661f 100644 --- a/backend/src/services/secret-sync/secret-sync-queue.ts +++ b/backend/src/services/secret-sync/secret-sync-queue.ts @@ -59,6 +59,7 @@ import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/se import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; import { TAppConnectionDALFactory } from "../app-connection/app-connection-dal"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; export type TSecretSyncQueueFactory = ReturnType; @@ -94,6 +95,7 @@ type TSecretSyncQueueFactoryDep = { secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; licenseService: Pick; }; @@ -136,6 +138,7 @@ export const secretSyncQueueFactory = ({ secretVersionV2BridgeDAL, secretVersionTagV2BridgeDAL, resourceMetadataDAL, + folderCommitService, licenseService }: TSecretSyncQueueFactoryDep) => { const appCfg = getConfig(); @@ -167,7 +170,8 @@ export const secretSyncQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const $updateManySecretsRawFn = updateManySecretsRawFnFactory({ @@ -183,7 +187,8 @@ export const secretSyncQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const $getInfisicalSecrets = async ( @@ -373,7 +378,7 @@ export const secretSyncQueueFactory = ({ if (Object.hasOwn(secretMap, key)) { // Only update secrets if the source value is not empty - if (value) { + if (value && value !== secretMap[key].value) { secretsToUpdate.push(secret); if (importBehavior === SecretSyncImportBehavior.PrioritizeDestination) importedSecretMap[key] = secretData; } diff --git a/backend/src/services/secret-sync/secret-sync-schemas.ts b/backend/src/services/secret-sync/secret-sync-schemas.ts index 80e96bf8b..3622ef3d0 100644 --- a/backend/src/services/secret-sync/secret-sync-schemas.ts +++ b/backend/src/services/secret-sync/secret-sync-schemas.ts @@ -28,10 +28,30 @@ const BaseSyncOptionsSchema = ({ keySchema: z .string() .optional() - .refine((val) => !val || new RE2(/^(?:[a-zA-Z0-9_\-/]*)(?:\{\{secretKey\}\})(?:[a-zA-Z0-9_\-/]*)$/).test(val), { - message: - "Key schema must include one {{secretKey}} and only contain letters, numbers, dashes, underscores, slashes, and the {{secretKey}} placeholder." - }) + .refine( + (val) => { + if (!val) return true; + + const allowedOptionalPlaceholders = ["{{environment}}"]; + + const allowedPlaceholdersRegexPart = ["{{secretKey}}", ...allowedOptionalPlaceholders] + .map((p) => p.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\$&")) // Escape regex special characters + .join("|"); + + const allowedContentRegex = new RE2(`^([a-zA-Z0-9_\\-/]|${allowedPlaceholdersRegexPart})*$`); + const contentIsValid = allowedContentRegex.test(val); + + // Check if {{secretKey}} is present + const secretKeyRegex = new RE2(/\{\{secretKey\}\}/); + const secretKeyIsPresent = secretKeyRegex.test(val); + + return contentIsValid && secretKeyIsPresent; + }, + { + message: + "Key schema must include exactly one {{secretKey}} placeholder. It can also include {{environment}} placeholders. Only alphanumeric characters (a-z, A-Z, 0-9), dashes (-), underscores (_), and slashes (/) are allowed besides the placeholders." + } + ) .describe(SecretSyncs.SYNC_OPTIONS(destination).keySchema), disableSecretDeletion: z.boolean().optional().describe(SecretSyncs.SYNC_OPTIONS(destination).disableSecretDeletion) }); diff --git a/backend/src/services/secret-sync/teamcity/teamcity-sync-fns.ts b/backend/src/services/secret-sync/teamcity/teamcity-sync-fns.ts index 0afe29beb..28ef2d0d3 100644 --- a/backend/src/services/secret-sync/teamcity/teamcity-sync-fns.ts +++ b/backend/src/services/secret-sync/teamcity/teamcity-sync-fns.ts @@ -127,7 +127,7 @@ export const TeamCitySyncFns = { for await (const [key, variable] of Object.entries(variables)) { // eslint-disable-next-line no-continue - if (!matchesSchema(key, secretSync.syncOptions.keySchema)) continue; + if (!matchesSchema(key, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema)) continue; if (!(key in secretMap)) { try { diff --git a/backend/src/services/secret-sync/terraform-cloud/terraform-cloud-sync-fns.ts b/backend/src/services/secret-sync/terraform-cloud/terraform-cloud-sync-fns.ts index a58ec213c..cb546ba63 100644 --- a/backend/src/services/secret-sync/terraform-cloud/terraform-cloud-sync-fns.ts +++ b/backend/src/services/secret-sync/terraform-cloud/terraform-cloud-sync-fns.ts @@ -232,8 +232,11 @@ export const TerraformCloudSyncFns = { if (secretSync.syncOptions.disableSecretDeletion) return; for (const terraformCloudVariable of terraformCloudVariables) { - // eslint-disable-next-line no-continue - if (!matchesSchema(terraformCloudVariable.key, secretSync.syncOptions.keySchema)) continue; + if ( + !matchesSchema(terraformCloudVariable.key, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema) + ) + // eslint-disable-next-line no-continue + continue; if (!Object.prototype.hasOwnProperty.call(secretMap, terraformCloudVariable.key)) { await deleteVariable(secretSync, terraformCloudVariable); diff --git a/backend/src/services/secret-sync/vercel/vercel-sync-fns.ts b/backend/src/services/secret-sync/vercel/vercel-sync-fns.ts index 90e9327e5..b5ea98265 100644 --- a/backend/src/services/secret-sync/vercel/vercel-sync-fns.ts +++ b/backend/src/services/secret-sync/vercel/vercel-sync-fns.ts @@ -291,8 +291,9 @@ export const VercelSyncFns = { if (secretSync.syncOptions.disableSecretDeletion) return; for await (const vercelSecret of vercelSecrets) { - // eslint-disable-next-line no-continue - if (!matchesSchema(vercelSecret.key, secretSync.syncOptions.keySchema)) continue; + if (!matchesSchema(vercelSecret.key, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema)) + // eslint-disable-next-line no-continue + continue; if (!secretMap[vercelSecret.key]) { await deleteSecret(secretSync, vercelSecret); diff --git a/backend/src/services/secret-sync/windmill/windmill-sync-fns.ts b/backend/src/services/secret-sync/windmill/windmill-sync-fns.ts index a09706581..b5e11c957 100644 --- a/backend/src/services/secret-sync/windmill/windmill-sync-fns.ts +++ b/backend/src/services/secret-sync/windmill/windmill-sync-fns.ts @@ -128,6 +128,7 @@ export const WindmillSyncFns = { syncSecrets: async (secretSync: TWindmillSyncWithCredentials, secretMap: TSecretMap) => { const { connection, + environment, destinationConfig: { path }, syncOptions: { disableSecretDeletion, keySchema } } = secretSync; @@ -171,7 +172,7 @@ export const WindmillSyncFns = { for await (const [key, variable] of Object.entries(variables)) { // eslint-disable-next-line no-continue - if (!matchesSchema(key, keySchema)) continue; + if (!matchesSchema(key, environment?.slug || "", keySchema)) continue; if (!(key in secretMap)) { try { diff --git a/backend/src/services/secret-tag/secret-tag-dal.ts b/backend/src/services/secret-tag/secret-tag-dal.ts index 3b9151557..8768990fe 100644 --- a/backend/src/services/secret-tag/secret-tag-dal.ts +++ b/backend/src/services/secret-tag/secret-tag-dal.ts @@ -11,6 +11,7 @@ export const secretTagDALFactory = (db: TDbClient) => { const secretTagOrm = ormify(db, TableName.SecretTag); const secretJnTagOrm = ormify(db, TableName.JnSecretTag); const secretV2JnTagOrm = ormify(db, TableName.SecretV2JnTag); + const secretVersionV2TagOrm = ormify(db, TableName.SecretVersionV2Tag); const findManyTagsById = async (projectId: string, ids: string[], tx?: Knex) => { try { @@ -48,14 +49,39 @@ export const secretTagDALFactory = (db: TDbClient) => { } }; + const findSecretTagsByVersionId = async (versionId: string, tx?: Knex) => { + try { + const tags = await (tx || db.replicaNode())(TableName.SecretVersionV2Tag) + .where(`${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id`, versionId) + .select(selectAllTableCols(TableName.SecretVersionV2Tag)); + return tags; + } catch (error) { + throw new DatabaseError({ error, name: "Find all by version id" }); + } + }; + + const findSecretTagsBySecretId = async (secretId: string, tx?: Knex) => { + try { + const tags = await (tx || db.replicaNode())(TableName.SecretV2JnTag) + .where(`${TableName.SecretV2JnTag}.${TableName.SecretV2}Id`, secretId) + .select(selectAllTableCols(TableName.SecretV2JnTag)); + return tags; + } catch (error) { + throw new DatabaseError({ error, name: "Find all by secret id" }); + } + }; + return { ...secretTagOrm, saveTagsToSecret: secretJnTagOrm.insertMany, deleteTagsToSecret: secretJnTagOrm.delete, saveTagsToSecretV2: secretV2JnTagOrm.batchInsert, deleteTagsToSecretV2: secretV2JnTagOrm.delete, + saveTagsToSecretVersionV2: secretVersionV2TagOrm.insertMany, findSecretTagsByProjectId, deleteTagsManySecret, - findManyTagsById + findManyTagsById, + findSecretTagsByVersionId, + findSecretTagsBySecretId }; }; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts index 6fdcadeff..1adfac22c 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts @@ -8,6 +8,7 @@ import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; import { ActorType } from "../auth/auth-type"; +import { CommitType } from "../folder-commit/folder-commit-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema"; import { INFISICAL_SECRET_VALUE_HIDDEN_MASK } from "../secret/secret-fns"; @@ -73,6 +74,7 @@ export const fnSecretBulkInsert = async ({ resourceMetadataDAL, secretTagDAL, secretVersionTagDAL, + folderCommitService, actor, tx }: TFnSecretBulkInsert) => { @@ -126,11 +128,36 @@ export const fnSecretBulkInsert = async ({ userActorId, identityActorId, actorType, + metadata: el.metadata ? JSON.stringify(el.metadata) : [], secretId: newSecretGroupedByKeyName[el.key][0].id })), tx ); + const commitChanges = secretVersions + .filter(({ type }) => type === SecretType.Shared) + .map((sv) => ({ + type: CommitType.ADD, + secretVersionId: sv.id + })); + + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType || ActorType.PLATFORM, + metadata: { + id: actor?.actorId + } + }, + message: "Secret Created", + folderId, + changes: commitChanges + }, + tx + ); + } + await secretDAL.upsertSecretReferences( inputSecrets.map(({ references = [], key }) => ({ secretId: newSecretGroupedByKeyName[key][0].id, @@ -185,6 +212,7 @@ export const fnSecretBulkUpdate = async ({ orgId, secretDAL, secretVersionDAL, + folderCommitService, secretTagDAL, secretVersionTagDAL, resourceMetadataDAL, @@ -246,7 +274,7 @@ export const fnSecretBulkUpdate = async ({ userId, encryptedComment, version, - metadata, + metadata: metadata ? JSON.stringify(metadata) : [], reminderNote, encryptedValue, reminderRepeatDays, @@ -259,6 +287,7 @@ export const fnSecretBulkUpdate = async ({ ), tx ); + await secretDAL.upsertSecretReferences( inputSecrets .filter(({ data: { references } }) => Boolean(references)) @@ -329,6 +358,31 @@ export const fnSecretBulkUpdate = async ({ }, { tx } ); + + const commitChanges = secretVersions + .filter(({ type }) => type === SecretType.Shared) + .map((sv) => ({ + type: CommitType.ADD, + isUpdate: true, + secretVersionId: sv.id + })); + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType || ActorType.PLATFORM, + metadata: { + id: actor?.actorId + } + }, + message: "Secret Updated", + folderId, + changes: commitChanges + }, + tx + ); + } + return secretsWithTags.map((secret) => ({ ...secret, _id: secret.id })); }; @@ -337,8 +391,11 @@ export const fnSecretBulkDelete = async ({ inputSecrets, tx, actorId, + actorType, secretDAL, - secretQueueService + secretQueueService, + folderCommitService, + secretVersionDAL }: TFnSecretBulkDelete) => { const deletedSecrets = await secretDAL.deleteMany( inputSecrets.map(({ type, secretKey }) => ({ @@ -358,6 +415,35 @@ export const fnSecretBulkDelete = async ({ ) ); + const secretVersions = await secretVersionDAL.findLatestVersionMany( + folderId, + deletedSecrets.map(({ id }) => id), + tx + ); + + const commitChanges = deletedSecrets + .filter(({ type }) => type === SecretType.Shared) + .map(({ id }) => ({ + type: CommitType.DELETE, + secretVersionId: secretVersions[id].id + })); + if (commitChanges.length > 0) { + await folderCommitService.createCommit( + { + actor: { + type: actorType || ActorType.PLATFORM, + metadata: { + id: actorId + } + }, + message: "Secret Deleted", + folderId, + changes: commitChanges + }, + tx + ); + } + return deletedSecrets; }; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index 1ef4a2d41..c4b5835c4 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -17,6 +17,7 @@ import { import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, + ProjectPermissionCommitsActions, ProjectPermissionSecretActions, ProjectPermissionSet, ProjectPermissionSub @@ -34,6 +35,7 @@ import { logger } from "@app/lib/logger"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { ActorType } from "../auth/auth-type"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { KmsDataKey } from "../kms/kms-types"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; @@ -90,6 +92,7 @@ type TSecretV2BridgeServiceFactoryDep = { secretVersionTagDAL: Pick; secretTagDAL: TSecretTagDALFactory; permissionService: Pick; + folderCommitService: Pick; projectEnvDAL: Pick; folderDAL: Pick< TSecretFolderDALFactory, @@ -124,6 +127,7 @@ export const secretV2BridgeServiceFactory = ({ projectEnvDAL, secretTagDAL, secretVersionDAL, + folderCommitService, folderDAL, permissionService, snapshotService, @@ -321,12 +325,14 @@ export const secretV2BridgeServiceFactory = ({ userId: inputSecret.type === SecretType.Personal ? actorId : null, tagIds: inputSecret.tagIds, references: nestedReferences, + metadata: secretMetadata ? JSON.stringify(secretMetadata) : [], secretMetadata } ], resourceMetadataDAL, secretDAL, secretVersionDAL, + folderCommitService, secretTagDAL, secretVersionTagDAL, actor: { @@ -510,6 +516,7 @@ export const secretV2BridgeServiceFactory = ({ folderId, orgId: actorOrgId, resourceMetadataDAL, + folderCommitService, inputSecrets: [ { filter: { id: secretId }, @@ -523,6 +530,7 @@ export const secretV2BridgeServiceFactory = ({ skipMultilineEncoding: inputSecret.skipMultilineEncoding, key: inputSecret.newSecretName || secretName, tags: inputSecret.tagIds, + metadata: secretMetadata ? JSON.stringify(secretMetadata) : [], secretMetadata, ...encryptedValue } @@ -650,6 +658,9 @@ export const secretV2BridgeServiceFactory = ({ projectId, folderId, actorId, + actorType: actor, + folderCommitService, + secretVersionDAL, secretDAL, secretQueueService, inputSecrets: [ @@ -1590,6 +1601,7 @@ export const secretV2BridgeServiceFactory = ({ orgId: actorOrgId, secretDAL, resourceMetadataDAL, + folderCommitService, secretVersionDAL, secretTagDAL, secretVersionTagDAL, @@ -1859,6 +1871,7 @@ export const secretV2BridgeServiceFactory = ({ const bulkUpdatedSecrets = await fnSecretBulkUpdate({ folderId, orgId: actorOrgId, + folderCommitService, tx, inputSecrets: secretsToUpdate.map((el) => { const originalSecret = secretsToUpdateInDBGroupedByKey[el.secretKey][0]; @@ -1928,6 +1941,7 @@ export const secretV2BridgeServiceFactory = ({ secretVersionDAL, secretTagDAL, secretVersionTagDAL, + folderCommitService, actor: { type: actor, actorId @@ -2061,6 +2075,8 @@ export const secretV2BridgeServiceFactory = ({ fnSecretBulkDelete({ secretDAL, secretQueueService, + folderCommitService, + secretVersionDAL, inputSecrets: inputSecrets.map(({ type, secretKey }) => ({ secretKey, type: type || SecretType.Shared @@ -2068,6 +2084,7 @@ export const secretV2BridgeServiceFactory = ({ projectId, folderId, actorId, + actorType: actor, tx }) ); @@ -2159,15 +2176,25 @@ export const secretV2BridgeServiceFactory = ({ actorOrgId, actionProjectType: ActionProjectType.SecretManager }); - ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback); + + const canRead = + permission.can(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback) || + permission.can(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits); + + if (!canRead) throw new ForbiddenRequestError({ message: "You do not have permission to read secret versions" }); + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, projectId: folder.projectId }); - const secretVersions = await secretVersionDAL.findVersionsBySecretIdWithActors(secretId, folder.projectId, { - offset, - limit, - sort: [["createdAt", "desc"]] + const secretVersions = await secretVersionDAL.findVersionsBySecretIdWithActors({ + secretId, + projectId: folder.projectId, + findOpt: { + offset, + limit, + sort: [["createdAt", "desc"]] + } }); return secretVersions.map((el) => { const secretValueHidden = !hasSecretReadValueOrDescribePermission( @@ -2469,6 +2496,7 @@ export const secretV2BridgeServiceFactory = ({ tx, secretTagDAL, resourceMetadataDAL, + folderCommitService, secretVersionTagDAL, actor: { type: actor, @@ -2495,6 +2523,7 @@ export const secretV2BridgeServiceFactory = ({ folderId: destinationFolder.id, orgId: actorOrgId, resourceMetadataDAL, + folderCommitService, secretVersionDAL, secretDAL, tx, @@ -2840,6 +2869,76 @@ export const secretV2BridgeServiceFactory = ({ }; }; + const getSecretVersionsByIds = async ({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + secretId, + secretVersionNumbers, + secretPath, + envId, + projectId + }: TGetSecretVersionsDTO & { + secretVersionNumbers: string[]; + secretPath: string; + envId: string; + projectId: string; + }) => { + const environment = await projectEnvDAL.findOne({ id: envId, projectId }); + + const { permission } = await permissionService.getProjectPermission({ + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId, + actionProjectType: ActionProjectType.SecretManager + }); + + const canRead = + permission.can(ProjectPermissionActions.Read, ProjectPermissionSub.SecretRollback) || + permission.can(ProjectPermissionCommitsActions.Read, ProjectPermissionSub.Commits); + + if (!canRead) throw new ForbiddenRequestError({ message: "You do not have permission to read secret versions" }); + + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + const secretVersions = await secretVersionDAL.findVersionsBySecretIdWithActors({ + secretId, + projectId, + secretVersions: secretVersionNumbers + }); + return secretVersions.map((el) => { + const secretValueHidden = !hasSecretReadValueOrDescribePermission( + permission, + ProjectPermissionSecretActions.ReadValue, + { + environment: environment.slug, + secretPath, + secretName: el.key, + ...(el.tags?.length && { + secretTags: el.tags.map((tag) => tag.slug) + }) + } + ); + + return reshapeBridgeSecret( + projectId, + environment.slug, + secretPath, + { + ...el, + value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : "", + comment: el.encryptedComment ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() : "" + }, + secretValueHidden + ); + }); + }; + return { createSecret, deleteSecret, @@ -2858,6 +2957,7 @@ export const secretV2BridgeServiceFactory = ({ getSecretReferenceTree, getSecretsByFolderMappings, getSecretById, - getAccessibleSecrets + getAccessibleSecrets, + getSecretVersionsByIds }; }; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts index f4a27d4c5..f4171b1a7 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts @@ -8,6 +8,7 @@ import { SecretsOrderBy } from "@app/services/secret/secret-types"; import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema"; import { TSecretV2BridgeDALFactory } from "./secret-v2-bridge-dal"; @@ -178,9 +179,10 @@ export type TFnSecretBulkInsert = { secretVersionDAL: Pick; secretTagDAL: Pick; secretVersionTagDAL: Pick; + folderCommitService: Pick; actor?: { type: string; - actorId: string; + actorId?: string; }; }; @@ -206,9 +208,10 @@ export type TFnSecretBulkUpdate = { secretVersionDAL: Pick; secretTagDAL: Pick; secretVersionTagDAL: Pick; + folderCommitService: Pick; actor?: { type: string; - actorId: string; + actorId?: string; }; tx?: Knex; }; @@ -218,11 +221,14 @@ export type TFnSecretBulkDelete = { projectId: string; inputSecrets: Array<{ type: SecretType; secretKey: string }>; actorId: string; + actorType?: string; tx?: Knex; secretDAL: Pick; secretQueueService: { removeSecretReminder: (data: TRemoveSecretReminderDTO, tx?: Knex) => Promise; }; + folderCommitService: Pick; + secretVersionDAL: Pick; }; export type THandleReminderDTO = { diff --git a/backend/src/services/secret-v2-bridge/secret-version-dal.ts b/backend/src/services/secret-v2-bridge/secret-version-dal.ts index b54b073a6..9537b79e3 100644 --- a/backend/src/services/secret-v2-bridge/secret-version-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-version-dal.ts @@ -4,7 +4,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; import { SecretVersionsV2Schema, TableName, TSecretVersionsV2, TSecretVersionsV2Update } from "@app/db/schemas"; import { BadRequestError, DatabaseError } from "@app/lib/errors"; -import { ormify, selectAllTableCols, sqlNestRelationships, TFindOpt } from "@app/lib/knex"; +import { buildFindFilter, ormify, selectAllTableCols, sqlNestRelationships, TFindOpt } from "@app/lib/knex"; import { logger } from "@app/lib/logger"; import { QueueName } from "@app/queue"; @@ -138,7 +138,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { {} ); } catch (error) { - throw new DatabaseError({ error, name: "FindLatestVersinMany" }); + throw new DatabaseError({ error, name: "FindLatestVersionMany" }); } }; @@ -162,6 +162,8 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`) .join("version_cte", "version_cte.id", `${TableName.SecretVersionV2}.id`) .whereRaw(`version_cte.row_num > ${TableName.Project}."pitVersionLimit"`) + // Projects with version >= 3 will require to have all secret versions for PIT + .andWhere(`${TableName.Project}.version`, "<", 3) .delete(); } catch (error) { throw new DatabaseError({ @@ -172,13 +174,21 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { logger.info(`${QueueName.DailyResourceCleanUp}: pruning secret version v2 completed`); }; - const findVersionsBySecretIdWithActors = async ( - secretId: string, - projectId: string, - { offset, limit, sort = [["createdAt", "desc"]] }: TFindOpt = {}, - tx?: Knex - ) => { + const findVersionsBySecretIdWithActors = async ({ + secretId, + projectId, + secretVersions, + findOpt = {}, + tx + }: { + secretId: string; + projectId: string; + secretVersions?: string[]; + findOpt?: TFindOpt; + tx?: Knex; + }) => { try { + const { offset, limit, sort = [["createdAt", "desc"]] } = findOpt; const query = (tx || db)(TableName.SecretVersionV2) .leftJoin(TableName.Users, `${TableName.Users}.id`, `${TableName.SecretVersionV2}.userActorId`) .leftJoin( @@ -189,22 +199,24 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { .leftJoin(TableName.Identity, `${TableName.Identity}.id`, `${TableName.SecretVersionV2}.identityActorId`) .leftJoin(TableName.SecretV2, `${TableName.SecretVersionV2}.secretId`, `${TableName.SecretV2}.id`) .leftJoin( - TableName.SecretV2JnTag, - `${TableName.SecretV2}.id`, - `${TableName.SecretV2JnTag}.${TableName.SecretV2}Id` + TableName.SecretVersionV2Tag, + `${TableName.SecretVersionV2}.id`, + `${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id` ) .leftJoin( TableName.SecretTag, - `${TableName.SecretV2JnTag}.${TableName.SecretTag}Id`, + `${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`, `${TableName.SecretTag}.id` ) .where((qb) => { void qb.where(`${TableName.SecretVersionV2}.secretId`, secretId); void qb.where(`${TableName.ProjectMembership}.projectId`, projectId); + if (secretVersions?.length) void qb.whereIn(`${TableName.SecretVersionV2}.version`, secretVersions); }) .orWhere((qb) => { void qb.where(`${TableName.SecretVersionV2}.secretId`, secretId); void qb.whereNull(`${TableName.ProjectMembership}.projectId`); + if (secretVersions?.length) void qb.whereIn(`${TableName.SecretVersionV2}.version`, secretVersions); }) .select( selectAllTableCols(TableName.SecretVersionV2), @@ -260,6 +272,178 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { } }; + // Function to fetch latest versions by secretIds + const getLatestVersionsBySecretIds = async ( + folderId: string, + secretIds: string[], + tx?: Knex + ): Promise> => { + if (!secretIds.length) return []; + + const knexInstance = tx || db.replicaNode(); + return knexInstance(TableName.SecretVersionV2) + .where("folderId", folderId) + .whereIn(`${TableName.SecretVersionV2}.secretId`, secretIds) + .join( + knexInstance(TableName.SecretVersionV2) + .groupBy("secretId") + .max("version") + .select("secretId") + .as("latestVersion"), + (bd) => { + bd.on(`${TableName.SecretVersionV2}.secretId`, "latestVersion.secretId").andOn( + `${TableName.SecretVersionV2}.version`, + "latestVersion.max" + ); + } + ); + }; + + // Function to fetch specific versions by versionIds + const getSpecificVersionsWithLatestInfo = async ( + folderId: string, + versionIds: string[], + tx?: Knex + ): Promise> => { + if (!versionIds.length) return []; + + const knexInstance = tx || db.replicaNode(); + + // Get the specific versions + const specificVersions = await knexInstance(TableName.SecretVersionV2) + .where("folderId", folderId) + .whereIn("id", versionIds); + + // Get the secretIds from these versions + const specificSecretIds = [...new Set(specificVersions.map((v) => v.secretId).filter(Boolean))]; + + if (!specificSecretIds.length) return specificVersions; + + // Get max versions for these secretIds + const maxVersionsQuery = await knexInstance(TableName.SecretVersionV2) + .whereIn("secretId", specificSecretIds) + .groupBy("secretId") + .select("secretId") + .max("version", { as: "maxVersion" }); + + // Create a lookup map for max versions + const maxVersionMap = maxVersionsQuery.reduce( + (acc, item) => { + acc[item.secretId] = item.maxVersion; + return acc; + }, + {} as Record + ); + + // Update the version field with maxVersion when needed + return specificVersions.map((version) => { + // Replace version with maxVersion + return { + ...version, + version: maxVersionMap[version.secretId] || version.version + }; + }); + }; + + const findByIdsWithLatestVersion = async ( + folderId: string, + secretIds: string[], + versionIds?: string[], + tx?: Knex + ) => { + try { + if (!secretIds.length && (!versionIds || !versionIds.length)) return {}; + + const [latestVersions, specificVersionsWithLatest] = await Promise.all([ + secretIds.length ? getLatestVersionsBySecretIds(folderId, secretIds, tx) : [], + versionIds?.length ? getSpecificVersionsWithLatestInfo(folderId, versionIds, tx) : [] + ]); + + const allDocs = [...latestVersions, ...specificVersionsWithLatest]; + + // Convert array to record with secretId as key + return allDocs.reduce>( + (prev, curr) => ({ ...prev, [curr.secretId || ""]: curr }), + {} + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindByIdsWithLatestVersion" }); + } + }; + + const findByIdAndPreviousVersion = async (secretVersionId: string, tx?: Knex) => { + try { + const targetSecretVersion = await (tx || db.replicaNode())(TableName.SecretVersionV2) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter({ id: secretVersionId }, TableName.SecretVersionV2)) + .leftJoin( + TableName.SecretVersionV2Tag, + `${TableName.SecretVersionV2}.id`, + `${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id` + ) + .leftJoin( + TableName.SecretTag, + `${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`, + `${TableName.SecretTag}.id` + ) + .select(selectAllTableCols(TableName.SecretVersionV2)) + .select(db.ref("id").withSchema(TableName.SecretTag).as("tagId")) + .select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor")) + .select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")) + .first(); + if (targetSecretVersion) { + const previousSecretVersion = await (tx || db.replicaNode())(TableName.SecretVersionV2) + .where( + // eslint-disable-next-line @typescript-eslint/no-misused-promises + buildFindFilter( + { version: targetSecretVersion.version - 1, secretId: targetSecretVersion.secretId }, + TableName.SecretVersionV2 + ) + ) + .leftJoin( + TableName.SecretVersionV2Tag, + `${TableName.SecretVersionV2}.id`, + `${TableName.SecretVersionV2Tag}.${TableName.SecretVersionV2}Id` + ) + .leftJoin( + TableName.SecretTag, + `${TableName.SecretVersionV2Tag}.${TableName.SecretTag}Id`, + `${TableName.SecretTag}.id` + ) + .select(selectAllTableCols(TableName.SecretVersionV2)) + .select(db.ref("id").withSchema(TableName.SecretTag).as("tagId")) + .select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor")) + .select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")) + .first(); + if (!previousSecretVersion) return []; + const docs = [previousSecretVersion, targetSecretVersion]; + + const data = sqlNestRelationships({ + data: docs, + key: "id", + parentMapper: (el) => ({ _id: el.id, ...SecretVersionsV2Schema.parse(el) }), + childrenMapper: [ + { + key: "tagId", + label: "tags" as const, + mapper: ({ tagId: id, tagColor: color, tagSlug: slug }) => ({ + id, + color, + slug, + name: slug + }) + } + ] + }); + + return data; + } + return []; + } catch (error) { + throw new DatabaseError({ error, name: "FindByIdAndPreviousVersion" }); + } + }; + return { ...secretVersionV2Orm, pruneExcessVersions, @@ -267,6 +451,8 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { bulkUpdate, findLatestVersionByFolderId, findVersionsBySecretIdWithActors, - findBySecretId + findBySecretId, + findByIdsWithLatestVersion, + findByIdAndPreviousVersion }; }; diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index e5f3acdea..96f89ab5f 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -778,6 +778,7 @@ export const createManySecretsRawFnFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, + folderCommitService, kmsService, resourceMetadataDAL }: TCreateManySecretsRawFnFactory) => { @@ -850,6 +851,7 @@ export const createManySecretsRawFnFactory = ({ secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService, tx }) ); @@ -942,6 +944,7 @@ export const updateManySecretsRawFnFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, resourceMetadataDAL, + folderCommitService, kmsService }: TUpdateManySecretsRawFnFactory) => { const getBotKeyFn = getBotKeyFnFactory(projectBotDAL, projectDAL); @@ -1032,7 +1035,8 @@ export const updateManySecretsRawFnFactory = ({ secretDAL: secretV2BridgeDAL, secretVersionDAL: secretVersionV2BridgeDAL, secretTagDAL, - secretVersionTagDAL: secretVersionTagV2BridgeDAL + secretVersionTagDAL: secretVersionTagV2BridgeDAL, + folderCommitService }) ); diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 714df0d3f..e9c6f2d88 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -35,6 +35,7 @@ import { TSecretSyncQueueFactory } from "@app/services/secret-sync/secret-sync-q import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { ActorType } from "../auth/auth-type"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TIntegrationDALFactory } from "../integration/integration-dal"; import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal"; import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service"; @@ -112,6 +113,7 @@ type TSecretQueueFactoryDep = { orgService: Pick; projectUserMembershipRoleDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; secretReminderRecipientsDAL: Pick< TSecretReminderRecipientsDALFactory, "delete" | "findUsersBySecretId" | "insertMany" | "transaction" @@ -178,7 +180,8 @@ export const secretQueueFactory = ({ projectKeyDAL, resourceMetadataDAL, secretReminderRecipientsDAL, - secretSyncQueue + secretSyncQueue, + folderCommitService }: TSecretQueueFactoryDep) => { const integrationMeter = opentelemetry.metrics.getMeter("Integrations"); const errorHistogram = integrationMeter.createHistogram("integration_secret_sync_errors", { @@ -366,7 +369,8 @@ export const secretQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); const updateManySecretsRawFn = updateManySecretsRawFnFactory({ @@ -382,7 +386,8 @@ export const secretQueueFactory = ({ secretVersionV2BridgeDAL, secretV2BridgeDAL, secretVersionTagV2BridgeDAL, - resourceMetadataDAL + resourceMetadataDAL, + folderCommitService }); /** @@ -1581,6 +1586,7 @@ export const secretQueueFactory = ({ projectDAL, webhookDAL, event: job.data, + auditLogService, secretManagerDecryptor: (value) => secretManagerDecryptor({ cipherTextBlob: value }).toString() }); }); diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 46ed7ef83..4fa7404d3 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -44,7 +44,8 @@ import { TGetSecretsRawByFolderMappingsDTO } from "@app/services/secret-v2-bridge/secret-v2-bridge-types"; -import { ActorType } from "../auth/auth-type"; +import { ActorAuthMethod, ActorType } from "../auth/auth-type"; +import { ChangeType } from "../folder-commit/folder-commit-service"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; @@ -2521,6 +2522,36 @@ export const secretServiceFactory = ({ }); }; + const getSecretVersionsV2ByIds = async ({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + secretId, + secretVersions, + secretPath, + envId, + projectId + }: TGetSecretVersionsDTO & { + secretVersions: string[]; + secretPath: string; + envId: string; + projectId: string; + }) => { + const secretVersionV2 = await secretV2BridgeService.getSecretVersionsByIds({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + secretId, + secretVersionNumbers: secretVersions, + secretPath, + envId, + projectId + }); + return secretVersionV2; + }; + const attachTags = async ({ secretName, tagSlugs, @@ -3279,6 +3310,53 @@ export const secretServiceFactory = ({ return secrets; }; + const getChangeVersions = async ( + change: { + secretVersion: string; + secretId?: string; + id?: string; + isUpdate?: boolean; + changeType?: string; + }, + previousVersion: string, + actorId: string, + actor: ActorType, + actorOrgId: string, + actorAuthMethod: ActorAuthMethod, + envId: string, + projectId: string, + secretPath: string + ) => { + const currentVersion = change.secretVersion; + const secretId = change.secretId ? change.secretId : change.id; + if (!secretId) { + return; + } + const versions = await getSecretVersionsV2ByIds({ + actorId, + actor, + actorOrgId, + actorAuthMethod, + secretId, + // if it's update add also the previous secretversionid + secretVersions: + change.isUpdate || change.changeType === ChangeType.UPDATE + ? [currentVersion, previousVersion] + : [currentVersion], + secretPath, + envId, + projectId + }); + return versions?.map((v) => ({ + secretKey: v.secretKey, + secretComment: v.secretComment, + skipMultilineEncoding: v.skipMultilineEncoding, + tags: v.tags?.map((tag) => tag.slug), + metadata: v.metadata, + secretValue: v.secretValue + })); + }; + return { attachTags, detachTags, @@ -3309,6 +3387,8 @@ export const secretServiceFactory = ({ getSecretsRawByFolderMappings, getSecretAccessList, getSecretByIdRaw, - getAccessibleSecrets + getAccessibleSecrets, + getSecretVersionsV2ByIds, + getChangeVersions }; }; diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 30e3dfafa..91fc2eb6a 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -14,6 +14,7 @@ import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-fold import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { ActorType } from "../auth/auth-type"; +import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service"; import { TKmsServiceFactory } from "../kms/kms-service"; import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal"; import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema"; @@ -441,6 +442,7 @@ export type TCreateManySecretsRawFnFactory = { secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; }; export type TCreateManySecretsRawFn = { @@ -478,6 +480,7 @@ export type TUpdateManySecretsRawFnFactory = { secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; resourceMetadataDAL: Pick; + folderCommitService: Pick; }; export type TUpdateManySecretsRawFn = { diff --git a/backend/src/services/user/user-dal.ts b/backend/src/services/user/user-dal.ts index b5a29fc8c..0f623dff1 100644 --- a/backend/src/services/user/user-dal.ts +++ b/backend/src/services/user/user-dal.ts @@ -21,6 +21,11 @@ export const userDALFactory = (db: TDbClient) => { const findUserByUsername = async (username: string, tx?: Knex) => (tx || db)(TableName.Users).whereRaw('lower("username") = :username', { username: username.toLowerCase() }); + const findUserByEmail = async (email: string, tx?: Knex) => + (tx || db)(TableName.Users).whereRaw('lower("email") = :email', { email: email.toLowerCase() }).where({ + isEmailVerified: true + }); + const getUsersByFilter = async ({ limit, offset, @@ -234,6 +239,7 @@ export const userDALFactory = (db: TDbClient) => { findOneUserAction, createUserAction, getUsersByFilter, - findAllMyAccounts + findAllMyAccounts, + findUserByEmail }; }; diff --git a/backend/src/services/user/user-service.ts b/backend/src/services/user/user-service.ts index aae32d91f..39300fec5 100644 --- a/backend/src/services/user/user-service.ts +++ b/backend/src/services/user/user-service.ts @@ -107,7 +107,8 @@ export const userServiceFactory = ({ }); await userDAL.updateById(user.id, { - isEmailVerified: true + isEmailVerified: true, + username: usersByusername.length === 1 && user.email ? user.email.toLowerCase() : undefined }); }; diff --git a/backend/src/services/webhook/webhook-fns.ts b/backend/src/services/webhook/webhook-fns.ts index a16158e14..6729e9a37 100644 --- a/backend/src/services/webhook/webhook-fns.ts +++ b/backend/src/services/webhook/webhook-fns.ts @@ -4,9 +4,12 @@ import { AxiosError } from "axios"; import picomatch from "picomatch"; import { TWebhooks } from "@app/db/schemas"; +import { TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-service"; +import { EventType, WebhookTriggeredEvent } from "@app/ee/services/audit-log/audit-log-types"; import { request } from "@app/lib/config/request"; import { NotFoundError } from "@app/lib/errors"; import { logger } from "@app/lib/logger"; +import { ActorType } from "@app/services/auth/auth-type"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; @@ -163,6 +166,7 @@ export type TFnTriggerWebhookDTO = { projectEnvDAL: Pick; projectDAL: Pick; secretManagerDecryptor: (value: Buffer) => string; + auditLogService: Pick; }; // this is reusable function @@ -175,7 +179,8 @@ export const fnTriggerWebhook = async ({ projectEnvDAL, event, secretManagerDecryptor, - projectDAL + projectDAL, + auditLogService }: TFnTriggerWebhookDTO) => { const webhooks = await webhookDAL.findAllWebhooks(projectId, environment); const toBeTriggeredHooks = webhooks.filter( @@ -200,16 +205,43 @@ export const fnTriggerWebhook = async ({ }) ); + const eventPayloads: WebhookTriggeredEvent["metadata"][] = []; // filter hooks by status const successWebhooks = webhooksTriggered .filter(({ status }) => status === "fulfilled") - .map((_, i) => toBeTriggeredHooks[i].id); + .map((_, i) => { + eventPayloads.push({ + webhookId: toBeTriggeredHooks[i].id, + type: event.type, + payload: { + type: toBeTriggeredHooks[i].type!, + ...event.payload, + projectName + }, + status: "success" + } as WebhookTriggeredEvent["metadata"]); + + return toBeTriggeredHooks[i].id; + }); const failedWebhooks = webhooksTriggered .filter(({ status }) => status === "rejected") - .map((data, i) => ({ - id: toBeTriggeredHooks[i].id, - error: data.status === "rejected" ? (data.reason as AxiosError).message : "" - })); + .map((data, i) => { + eventPayloads.push({ + webhookId: toBeTriggeredHooks[i].id, + type: event.type, + payload: { + type: toBeTriggeredHooks[i].type!, + ...event.payload, + projectName + }, + status: "failed" + } as WebhookTriggeredEvent["metadata"]); + + return { + id: toBeTriggeredHooks[i].id, + error: data.status === "rejected" ? (data.reason as AxiosError).message : "" + }; + }); await webhookDAL.transaction(async (tx) => { const env = await projectEnvDAL.findOne({ projectId, slug: environment }, tx); @@ -236,5 +268,21 @@ export const fnTriggerWebhook = async ({ ); } }); + + for (const eventPayload of eventPayloads) { + // eslint-disable-next-line no-await-in-loop + await auditLogService.createAuditLog({ + actor: { + type: ActorType.PLATFORM, + metadata: {} + }, + projectId, + event: { + type: EventType.WEBHOOK_TRIGGERED, + metadata: eventPayload + } + }); + } + logger.info({ environment, secretPath, projectId }, "Secret webhook job ended"); }; diff --git a/cli/go.mod b/cli/go.mod index 229e37137..e6d55eb49 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -14,7 +14,7 @@ require ( github.com/fatih/semgroup v1.2.0 github.com/gitleaks/go-gitdiff v0.9.1 github.com/h2non/filetype v1.1.3 - github.com/infisical/go-sdk v0.5.92 + github.com/infisical/go-sdk v0.5.96 github.com/infisical/infisical-kmip v0.3.5 github.com/mattn/go-isatty v0.0.20 github.com/muesli/ansi v0.0.0-20221106050444-61f0cd9a192a @@ -25,6 +25,7 @@ require ( github.com/pion/logging v0.2.3 github.com/pion/turn/v4 v4.0.0 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c + github.com/pkg/errors v0.9.1 github.com/posthog/posthog-go v0.0.0-20221221115252-24dfed35d71a github.com/quic-go/quic-go v0.50.0 github.com/rs/cors v1.11.0 @@ -106,7 +107,6 @@ require ( github.com/pion/randutil v0.1.0 // indirect github.com/pion/stun/v3 v3.0.0 // indirect github.com/pion/transport/v3 v3.0.7 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect diff --git a/cli/go.sum b/cli/go.sum index d253c70d5..2e41c756b 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -292,8 +292,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/infisical/go-sdk v0.5.92 h1:PoCnVndrd6Dbkipuxl9fFiwlD5vCKsabtQo09mo8lUE= -github.com/infisical/go-sdk v0.5.92/go.mod h1:ExjqFLRz7LSpZpGluqDLvFl6dFBLq5LKyLW7GBaMAIs= +github.com/infisical/go-sdk v0.5.96 h1:huky6bQ1Y3oRdPb5MO3Ru868qZaPHUxZ7kP7FPNRn48= +github.com/infisical/go-sdk v0.5.96/go.mod h1:ExjqFLRz7LSpZpGluqDLvFl6dFBLq5LKyLW7GBaMAIs= github.com/infisical/infisical-kmip v0.3.5 h1:QM3s0e18B+mYv3a9HQNjNAlbwZJBzXq5BAJM2scIeiE= github.com/infisical/infisical-kmip v0.3.5/go.mod h1:bO1M4YtKyutNg1bREPmlyZspC5duSR7hyQ3lPmLzrIs= github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo= diff --git a/cli/packages/cmd/dynamic_secrets.go b/cli/packages/cmd/dynamic_secrets.go index 8761b84ef..45bc323d8 100644 --- a/cli/packages/cmd/dynamic_secrets.go +++ b/cli/packages/cmd/dynamic_secrets.go @@ -232,13 +232,26 @@ func createDynamicSecretLeaseByName(cmd *cobra.Command, args []string) { util.HandleError(err, "To fetch dynamic secret root credentials details") } + // for Kubernetes dynamic secrets only + kubernetesNamespace, err := cmd.Flags().GetString("kubernetesNamespace") + if err != nil { + util.HandleError(err, "Unable to parse flag") + } + + config := map[string]any{} + if kubernetesNamespace != "" { + config["namespace"] = kubernetesNamespace + } + leaseCredentials, _, leaseDetails, err := infisicalClient.DynamicSecrets().Leases().Create(infisicalSdk.CreateDynamicSecretLeaseOptions{ DynamicSecretName: dynamicSecretRootCredential.Name, ProjectSlug: projectDetails.Slug, TTL: ttl, SecretPath: secretsPath, EnvironmentSlug: environmentName, + Config: config, }) + if err != nil { util.HandleError(err, "To lease dynamic secret") } @@ -585,6 +598,10 @@ func init() { dynamicSecretLeaseCreateCmd.Flags().String("projectId", "", "Manually set the projectId to fetch leased from when using machine identity based auth") dynamicSecretLeaseCreateCmd.Flags().String("ttl", "", "The lease lifetime TTL. If not provided the default TTL of dynamic secret will be used.") dynamicSecretLeaseCreateCmd.Flags().Bool("plain", false, "Print leased credentials without formatting, one per line") + + // Kubernetes specific flags + dynamicSecretLeaseCreateCmd.Flags().String("kubernetesNamespace", "", "The namespace to create the lease in. Only used for Kubernetes dynamic secrets.") + dynamicSecretLeaseCmd.AddCommand(dynamicSecretLeaseCreateCmd) dynamicSecretLeaseListCmd.Flags().StringP("path", "p", "/", "The path from where dynamic secret should be leased from") diff --git a/cli/packages/cmd/gateway.go b/cli/packages/cmd/gateway.go index 51565b6fd..abc4d6949 100644 --- a/cli/packages/cmd/gateway.go +++ b/cli/packages/cmd/gateway.go @@ -7,16 +7,77 @@ import ( "os/exec" "os/signal" "runtime" + "sync/atomic" "syscall" "time" + "github.com/Infisical/infisical-merge/packages/api" + "github.com/Infisical/infisical-merge/packages/config" "github.com/Infisical/infisical-merge/packages/gateway" "github.com/Infisical/infisical-merge/packages/util" + infisicalSdk "github.com/infisical/go-sdk" + "github.com/pkg/errors" "github.com/posthog/posthog-go" "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) +func getInfisicalSdkInstance(cmd *cobra.Command) (infisicalSdk.InfisicalClientInterface, context.CancelFunc, error) { + + ctx, cancel := context.WithCancel(cmd.Context()) + infisicalClient := infisicalSdk.NewInfisicalClient(ctx, infisicalSdk.Config{ + SiteUrl: config.INFISICAL_URL, + UserAgent: api.USER_AGENT, + }) + + token, err := util.GetInfisicalToken(cmd) + if err != nil { + cancel() + return nil, nil, err + } + + // if the --token param is set, we use it directly for authentication + if token != nil { + infisicalClient.Auth().SetAccessToken(token.Token) + return infisicalClient, cancel, nil + } + + // if the --token param is not set, we use the auth-method flag to determine the authentication method, and perform the appropriate login flow based on that + authMethod, err := util.GetCmdFlagOrEnv(cmd, "auth-method", []string{util.INFISICAL_AUTH_METHOD_NAME}) + + if err != nil { + cancel() + return nil, nil, err + } + + authMethodValid, strategy := util.IsAuthMethodValid(authMethod, false) + if !authMethodValid { + util.PrintErrorMessageAndExit(fmt.Sprintf("Invalid login method: %s", authMethod)) + } + + sdkAuthenticator := util.NewSdkAuthenticator(infisicalClient, cmd) + + authStrategies := map[util.AuthStrategyType]func() (credential infisicalSdk.MachineIdentityCredential, e error){ + util.AuthStrategy.UNIVERSAL_AUTH: sdkAuthenticator.HandleUniversalAuthLogin, + util.AuthStrategy.KUBERNETES_AUTH: sdkAuthenticator.HandleKubernetesAuthLogin, + util.AuthStrategy.AZURE_AUTH: sdkAuthenticator.HandleAzureAuthLogin, + util.AuthStrategy.GCP_ID_TOKEN_AUTH: sdkAuthenticator.HandleGcpIdTokenAuthLogin, + util.AuthStrategy.GCP_IAM_AUTH: sdkAuthenticator.HandleGcpIamAuthLogin, + util.AuthStrategy.AWS_IAM_AUTH: sdkAuthenticator.HandleAwsIamAuthLogin, + util.AuthStrategy.OIDC_AUTH: sdkAuthenticator.HandleOidcAuthLogin, + util.AuthStrategy.JWT_AUTH: sdkAuthenticator.HandleJwtAuthLogin, + } + + _, err = authStrategies[strategy]() + + if err != nil { + cancel() + return nil, nil, err + } + + return infisicalClient, cancel, nil +} + var gatewayCmd = &cobra.Command{ Use: "gateway", Short: "Run the Infisical gateway or manage its systemd service", @@ -26,13 +87,18 @@ var gatewayCmd = &cobra.Command{ DisableFlagsInUseLine: true, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { - token, err := util.GetInfisicalToken(cmd) - if err != nil { - util.HandleError(err, "Unable to parse token flag") - } - if token == nil { - util.HandleError(fmt.Errorf("Token not found")) + infisicalClient, cancelSdk, err := getInfisicalSdkInstance(cmd) + if err != nil { + util.HandleError(err, "unable to get infisical client") + } + defer cancelSdk() + + var accessToken atomic.Value + accessToken.Store(infisicalClient.Auth().GetAccessToken()) + + if accessToken.Load().(string) == "" { + util.HandleError(errors.New("no access token found")) } Telemetry.CaptureEvent("cli-command:gateway", posthog.NewProperties().Set("version", util.CLI_VERSION)) @@ -41,13 +107,14 @@ var gatewayCmd = &cobra.Command{ signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) sigStopCh := make(chan bool, 1) - ctx, cancel := context.WithCancel(cmd.Context()) - defer cancel() + ctx, cancelCmd := context.WithCancel(cmd.Context()) + defer cancelCmd() go func() { <-sigCh close(sigStopCh) - cancel() + cancelCmd() + cancelSdk() // If we get a second signal, force exit <-sigCh @@ -55,6 +122,34 @@ var gatewayCmd = &cobra.Command{ os.Exit(1) }() + var gatewayInstance *gateway.Gateway + + // Token refresh goroutine - runs every 10 seconds + go func() { + tokenRefreshTicker := time.NewTicker(10 * time.Second) + defer tokenRefreshTicker.Stop() + + for { + select { + case <-tokenRefreshTicker.C: + if ctx.Err() != nil { + return + } + + newToken := infisicalClient.Auth().GetAccessToken() + if newToken != "" && newToken != accessToken.Load().(string) { + accessToken.Store(newToken) + if gatewayInstance != nil { + gatewayInstance.UpdateIdentityAccessToken(newToken) + } + } + + case <-ctx.Done(): + return + } + } + }() + // Main gateway retry loop with proper context handling retryTicker := time.NewTicker(5 * time.Second) defer retryTicker.Stop() @@ -64,7 +159,7 @@ var gatewayCmd = &cobra.Command{ log.Info().Msg("Shutting down gateway") return } - gatewayInstance, err := gateway.NewGateway(token.Token) + gatewayInstance, err := gateway.NewGateway(accessToken.Load().(string)) if err != nil { util.HandleError(err) } @@ -126,7 +221,7 @@ var gatewayInstallCmd = &cobra.Command{ } if token == nil { - util.HandleError(fmt.Errorf("Token not found")) + util.HandleError(errors.New("Token not found")) } domain, err := cmd.Flags().GetString("domain") @@ -183,7 +278,7 @@ var gatewayRelayCmd = &cobra.Command{ } if relayConfigFilePath == "" { - util.HandleError(fmt.Errorf("Missing config file")) + util.HandleError(errors.New("Missing config file")) } gatewayRelay, err := gateway.NewGatewayRelay(relayConfigFilePath) @@ -198,7 +293,19 @@ var gatewayRelayCmd = &cobra.Command{ } func init() { - gatewayCmd.Flags().String("token", "", "Connect with Infisical using machine identity access token") + gatewayCmd.Flags().String("token", "", "connect with Infisical using machine identity access token. if not provided, you must set the auth-method flag") + + gatewayCmd.Flags().String("auth-method", "", "login method [universal-auth, kubernetes, azure, gcp-id-token, gcp-iam, aws-iam, oidc-auth]. if not provided, you must set the token flag") + + gatewayCmd.Flags().String("client-id", "", "client id for universal auth") + gatewayCmd.Flags().String("client-secret", "", "client secret for universal auth") + + gatewayCmd.Flags().String("machine-identity-id", "", "machine identity id for kubernetes, azure, gcp-id-token, gcp-iam, and aws-iam auth methods") + gatewayCmd.Flags().String("service-account-token-path", "", "service account token path for kubernetes auth") + gatewayCmd.Flags().String("service-account-key-file-path", "", "service account key file path for GCP IAM auth") + + gatewayCmd.Flags().String("jwt", "", "JWT for jwt-based auth methods [oidc-auth, jwt-auth]") + gatewayInstallCmd.Flags().String("token", "", "Connect with Infisical using machine identity access token") gatewayInstallCmd.Flags().String("domain", "", "Domain of your self-hosted Infisical instance") diff --git a/cli/packages/cmd/kmip.go b/cli/packages/cmd/kmip.go index b0c397895..91335d122 100644 --- a/cli/packages/cmd/kmip.go +++ b/cli/packages/cmd/kmip.go @@ -49,13 +49,13 @@ func startKmipServer(cmd *cobra.Command, args []string) { var identityClientSecret string if strategy == util.AuthStrategy.UNIVERSAL_AUTH { - identityClientId, err = util.GetCmdFlagOrEnv(cmd, "identity-client-id", util.INFISICAL_UNIVERSAL_AUTH_CLIENT_ID_NAME) + identityClientId, err = util.GetCmdFlagOrEnv(cmd, "identity-client-id", []string{util.INFISICAL_UNIVERSAL_AUTH_CLIENT_ID_NAME}) if err != nil { util.HandleError(err, "Unable to parse identity client ID") } - identityClientSecret, err = util.GetCmdFlagOrEnv(cmd, "identity-client-secret", util.INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET_NAME) + identityClientSecret, err = util.GetCmdFlagOrEnv(cmd, "identity-client-secret", []string{util.INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET_NAME}) if err != nil { util.HandleError(err, "Unable to parse identity client secret") } diff --git a/cli/packages/cmd/login.go b/cli/packages/cmd/login.go index b0ce7564b..fd3ce1569 100644 --- a/cli/packages/cmd/login.go +++ b/cli/packages/cmd/login.go @@ -49,97 +49,6 @@ type params struct { keyLength uint32 } -func handleUniversalAuthLogin(cmd *cobra.Command, infisicalClient infisicalSdk.InfisicalClientInterface) (credential infisicalSdk.MachineIdentityCredential, e error) { - - clientId, err := util.GetCmdFlagOrEnv(cmd, "client-id", util.INFISICAL_UNIVERSAL_AUTH_CLIENT_ID_NAME) - - if err != nil { - return infisicalSdk.MachineIdentityCredential{}, err - } - - clientSecret, err := util.GetCmdFlagOrEnv(cmd, "client-secret", util.INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET_NAME) - if err != nil { - return infisicalSdk.MachineIdentityCredential{}, err - } - - return infisicalClient.Auth().UniversalAuthLogin(clientId, clientSecret) -} - -func handleKubernetesAuthLogin(cmd *cobra.Command, infisicalClient infisicalSdk.InfisicalClientInterface) (credential infisicalSdk.MachineIdentityCredential, e error) { - - identityId, err := util.GetCmdFlagOrEnv(cmd, "machine-identity-id", util.INFISICAL_MACHINE_IDENTITY_ID_NAME) - if err != nil { - return infisicalSdk.MachineIdentityCredential{}, err - } - - serviceAccountTokenPath, err := util.GetCmdFlagOrEnv(cmd, "service-account-token-path", util.INFISICAL_KUBERNETES_SERVICE_ACCOUNT_TOKEN_NAME) - if err != nil { - return infisicalSdk.MachineIdentityCredential{}, err - } - - return infisicalClient.Auth().KubernetesAuthLogin(identityId, serviceAccountTokenPath) -} - -func handleAzureAuthLogin(cmd *cobra.Command, infisicalClient infisicalSdk.InfisicalClientInterface) (credential infisicalSdk.MachineIdentityCredential, e error) { - - identityId, err := util.GetCmdFlagOrEnv(cmd, "machine-identity-id", util.INFISICAL_MACHINE_IDENTITY_ID_NAME) - if err != nil { - return infisicalSdk.MachineIdentityCredential{}, err - } - - return infisicalClient.Auth().AzureAuthLogin(identityId, "") -} - -func handleGcpIdTokenAuthLogin(cmd *cobra.Command, infisicalClient infisicalSdk.InfisicalClientInterface) (credential infisicalSdk.MachineIdentityCredential, e error) { - - identityId, err := util.GetCmdFlagOrEnv(cmd, "machine-identity-id", util.INFISICAL_MACHINE_IDENTITY_ID_NAME) - if err != nil { - return infisicalSdk.MachineIdentityCredential{}, err - } - - return infisicalClient.Auth().GcpIdTokenAuthLogin(identityId) -} - -func handleGcpIamAuthLogin(cmd *cobra.Command, infisicalClient infisicalSdk.InfisicalClientInterface) (credential infisicalSdk.MachineIdentityCredential, e error) { - - identityId, err := util.GetCmdFlagOrEnv(cmd, "machine-identity-id", util.INFISICAL_MACHINE_IDENTITY_ID_NAME) - if err != nil { - return infisicalSdk.MachineIdentityCredential{}, err - } - - serviceAccountKeyFilePath, err := util.GetCmdFlagOrEnv(cmd, "service-account-key-file-path", util.INFISICAL_GCP_IAM_SERVICE_ACCOUNT_KEY_FILE_PATH_NAME) - if err != nil { - return infisicalSdk.MachineIdentityCredential{}, err - } - - return infisicalClient.Auth().GcpIamAuthLogin(identityId, serviceAccountKeyFilePath) -} - -func handleAwsIamAuthLogin(cmd *cobra.Command, infisicalClient infisicalSdk.InfisicalClientInterface) (credential infisicalSdk.MachineIdentityCredential, e error) { - - identityId, err := util.GetCmdFlagOrEnv(cmd, "machine-identity-id", util.INFISICAL_MACHINE_IDENTITY_ID_NAME) - if err != nil { - return infisicalSdk.MachineIdentityCredential{}, err - } - - return infisicalClient.Auth().AwsIamAuthLogin(identityId) -} - -func handleOidcAuthLogin(cmd *cobra.Command, infisicalClient infisicalSdk.InfisicalClientInterface) (credential infisicalSdk.MachineIdentityCredential, e error) { - - identityId, err := util.GetCmdFlagOrEnv(cmd, "machine-identity-id", util.INFISICAL_MACHINE_IDENTITY_ID_NAME) - if err != nil { - return infisicalSdk.MachineIdentityCredential{}, err - } - - jwt, err := util.GetCmdFlagOrEnv(cmd, "oidc-jwt", util.INFISICAL_OIDC_AUTH_JWT_NAME) - if err != nil { - return infisicalSdk.MachineIdentityCredential{}, err - } - - return infisicalClient.Auth().OidcAuthLogin(identityId, jwt) -} - func formatAuthMethod(authMethod string) string { return strings.ReplaceAll(authMethod, "-", " ") } @@ -154,8 +63,22 @@ var loginCmd = &cobra.Command{ Use: "login", Short: "Login into your Infisical account", DisableFlagsInUseLine: true, - Run: func(cmd *cobra.Command, args []string) { + PreRunE: func(cmd *cobra.Command, args []string) error { + // daniel: oidc-jwt is deprecated in favor of `jwt`. we backfill the `jwt` flag with the value of `oidc-jwt` if it's set. + if cmd.Flags().Changed("oidc-jwt") && !cmd.Flags().Changed("jwt") { + oidcJWT, err := cmd.Flags().GetString("oidc-jwt") + if err != nil { + return err + } + err = cmd.Flags().Set("jwt", oidcJWT) + if err != nil { + return err + } + } + return nil + }, + Run: func(cmd *cobra.Command, args []string) { presetDomain := config.INFISICAL_URL clearSelfHostedDomains, err := cmd.Flags().GetBool("clear-domains") @@ -310,17 +233,20 @@ var loginCmd = &cobra.Command{ Telemetry.CaptureEvent("cli-command:login", posthog.NewProperties().Set("infisical-backend", config.INFISICAL_URL).Set("version", util.CLI_VERSION)) } else { - authStrategies := map[util.AuthStrategyType]func(cmd *cobra.Command, infisicalClient infisicalSdk.InfisicalClientInterface) (credential infisicalSdk.MachineIdentityCredential, e error){ - util.AuthStrategy.UNIVERSAL_AUTH: handleUniversalAuthLogin, - util.AuthStrategy.KUBERNETES_AUTH: handleKubernetesAuthLogin, - util.AuthStrategy.AZURE_AUTH: handleAzureAuthLogin, - util.AuthStrategy.GCP_ID_TOKEN_AUTH: handleGcpIdTokenAuthLogin, - util.AuthStrategy.GCP_IAM_AUTH: handleGcpIamAuthLogin, - util.AuthStrategy.AWS_IAM_AUTH: handleAwsIamAuthLogin, - util.AuthStrategy.OIDC_AUTH: handleOidcAuthLogin, + sdkAuthenticator := util.NewSdkAuthenticator(infisicalClient, cmd) + + authStrategies := map[util.AuthStrategyType]func() (credential infisicalSdk.MachineIdentityCredential, e error){ + util.AuthStrategy.UNIVERSAL_AUTH: sdkAuthenticator.HandleUniversalAuthLogin, + util.AuthStrategy.KUBERNETES_AUTH: sdkAuthenticator.HandleKubernetesAuthLogin, + util.AuthStrategy.AZURE_AUTH: sdkAuthenticator.HandleAzureAuthLogin, + util.AuthStrategy.GCP_ID_TOKEN_AUTH: sdkAuthenticator.HandleGcpIdTokenAuthLogin, + util.AuthStrategy.GCP_IAM_AUTH: sdkAuthenticator.HandleGcpIamAuthLogin, + util.AuthStrategy.AWS_IAM_AUTH: sdkAuthenticator.HandleAwsIamAuthLogin, + util.AuthStrategy.OIDC_AUTH: sdkAuthenticator.HandleOidcAuthLogin, + util.AuthStrategy.JWT_AUTH: sdkAuthenticator.HandleJwtAuthLogin, } - credential, err := authStrategies[strategy](cmd, infisicalClient) + credential, err := authStrategies[strategy]() if err != nil { euErrorMessage := "" @@ -518,14 +444,18 @@ func init() { rootCmd.AddCommand(loginCmd) loginCmd.Flags().Bool("clear-domains", false, "clear all self-hosting domains from the config file") loginCmd.Flags().BoolP("interactive", "i", false, "login via the command line") - loginCmd.Flags().String("method", "user", "login method [user, universal-auth]") loginCmd.Flags().Bool("plain", false, "only output the token without any formatting") + loginCmd.Flags().String("method", "user", "login method [user, universal-auth, kubernetes, azure, gcp-id-token, gcp-iam, aws-iam, oidc-auth]") loginCmd.Flags().String("client-id", "", "client id for universal auth") loginCmd.Flags().String("client-secret", "", "client secret for universal auth") loginCmd.Flags().String("machine-identity-id", "", "machine identity id for kubernetes, azure, gcp-id-token, gcp-iam, and aws-iam auth methods") loginCmd.Flags().String("service-account-token-path", "", "service account token path for kubernetes auth") loginCmd.Flags().String("service-account-key-file-path", "", "service account key file path for GCP IAM auth") - loginCmd.Flags().String("oidc-jwt", "", "JWT for OIDC authentication") + loginCmd.Flags().String("jwt", "", "jwt for jwt-based auth methods [oidc-auth, jwt-auth]") + loginCmd.Flags().String("oidc-jwt", "", "JWT for OIDC authentication. Deprecated, use --jwt instead") + + loginCmd.Flags().MarkDeprecated("oidc-jwt", "use --jwt instead") + } func DomainOverridePrompt() (bool, error) { diff --git a/cli/packages/gateway/connection.go b/cli/packages/gateway/connection.go index 58a0503ff..980137374 100644 --- a/cli/packages/gateway/connection.go +++ b/cli/packages/gateway/connection.go @@ -4,11 +4,19 @@ import ( "bufio" "bytes" "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" "errors" + "fmt" "io" "net" + "net/http" + "net/url" + "os" "strings" "sync" + "time" "github.com/quic-go/quic-go" "github.com/rs/zerolog/log" @@ -18,9 +26,13 @@ func handleConnection(ctx context.Context, quicConn quic.Connection) { log.Info().Msgf("New connection from: %s", quicConn.RemoteAddr().String()) // Use WaitGroup to track all streams var wg sync.WaitGroup + + contextWithTimeout, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + for { // Accept the first stream, which we'll use for commands - stream, err := quicConn.AcceptStream(ctx) + stream, err := quicConn.AcceptStream(contextWithTimeout) if err != nil { log.Printf("Failed to accept QUIC stream: %v", err) break @@ -44,7 +56,12 @@ func handleStream(stream quic.Stream, quicConn quic.Connection) { // Use buffered reader for better handling of fragmented data reader := bufio.NewReader(stream) - defer stream.Close() + defer func() { + log.Info().Msgf("Closing stream %d", streamID) + if stream != nil { + stream.Close() + } + }() for { msg, err := reader.ReadBytes('\n') @@ -89,6 +106,39 @@ func handleStream(stream quic.Stream, quicConn quic.Connection) { CopyDataFromQuicToTcp(stream, destTarget) log.Info().Msgf("Ending secure transmission between %s->%s", quicConn.LocalAddr().String(), destTarget.LocalAddr().String()) return + + case "FORWARD-HTTP": + targetURL := "" + argParts := bytes.Split(args, []byte(" ")) + + if len(argParts) == 0 || len(argParts[0]) == 0 { + log.Warn().Msg("FORWARD-HTTP used without a target URL.") + } else { + targetURL = string(argParts[0]) + if !isValidURL(targetURL) { + log.Error().Msgf("Invalid target URL: %s", targetURL) + return + } + } + + // Parse optional parameters + var caCertB64, verifyParam string + for _, part := range argParts[1:] { + partStr := string(part) + if strings.HasPrefix(partStr, "ca=") { + caCertB64 = strings.TrimPrefix(partStr, "ca=") + } else if strings.HasPrefix(partStr, "verify=") { + verifyParam = strings.TrimPrefix(partStr, "verify=") + } + } + + log.Info().Msgf("Starting HTTP proxy to: %s", targetURL) + + if err := handleHTTPProxy(stream, reader, targetURL, caCertB64, verifyParam); err != nil { + log.Error().Msgf("HTTP proxy error: %v", err) + } + return + case "PING": if _, err := stream.Write([]byte("PONG\n")); err != nil { log.Error().Msgf("Error writing PONG response: %v", err) @@ -100,11 +150,177 @@ func handleStream(stream quic.Stream, quicConn quic.Connection) { } } } +func handleHTTPProxy(stream quic.Stream, reader *bufio.Reader, targetURL string, caCertB64 string, verifyParam string) error { + transport := &http.Transport{ + DisableKeepAlives: false, + MaxIdleConns: 10, + IdleConnTimeout: 30 * time.Second, + } + + if strings.HasPrefix(targetURL, "https://") { + tlsConfig := &tls.Config{} + + if caCertB64 != "" { + caCert, err := base64.StdEncoding.DecodeString(caCertB64) + if err == nil { + caCertPool := x509.NewCertPool() + if caCertPool.AppendCertsFromPEM(caCert) { + tlsConfig.RootCAs = caCertPool + log.Info().Msg("Using provided CA certificate from gateway client") + } else { + log.Error().Msg("Failed to parse provided CA certificate") + } + } else { + log.Error().Msgf("Failed to decode CA certificate: %v", err) + } + } + + if verifyParam != "" { + tlsConfig.InsecureSkipVerify = verifyParam == "false" + log.Info().Msgf("TLS verification set to: %s", verifyParam) + } + + transport.TLSClientConfig = tlsConfig + } + + // Loop to handle multiple HTTP requests on the same stream + for { + req, err := http.ReadRequest(reader) + + if err != nil { + if errors.Is(err, io.EOF) { + log.Info().Msg("Client closed HTTP connection") + return nil + } + return fmt.Errorf("failed to read HTTP request: %v", err) + } + log.Info().Msgf("Received HTTP request: %s", req.URL.Path) + + actionHeader := HttpProxyAction(req.Header.Get(INFISICAL_HTTP_PROXY_ACTION_HEADER)) + if actionHeader != "" { + if actionHeader == HttpProxyActionInjectGatewayK8sServiceAccountToken { + token, err := os.ReadFile(KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH) + if err != nil { + stream.Write([]byte(buildHttpInternalServerError("failed to read k8s sa auth token"))) + continue // Continue to next request instead of returning + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", string(token))) + log.Info().Msgf("Injected gateway k8s SA auth token in request to %s", targetURL) + } else if actionHeader == HttpProxyActionUseGatewayK8sServiceAccount { // will work without a target URL set + // set the ca cert to the pod's k8s service account ca cert: + caCert, err := os.ReadFile(KUBERNETES_SERVICE_ACCOUNT_CA_CERT_PATH) + if err != nil { + stream.Write([]byte(buildHttpInternalServerError("failed to read k8s sa ca cert"))) + continue + } + + caCertPool := x509.NewCertPool() + if ok := caCertPool.AppendCertsFromPEM(caCert); !ok { + stream.Write([]byte(buildHttpInternalServerError("failed to parse k8s sa ca cert"))) + continue + } + + transport.TLSClientConfig = &tls.Config{ + RootCAs: caCertPool, + } + + // set authorization header to the pod's k8s service account token: + token, err := os.ReadFile(KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH) + if err != nil { + stream.Write([]byte(buildHttpInternalServerError("failed to read k8s sa auth token"))) + continue + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", string(token))) + + // update the target URL to point to the kubernetes API server: + kubernetesServiceHost := os.Getenv(KUBERNETES_SERVICE_HOST_ENV_NAME) + kubernetesServicePort := os.Getenv(KUBERNETES_SERVICE_PORT_HTTPS_ENV_NAME) + + fullBaseUrl := fmt.Sprintf("https://%s:%s", kubernetesServiceHost, kubernetesServicePort) + targetURL = fullBaseUrl + + log.Info().Msgf("Redirected request to Kubernetes API server: %s", targetURL) + } + + req.Header.Del(INFISICAL_HTTP_PROXY_ACTION_HEADER) + } + + // Build full target URL + var targetFullURL string + if strings.HasPrefix(targetURL, "http://") || strings.HasPrefix(targetURL, "https://") { + baseURL := strings.TrimSuffix(targetURL, "/") + targetFullURL = baseURL + req.URL.Path + if req.URL.RawQuery != "" { + targetFullURL += "?" + req.URL.RawQuery + } + } else { + baseURL := strings.TrimSuffix("http://"+targetURL, "/") + targetFullURL = baseURL + req.URL.Path + if req.URL.RawQuery != "" { + targetFullURL += "?" + req.URL.RawQuery + } + } + + // create the request to the target + proxyReq, err := http.NewRequest(req.Method, targetFullURL, req.Body) + if err != nil { + log.Error().Msgf("Failed to create proxy request: %v", err) + stream.Write([]byte(buildHttpInternalServerError("failed to create proxy request"))) + continue // Continue to next request + } + proxyReq.Header = req.Header.Clone() + + log.Info().Msgf("Proxying %s %s to %s", req.Method, req.URL.Path, targetFullURL) + + client := &http.Client{ + Transport: transport, + Timeout: 30 * time.Second, + } + + resp, err := client.Do(proxyReq) + if err != nil { + log.Error().Msgf("Failed to reach target: %v", err) + stream.Write([]byte(buildHttpInternalServerError(fmt.Sprintf("failed to reach target due to networking error: %s", err.Error())))) + continue // Continue to next request + } + + // Write the entire response (status line, headers, body) to the stream + // http.Response.Write handles this for "Connection: close" correctly. + // For other connection tokens, manual removal might be needed if they cause issues with QUIC. + // For a simple proxy, this is generally sufficient. + resp.Header.Del("Connection") // Good practice for proxies + + log.Info().Msgf("Writing response to stream: %s", resp.Status) + + if err := resp.Write(stream); err != nil { + log.Error().Err(err).Msg("Failed to write response to stream") + resp.Body.Close() + return fmt.Errorf("failed to write response to stream: %w", err) + } + + resp.Body.Close() + + // Check if client wants to close connection + if req.Header.Get("Connection") == "close" { + log.Info().Msg("Client requested connection close") + return nil + } + } +} + +func buildHttpInternalServerError(message string) string { + return fmt.Sprintf("HTTP/1.1 500 Internal Server Error\r\nContent-Type: application/json\r\n\r\n{\"message\": \"gateway: %s\"}", message) +} type CloseWrite interface { CloseWrite() error } +func isValidURL(str string) bool { + u, err := url.Parse(str) + return err == nil && u.Scheme != "" && u.Host != "" +} + func CopyDataFromQuicToTcp(quicStream quic.Stream, tcpConn net.Conn) { // Create a WaitGroup to wait for both copy operations var wg sync.WaitGroup diff --git a/cli/packages/gateway/constants.go b/cli/packages/gateway/constants.go new file mode 100644 index 000000000..aa260ed2e --- /dev/null +++ b/cli/packages/gateway/constants.go @@ -0,0 +1,17 @@ +package gateway + +const ( + KUBERNETES_SERVICE_HOST_ENV_NAME = "KUBERNETES_SERVICE_HOST" + KUBERNETES_SERVICE_PORT_HTTPS_ENV_NAME = "KUBERNETES_SERVICE_PORT_HTTPS" + KUBERNETES_SERVICE_ACCOUNT_CA_CERT_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + INFISICAL_HTTP_PROXY_ACTION_HEADER = "x-infisical-action" +) + +type HttpProxyAction string + +const ( + HttpProxyActionInjectGatewayK8sServiceAccountToken HttpProxyAction = "inject-k8s-sa-auth-token" + HttpProxyActionUseGatewayK8sServiceAccount HttpProxyAction = "use-k8s-sa" +) diff --git a/cli/packages/gateway/gateway.go b/cli/packages/gateway/gateway.go index d0a25ca9c..eb0c72d5d 100644 --- a/cli/packages/gateway/gateway.go +++ b/cli/packages/gateway/gateway.go @@ -54,6 +54,10 @@ func NewGateway(identityToken string) (Gateway, error) { }, nil } +func (g *Gateway) UpdateIdentityAccessToken(accessToken string) { + g.httpClient.SetAuthToken(accessToken) +} + func (g *Gateway) ConnectWithRelay() error { relayDetails, err := api.CallRegisterGatewayIdentityV1(g.httpClient) if err != nil { diff --git a/cli/packages/util/auth.go b/cli/packages/util/auth.go index b54bde45b..eaf7cecc1 100644 --- a/cli/packages/util/auth.go +++ b/cli/packages/util/auth.go @@ -5,7 +5,9 @@ import ( "os" "os/exec" + infisicalSdk "github.com/infisical/go-sdk" "github.com/rs/zerolog/log" + "github.com/spf13/cobra" ) type AuthStrategyType string @@ -18,6 +20,7 @@ var AuthStrategy = struct { GCP_IAM_AUTH AuthStrategyType AWS_IAM_AUTH AuthStrategyType OIDC_AUTH AuthStrategyType + JWT_AUTH AuthStrategyType }{ UNIVERSAL_AUTH: "universal-auth", KUBERNETES_AUTH: "kubernetes", @@ -26,6 +29,7 @@ var AuthStrategy = struct { GCP_IAM_AUTH: "gcp-iam", AWS_IAM_AUTH: "aws-iam", OIDC_AUTH: "oidc-auth", + JWT_AUTH: "jwt-auth", } var AVAILABLE_AUTH_STRATEGIES = []AuthStrategyType{ @@ -36,6 +40,7 @@ var AVAILABLE_AUTH_STRATEGIES = []AuthStrategyType{ AuthStrategy.GCP_IAM_AUTH, AuthStrategy.AWS_IAM_AUTH, AuthStrategy.OIDC_AUTH, + AuthStrategy.JWT_AUTH, } func IsAuthMethodValid(authMethod string, allowUserAuth bool) (isValid bool, strategy AuthStrategyType) { @@ -84,3 +89,120 @@ func EstablishUserLoginSession() LoggedInUserDetails { return loggedInUserDetails } + +type SdkAuthenticator struct { + infisicalClient infisicalSdk.InfisicalClientInterface + cmd *cobra.Command +} + +func NewSdkAuthenticator(infisicalClient infisicalSdk.InfisicalClientInterface, cmd *cobra.Command) *SdkAuthenticator { + return &SdkAuthenticator{ + infisicalClient: infisicalClient, + cmd: cmd, + } +} +func (a *SdkAuthenticator) HandleUniversalAuthLogin() (credential infisicalSdk.MachineIdentityCredential, e error) { + + clientId, err := GetCmdFlagOrEnv(a.cmd, "client-id", []string{INFISICAL_UNIVERSAL_AUTH_CLIENT_ID_NAME}) + + if err != nil { + return infisicalSdk.MachineIdentityCredential{}, err + } + + clientSecret, err := GetCmdFlagOrEnv(a.cmd, "client-secret", []string{INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET_NAME}) + if err != nil { + return infisicalSdk.MachineIdentityCredential{}, err + } + + return a.infisicalClient.Auth().UniversalAuthLogin(clientId, clientSecret) +} + +func (a *SdkAuthenticator) HandleJwtAuthLogin() (credential infisicalSdk.MachineIdentityCredential, e error) { + + identityId, err := GetCmdFlagOrEnv(a.cmd, "machine-identity-id", []string{INFISICAL_MACHINE_IDENTITY_ID_NAME}) + if err != nil { + return infisicalSdk.MachineIdentityCredential{}, err + } + + jwt, err := GetCmdFlagOrEnv(a.cmd, "jwt", []string{INFISICAL_JWT_NAME}) + if err != nil { + return infisicalSdk.MachineIdentityCredential{}, err + } + + return a.infisicalClient.Auth().JwtAuthLogin(identityId, jwt) +} + +func (a *SdkAuthenticator) HandleKubernetesAuthLogin() (credential infisicalSdk.MachineIdentityCredential, e error) { + + identityId, err := GetCmdFlagOrEnv(a.cmd, "machine-identity-id", []string{INFISICAL_MACHINE_IDENTITY_ID_NAME}) + if err != nil { + return infisicalSdk.MachineIdentityCredential{}, err + } + + serviceAccountTokenPath, err := GetCmdFlagOrEnv(a.cmd, "service-account-token-path", []string{INFISICAL_KUBERNETES_SERVICE_ACCOUNT_TOKEN_NAME}) + if err != nil { + return infisicalSdk.MachineIdentityCredential{}, err + } + + return a.infisicalClient.Auth().KubernetesAuthLogin(identityId, serviceAccountTokenPath) +} + +func (a *SdkAuthenticator) HandleAzureAuthLogin() (credential infisicalSdk.MachineIdentityCredential, e error) { + + identityId, err := GetCmdFlagOrEnv(a.cmd, "machine-identity-id", []string{INFISICAL_MACHINE_IDENTITY_ID_NAME}) + if err != nil { + return infisicalSdk.MachineIdentityCredential{}, err + } + + return a.infisicalClient.Auth().AzureAuthLogin(identityId, "") +} + +func (a *SdkAuthenticator) HandleGcpIdTokenAuthLogin() (credential infisicalSdk.MachineIdentityCredential, e error) { + + identityId, err := GetCmdFlagOrEnv(a.cmd, "machine-identity-id", []string{INFISICAL_MACHINE_IDENTITY_ID_NAME}) + if err != nil { + return infisicalSdk.MachineIdentityCredential{}, err + } + + return a.infisicalClient.Auth().GcpIdTokenAuthLogin(identityId) +} + +func (a *SdkAuthenticator) HandleGcpIamAuthLogin() (credential infisicalSdk.MachineIdentityCredential, e error) { + + identityId, err := GetCmdFlagOrEnv(a.cmd, "machine-identity-id", []string{INFISICAL_MACHINE_IDENTITY_ID_NAME}) + if err != nil { + return infisicalSdk.MachineIdentityCredential{}, err + } + + serviceAccountKeyFilePath, err := GetCmdFlagOrEnv(a.cmd, "service-account-key-file-path", []string{INFISICAL_GCP_IAM_SERVICE_ACCOUNT_KEY_FILE_PATH_NAME}) + if err != nil { + return infisicalSdk.MachineIdentityCredential{}, err + } + + return a.infisicalClient.Auth().GcpIamAuthLogin(identityId, serviceAccountKeyFilePath) +} + +func (a *SdkAuthenticator) HandleAwsIamAuthLogin() (credential infisicalSdk.MachineIdentityCredential, e error) { + + identityId, err := GetCmdFlagOrEnv(a.cmd, "machine-identity-id", []string{INFISICAL_MACHINE_IDENTITY_ID_NAME}) + if err != nil { + return infisicalSdk.MachineIdentityCredential{}, err + } + + return a.infisicalClient.Auth().AwsIamAuthLogin(identityId) +} + +func (a *SdkAuthenticator) HandleOidcAuthLogin() (credential infisicalSdk.MachineIdentityCredential, e error) { + + identityId, err := GetCmdFlagOrEnv(a.cmd, "machine-identity-id", []string{INFISICAL_MACHINE_IDENTITY_ID_NAME}) + if err != nil { + return infisicalSdk.MachineIdentityCredential{}, err + } + + jwt, err := GetCmdFlagOrEnv(a.cmd, "jwt", []string{INFISICAL_JWT_NAME, INFISICAL_OIDC_AUTH_JWT_NAME}) + if err != nil { + return infisicalSdk.MachineIdentityCredential{}, err + } + + return a.infisicalClient.Auth().OidcAuthLogin(identityId, jwt) +} diff --git a/cli/packages/util/constants.go b/cli/packages/util/constants.go index 8b4c586e6..126e5a5d0 100644 --- a/cli/packages/util/constants.go +++ b/cli/packages/util/constants.go @@ -13,6 +13,8 @@ const ( VAULT_BACKEND_AUTO_MODE = "auto" VAULT_BACKEND_FILE_MODE = "file" + INFISICAL_AUTH_METHOD_NAME = "INFISICAL_AUTH_METHOD" + // Universal Auth INFISICAL_UNIVERSAL_AUTH_CLIENT_ID_NAME = "INFISICAL_UNIVERSAL_AUTH_CLIENT_ID" INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET_NAME = "INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET" @@ -24,7 +26,12 @@ const ( INFISICAL_GCP_IAM_SERVICE_ACCOUNT_KEY_FILE_PATH_NAME = "INFISICAL_GCP_IAM_SERVICE_ACCOUNT_KEY_FILE_PATH" // OIDC Auth - INFISICAL_OIDC_AUTH_JWT_NAME = "INFISICAL_OIDC_AUTH_JWT" + INFISICAL_OIDC_AUTH_JWT_NAME = "INFISICAL_OIDC_AUTH_JWT" // deprecated in favor of INFISICAL_JWT + + // JWT AUTH + INFISICAL_JWT_NAME = "INFISICAL_JWT" + + INFISICAL_GATEWAY_TOKEN_NAME_LEGACY = "TOKEN" // backwards compatibility with gateway helm chart, where token was the only supported auth method // Generic env variable used for auth methods that require a machine identity ID INFISICAL_MACHINE_IDENTITY_ID_NAME = "INFISICAL_MACHINE_IDENTITY_ID" diff --git a/cli/packages/util/helper.go b/cli/packages/util/helper.go index 153a5a281..abd9768aa 100644 --- a/cli/packages/util/helper.go +++ b/cli/packages/util/helper.go @@ -96,6 +96,11 @@ func GetInfisicalToken(cmd *cobra.Command) (token *models.TokenDetails, err erro infisicalToken = os.Getenv(INFISICAL_TOKEN_NAME) source = fmt.Sprintf("%s environment variable", INFISICAL_TOKEN_NAME) } + + if infisicalToken == "" { // if its still empty, check for the `TOKEN` environment variable (for gateway helm) + infisicalToken = os.Getenv(INFISICAL_GATEWAY_TOKEN_NAME_LEGACY) + source = fmt.Sprintf("%s environment variable", INFISICAL_GATEWAY_TOKEN_NAME_LEGACY) + } } if infisicalToken == "" { // If it's empty, we return nothing at all. @@ -292,13 +297,18 @@ func GetEnvVarOrFileContent(envName string, filePath string) (string, error) { return fileContent, nil } -func GetCmdFlagOrEnv(cmd *cobra.Command, flag, envName string) (string, error) { +func GetCmdFlagOrEnv(cmd *cobra.Command, flag string, envNames []string) (string, error) { value, flagsErr := cmd.Flags().GetString(flag) if flagsErr != nil { return "", flagsErr } if value == "" { - value = os.Getenv(envName) + for _, env := range envNames { + value = strings.TrimSpace(os.Getenv(env)) + if value != "" { + break + } + } } if value == "" { return "", fmt.Errorf("please provide %s flag", flag) diff --git a/docs/api-reference/endpoints/dynamic-secrets/kubernetes/create-lease.mdx b/docs/api-reference/endpoints/dynamic-secrets/kubernetes/create-lease.mdx new file mode 100644 index 000000000..1a4e26709 --- /dev/null +++ b/docs/api-reference/endpoints/dynamic-secrets/kubernetes/create-lease.mdx @@ -0,0 +1,4 @@ +--- +title: "Create Kubernetes Lease" +openapi: "POST /api/v1/dynamic-secrets/leases/kubernetes" +--- diff --git a/docs/cli/commands/dynamic-secrets.mdx b/docs/cli/commands/dynamic-secrets.mdx index c345c3e2d..5db90b564 100644 --- a/docs/cli/commands/dynamic-secrets.mdx +++ b/docs/cli/commands/dynamic-secrets.mdx @@ -148,6 +148,22 @@ infisical dynamic-secrets lease create --ttl= +### Provider-specific flags + +The following flags are specific to certain providers or integrations: + + + + The namespace to create the lease in. Only used for Kubernetes dynamic secrets. + + ```bash + # Example + infisical dynamic-secrets lease create --kubernetesNamespace= + ``` + + + + This command is used to list leases for a dynamic secret. diff --git a/docs/cli/commands/gateway.mdx b/docs/cli/commands/gateway.mdx index fd035f1fd..a12493c58 100644 --- a/docs/cli/commands/gateway.mdx +++ b/docs/cli/commands/gateway.mdx @@ -26,28 +26,215 @@ Run the Infisical gateway in the foreground or manage its systemd service instal Run the Infisical gateway in the foreground. The gateway will connect to the relay service and maintain a persistent connection. ```bash - infisical gateway --token= --domain= + infisical gateway --domain= --auth-method= ``` - ### Flags + ### Authentication - - The machine identity access token to authenticate with Infisical. + The Infisical CLI supports multiple authentication methods. Below are the available authentication methods, with their respective flags. + + + + The Universal Auth method is a simple and secure way to authenticate with Infisical. It requires a client ID and a client secret to authenticate with Infisical. + + + + + Your machine identity client ID. + + + Your machine identity client secret. + + + The authentication method to use. Must be `universal-auth` when using Universal Auth. + + + ```bash - # Example - infisical gateway --token= + infisical gateway --auth-method=universal-auth --client-id= --client-secret= ``` - You may also expose the token to the CLI by setting the environment variable `INFISICAL_TOKEN` before executing the gateway command. + + The Native Kubernetes method is used to authenticate with Infisical when running in a Kubernetes environment. It requires a service account token to authenticate with Infisical. + + + + + Your machine identity ID. + + + Path to the Kubernetes service account token to use. Default: `/var/run/secrets/kubernetes.io/serviceaccount/token`. + + + The authentication method to use. Must be `kubernetes` when using Native Kubernetes. + + + + + + + ```bash + infisical gateway --auth-method=kubernetes --machine-identity-id= + ``` + + + + The Native Azure method is used to authenticate with Infisical when running in an Azure environment. + + + + + Your machine identity ID. + + + The authentication method to use. Must be `azure` when using Native Azure. + + + + + + + ```bash + infisical gateway --auth-method=azure --machine-identity-id= + ``` + + + + The Native GCP ID Token method is used to authenticate with Infisical when running in a GCP environment. + + + + + Your machine identity ID. + + + The authentication method to use. Must be `gcp-id-token` when using Native GCP ID Token. + + + + + + + ```bash + infisical gateway --auth-method=gcp-id-token --machine-identity-id= + ``` + + + + + The GCP IAM method is used to authenticate with Infisical with a GCP service account key. + + + + + Your machine identity ID. + + + Path to your GCP service account key file _(Must be in JSON format!)_ + + + The authentication method to use. Must be `gcp-iam` when using GCP IAM. + + + + + ```bash + infisical gateway --auth-method=gcp-iam --machine-identity-id= --service-account-key-file-path= + ``` + + + + The AWS IAM method is used to authenticate with Infisical with an AWS IAM role while running in an AWS environment like EC2, Lambda, etc. + + + + + Your machine identity ID. + + + The authentication method to use. Must be `aws-iam` when using Native AWS IAM. + + + + + ```bash + infisical gateway --auth-method=aws-iam --machine-identity-id= + ``` + + + + + The OIDC Auth method is used to authenticate with Infisical via identity tokens with OIDC. + + + + + Your machine identity ID. + + + The OIDC JWT from the identity provider. + + + The authentication method to use. Must be `oidc-auth` when using OIDC Auth. + + + + + ```bash + infisical gateway --auth-method=oidc-auth --machine-identity-id= --jwt= + ``` + + + + The JWT Auth method is used to authenticate with Infisical via a JWT token. + + + + + The JWT token to use for authentication. + + + Your machine identity ID. + + + The authentication method to use. Must be `jwt-auth` when using JWT Auth. + + + + + + ```bash + infisical gateway --auth-method=jwt-auth --jwt= --machine-identity-id= + ``` + + + You can use the `INFISICAL_TOKEN` environment variable to authenticate with Infisical with a raw machine identity access token. + + + + + The machine identity access token to use for authentication. + + + + + ```bash + infisical gateway --token= + ``` + + + + + ### Other Flags Domain of your self-hosted Infisical instance. ```bash # Example - sudo infisical gateway install --domain=https://app.your-domain.com + infisical gateway --domain=https://app.your-domain.com ``` diff --git a/docs/cli/commands/login.mdx b/docs/cli/commands/login.mdx index f493ff5d2..f93e3b4b2 100644 --- a/docs/cli/commands/login.mdx +++ b/docs/cli/commands/login.mdx @@ -190,7 +190,7 @@ The Infisical CLI supports multiple authentication methods. Below are the availa - + The OIDC Auth method is used to authenticate with Infisical via identity tokens with OIDC. @@ -198,7 +198,7 @@ The Infisical CLI supports multiple authentication methods. Below are the availa Your machine identity ID. - + The OIDC JWT from the identity provider. @@ -212,11 +212,35 @@ The Infisical CLI supports multiple authentication methods. Below are the availa Run the `login` command with the following flags to obtain an access token: ```bash - infisical login --method=oidc-auth --machine-identity-id= --oidc-jwt= + infisical login --method=oidc-auth --machine-identity-id= --jwt= ``` + + + The JWT Auth method is used to authenticate with Infisical via a JWT token. + + + + + The JWT token to use for authentication. + + + Your machine identity ID. + + + + + + + Run the `login` command with the following flags to obtain an access token: + + ```bash + infisical login --method=jwt-auth --jwt= --machine-identity-id= + ``` + + diff --git a/docs/cli/usage.mdx b/docs/cli/usage.mdx index a3bfd83f0..8d0236ed5 100644 --- a/docs/cli/usage.mdx +++ b/docs/cli/usage.mdx @@ -120,6 +120,12 @@ The CLI is designed for a variety of secret management applications ranging from + + Starting with CLI version v0.4.0, you can now choose to log in via Infisical Cloud (US/EU) or your own self-hosted instance by simply running `infisical login` and following the on-screen instructions — no need to manually set the `INFISICAL_API_URL` environment variable. + + For versions prior to v0.4.0, the CLI defaults to the US Cloud. To connect to the EU Cloud or a self-hosted instance, set the `INFISICAL_API_URL` environment variable to `https://eu.infisical.com` or your custom URL. + + ## Custom Request Headers diff --git a/docs/documentation/platform/access-controls/abac/managing-machine-identity-attributes.mdx b/docs/documentation/platform/access-controls/abac/managing-machine-identity-attributes.mdx index 5e1cc7093..b953a80bf 100644 --- a/docs/documentation/platform/access-controls/abac/managing-machine-identity-attributes.mdx +++ b/docs/documentation/platform/access-controls/abac/managing-machine-identity-attributes.mdx @@ -1,5 +1,5 @@ --- -title: "Machine identities" +title: "Machine identities" description: "Learn how to set metadata and leverage authentication attributes for machine identities." --- @@ -25,7 +25,7 @@ Machine identities can have metadata set manually, just like users. In addition, #### Accessing Attributes From Machine Identity Login -When machine identities authenticate, they may receive additional payloads/attributes from the service provider. +When machine identities authenticate, they may receive additional payloads/attributes from the service provider. For methods like OIDC, these come as claims in the token and can be made available in your policies. @@ -50,17 +50,29 @@ For methods like OIDC, these come as claims in the token and can be made availab ``` You might map: - - - **department:** to `user.department` + + - **department:** to `user.department` - **role:** to `user.role` Once configured, these attributes become available in your policies using the following format: - + ``` {{ identity.auth.oidc.claims. }} ``` + + + + For identities authenticated using Kubernetes, the service account's namespace and name are available in their policy and can be accessed as follows: + + ``` + {{ identity.auth.kubernetes.namespace }} + {{ identity.auth.kubernetes.name }} + ``` + + + At the moment we only support OIDC claims. Payloads on other authentication methods are not yet accessible. diff --git a/docs/documentation/platform/audit-log-streams/audit-log-streams.mdx b/docs/documentation/platform/audit-log-streams/audit-log-streams.mdx index 2f38a3930..2ee4dff92 100644 --- a/docs/documentation/platform/audit-log-streams/audit-log-streams.mdx +++ b/docs/documentation/platform/audit-log-streams/audit-log-streams.mdx @@ -5,12 +5,12 @@ description: "Learn how to stream Infisical Audit Logs to external logging provi Audit log streams is a paid feature. - + If you're using Infisical Cloud, then it is available under the **Enterprise Tier**. If you're self-hosting Infisical, then you should contact team@infisical.com to purchase an enterprise license to use it. -Infisical Audit Log Streaming enables you to transmit your organization's Audit Logs to external logging providers for monitoring and analysis. +Infisical Audit Log Streaming enables you to transmit your organization's Audit Logs to external logging providers for monitoring and analysis. The logs are formatted in JSON, requiring your logging provider to support JSON-based log parsing. @@ -118,7 +118,7 @@ Each log entry sent to the external logging provider will follow the same struct ### Audit Logs Structure - The unique identifier for the log entry. + The unique identifier for the log entry. @@ -168,7 +168,7 @@ Each log entry sent to the external logging provider will follow the same struct If the `actor` field is set to `platform`, `scimClient`, or `unknownUser`, the `actorMetadata` field will be an empty object. - + @@ -178,7 +178,7 @@ Each log entry sent to the external logging provider will follow the same struct The type of event that occurred. Below you can see a list of possible event types. More event types will be added in the future as we expand our audit logs further. - `get-secrets`, `delete-secrets`, `get-secret`, `create-secret`, `update-secret`, `delete-secret`, `get-workspace-key`, `authorize-integration`, `update-integration-auth`, `unauthorize-integration`, `create-integration`, `delete-integration`, `add-trusted-ip`, `update-trusted-ip`, `delete-trusted-ip`, `create-service-token`, `delete-service-token`, `create-identity`, `update-identity`, `delete-identity`, `login-identity-universal-auth`, `add-identity-universal-auth`, `update-identity-universal-auth`, `get-identity-universal-auth`, `create-identity-universal-auth-client-secret`, `revoke-identity-universal-auth-client-secret`, `get-identity-universal-auth-client-secret`, `create-environment`, `update-environment`, `delete-environment`, `add-workspace-member`, `remove-workspace-member`, `create-folder`, `update-folder`, `delete-folder`, `create-webhook`, `update-webhook-status`, `delete-webhook`, `get-secret-imports`, `create-secret-import`, `update-secret-import`, `delete-secret-import`, `update-user-workspace-role`, `update-user-workspace-denied-permissions`, `create-certificate-authority`, `get-certificate-authority`, `update-certificate-authority`, `delete-certificate-authority`, `get-certificate-authority-csr`, `get-certificate-authority-cert`, `sign-intermediate`, `import-certificate-authority-cert`, `get-certificate-authority-crl`, `issue-cert`, `get-cert`, `delete-cert`, `revoke-cert`, `get-cert-body`, `create-pki-alert`, `get-pki-alert`, `update-pki-alert`, `delete-pki-alert`, `create-pki-collection`, `get-pki-collection`, `update-pki-collection`, `delete-pki-collection`, `get-pki-collection-items`, `add-pki-collection-item`, `delete-pki-collection-item`, `org-admin-accessed-project`, `create-certificate-template`, `update-certificate-template`, `delete-certificate-template`, `get-certificate-template`, `create-certificate-template-est-config`, `update-certificate-template-est-config`, `get-certificate-template-est-config`, `update-project-slack-config`, `get-project-slack-config`, `integration-synced`, `create-shared-secret`, `delete-shared-secret`, `read-shared-secret`. + `get-secrets`, `delete-secrets`, `get-secret`, `create-secret`, `update-secret`, `delete-secret`, `get-workspace-key`, `authorize-integration`, `update-integration-auth`, `unauthorize-integration`, `create-integration`, `delete-integration`, `add-trusted-ip`, `update-trusted-ip`, `delete-trusted-ip`, `create-service-token`, `delete-service-token`, `create-identity`, `update-identity`, `delete-identity`, `login-identity-universal-auth`, `add-identity-universal-auth`, `update-identity-universal-auth`, `get-identity-universal-auth`, `create-identity-universal-auth-client-secret`, `revoke-identity-universal-auth-client-secret`, `get-identity-universal-auth-client-secret`, `create-environment`, `update-environment`, `delete-environment`, `add-workspace-member`, `remove-workspace-member`, `create-folder`, `update-folder`, `delete-folder`, `create-webhook`, `update-webhook-status`, `delete-webhook`, `webhook-triggered`, `get-secret-imports`, `create-secret-import`, `update-secret-import`, `delete-secret-import`, `update-user-workspace-role`, `update-user-workspace-denied-permissions`, `create-certificate-authority`, `get-certificate-authority`, `update-certificate-authority`, `delete-certificate-authority`, `get-certificate-authority-csr`, `get-certificate-authority-cert`, `sign-intermediate`, `import-certificate-authority-cert`, `get-certificate-authority-crl`, `issue-cert`, `get-cert`, `delete-cert`, `revoke-cert`, `get-cert-body`, `create-pki-alert`, `get-pki-alert`, `update-pki-alert`, `delete-pki-alert`, `create-pki-collection`, `get-pki-collection`, `update-pki-collection`, `delete-pki-collection`, `get-pki-collection-items`, `add-pki-collection-item`, `delete-pki-collection-item`, `org-admin-accessed-project`, `create-certificate-template`, `update-certificate-template`, `delete-certificate-template`, `get-certificate-template`, `create-certificate-template-est-config`, `update-certificate-template-est-config`, `get-certificate-template-est-config`, `update-project-slack-config`, `get-project-slack-config`, `integration-synced`, `create-shared-secret`, `delete-shared-secret`, `read-shared-secret`. @@ -219,4 +219,4 @@ Each log entry sent to the external logging provider will follow the same struct The name of the project where the event occurred. The `projectName` field will only be present if the event occurred at the project level, not the organization level. - \ No newline at end of file + diff --git a/docs/documentation/platform/dynamic-secrets/aws-elasticache.mdx b/docs/documentation/platform/dynamic-secrets/aws-elasticache.mdx index 66c4c706a..6c70612c8 100644 --- a/docs/documentation/platform/dynamic-secrets/aws-elasticache.mdx +++ b/docs/documentation/platform/dynamic-secrets/aws-elasticache.mdx @@ -95,12 +95,28 @@ The Infisical AWS ElastiCache dynamic secret allows you to generate AWS ElastiCa ![Modify ElastiCache Statements Modal](/images/platform/dynamic-secrets/modify-elasticache-statement.png) - - Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. + + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. - Allowed template variables are - - `{{randomUsername}}`: Random username string - - `{{unixTimestamp}}`: Current Unix timestamp + Allowed template variables are + - `{{randomUsername}}`: Random username string + - `{{unixTimestamp}}`: Current Unix timestamp + - `{{identity.name}}`: Name of the identity that is generating the secret + - `{{random N}}`: Random string of N characters + + Allowed template functions are + - `truncate`: Truncates a string to a specified length + - `replace`: Replaces a substring with another value + + Examples: + ``` + {{randomUsername}} // 3POnzeFyK9gW2nioK0q2gMjr6CZqsRiX + {{unixTimestamp}} // 17490641580 + {{identity.name}} // testuser + {{random-5}} // x9k2m + {{truncate identity.name 4}} // test + {{replace identity.name 'user' 'replace'}} // testreplace + ``` If you want to provide specific privileges for the generated dynamic credentials, you can modify the ElastiCache statement to your needs. This is useful if you want to only give access to a specific resource. diff --git a/docs/documentation/platform/dynamic-secrets/aws-iam.mdx b/docs/documentation/platform/dynamic-secrets/aws-iam.mdx index 56a10419c..a02d80a5c 100644 --- a/docs/documentation/platform/dynamic-secrets/aws-iam.mdx +++ b/docs/documentation/platform/dynamic-secrets/aws-iam.mdx @@ -32,7 +32,8 @@ Infisical needs an initial AWS IAM user with the required permissions to create "iam:ListUserPolicies", "iam:PutUserPolicy", "iam:AddUserToGroup", - "iam:RemoveUserFromGroup" + "iam:RemoveUserFromGroup", + "iam:TagUser" ], "Resource": ["*"] } @@ -50,110 +51,304 @@ Replace **\** with your AWS account id and **\** w ## Set up Dynamic Secrets with AWS IAM - - - Navigate to the Secret Overview dashboard and select the environment in which you would like to add a dynamic secret to. - - - ![Add Dynamic Secret Button](../../../images/platform/dynamic-secrets/add-dynamic-secret-button.png) - - - ![Dynamic Secret Modal](../../../images/platform/dynamic-secrets/dynamic-secret-modal-aws-iam.png) - - - - Name by which you want the secret to be referenced - + + + Infisical will assume the provided role in your AWS account securely, without the need to share any credentials. + + To connect your self-hosted Infisical instance with AWS, you need to set up an AWS IAM User account that can assume the configured AWS IAM Role. - - Default time-to-live for a generated secret (it is possible to modify this value after a secret is generated) - + If your instance is deployed on AWS, the aws-sdk will automatically retrieve the credentials. Ensure that you assign the provided permission policy to your deployed instance, such as ECS or EC2. - - Maximum time-to-live for a generated secret - + The following steps are for instances not deployed on AWS: + + + Navigate to [Create IAM User](https://console.aws.amazon.com/iamv2/home#/users/create) in your AWS Console. + + + Attach the following inline permission policy to the IAM User to allow it to assume any IAM Roles: + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowAssumeAnyRole", + "Effect": "Allow", + "Action": "sts:AssumeRole", + "Resource": "arn:aws:iam::*:role/*" + } + ] + } + ``` + + + Obtain the AWS access key ID and secret access key for your IAM User by navigating to **IAM > Users > [Your User] > Security credentials > Access keys**. - - The managing AWS IAM User Access Key - + ![Access Key Step 1](/images/integrations/aws/integrations-aws-access-key-1.png) + ![Access Key Step 2](/images/integrations/aws/integrations-aws-access-key-2.png) + ![Access Key Step 3](/images/integrations/aws/integrations-aws-access-key-3.png) + + + 1. Set the access key as **DYNAMIC_SECRET_AWS_ACCESS_KEY_ID**. + 2. Set the secret key as **DYNAMIC_SECRET_AWS_SECRET_ACCESS_KEY**. + + + - - The managing AWS IAM User Secret Key - + + + 1. Navigate to the [Create IAM Role](https://console.aws.amazon.com/iamv2/home#/roles/create?step=selectEntities) page in your AWS Console. + ![IAM Role Creation](/images/integrations/aws/integration-aws-iam-assume-role.png) - - [IAM AWS Path](https://aws.amazon.com/blogs/security/optimize-aws-administration-with-iam-paths/) to scope created IAM User resource access. - + 2. Select **AWS Account** as the **Trusted Entity Type**. + 3. Select **Another AWS Account** and provide the appropriate Infisical AWS Account ID: use **381492033652** for the **US region**, and **345594589636** for the **EU region**. This restricts the role to be assumed only by Infisical. If self-hosting, provide your AWS account number instead. + 4. (Recommended) Enable "Require external ID" and input your **Project ID** to strengthen security and mitigate the [confused deputy problem](https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html). + 5. Assign permission as shared in prerequisite. - - The AWS data center region. - + + When configuring an IAM Role that Infisical will assume, it’s highly recommended to enable the **"Require external ID"** option and specify your **Project ID**. - - The IAM Policy ARN of the [AWS Permissions Boundary](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_boundaries.html) to attach to IAM users created in the role. - + This precaution helps protect your AWS account against the [confused deputy problem](https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html), a potential security vulnerability where Infisical could be tricked into performing actions on your behalf by an unauthorized actor. - - The AWS IAM groups that should be assigned to the created users. Multiple values can be provided by separating them with commas - + Always enable "Require external ID" and use your Project ID when setting up the IAM Role. + + + + ![Copy IAM Role ARN](/images/integrations/aws/integration-aws-iam-assume-arn.png) + + + Navigate to the Secret Overview dashboard and select the environment in which you would like to add a dynamic secret to. + + + ![Add Dynamic Secret Button](../../../images/platform/dynamic-secrets/add-dynamic-secret-button.png) + + + ![Dynamic Secret Modal](../../../images/platform/dynamic-secrets/dynamic-secret-modal-aws-iam.png) + + + ![Dynamic Secret Setup Modal](../../../images/platform/dynamic-secrets/dynamic-secret-setup-modal-aws-iam-assume-role.png) + + Name by which you want the secret to be referenced + - - The AWS IAM managed policies that should be attached to the created users. Multiple values can be provided by separating them with commas - + + Default time-to-live for a generated secret (it is possible to modify this value after a secret is generated) + - - The AWS IAM inline policy that should be attached to the created users. Multiple values can be provided by separating them with commas - + + Maximum time-to-live for a generated secret + - -Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. + + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. -Allowed template variables are -- `{{randomUsername}}`: Random username string -- `{{unixTimestamp}}`: Current Unix timestamp - + Allowed template variables are + - `{{randomUsername}}`: Random username string + - `{{unixTimestamp}}`: Current Unix timestamp + - `{{identity.name}}`: Name of the identity that is generating the secret + - `{{random N}}`: Random string of N characters - ![Dynamic Secret Setup Modal](../../../images/platform/dynamic-secrets/dynamic-secret-setup-modal-aws-iam.png) + Allowed template functions are + - `truncate`: Truncates a string to a specified length + - `replace`: Replaces a substring with another value - - - After submitting the form, you will see a dynamic secret created in the dashboard. + Examples: + ``` + {{randomUsername}} // 3POnzeFyK9gW2nioK0q2gMjr6CZqsRiX + {{unixTimestamp}} // 17490641580 + {{identity.name}} // testuser + {{random-5}} // x9k2m + {{truncate identity.name 4}} // test + {{replace identity.name 'user' 'replace'}} // testreplace + ``` + + + Select *Assume Role* method. + - ![Dynamic Secret](../../../images/platform/dynamic-secrets/dynamic-secret.png) - - - Once you've successfully configured the dynamic secret, you're ready to generate on-demand credentials. - To do this, simply click on the 'Generate' button which appears when hovering over the dynamic secret item. - Alternatively, you can initiate the creation of a new lease by selecting 'New Lease' from the dynamic secret lease list section. + + The ARN of the AWS Role to assume. + - ![Dynamic Secret](/images/platform/dynamic-secrets/dynamic-secret-generate.png) - ![Dynamic Secret](/images/platform/dynamic-secrets/dynamic-secret-lease-empty.png) + + [IAM AWS Path](https://aws.amazon.com/blogs/security/optimize-aws-administration-with-iam-paths/) to scope created IAM User resource access. + - When generating these secrets, it's important to specify a Time-to-Live (TTL) duration. This will dictate how long the credentials are valid for. + + The AWS data center region. + - ![Provision Lease](/images/platform/dynamic-secrets/provision-lease.png) + + The IAM Policy ARN of the [AWS Permissions Boundary](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_boundaries.html) to attach to IAM users created in the role. + - - Ensure that the TTL for the lease fall within the maximum TTL defined when configuring the dynamic secret in step 4. - + + The AWS IAM groups that should be assigned to the created users. Multiple values can be provided by separating them with commas + + + The AWS IAM managed policies that should be attached to the created users. Multiple values can be provided by separating them with commas + - Once you click the `Submit` button, a new secret lease will be generated and the credentials for it will be shown to you. + + The AWS IAM inline policy that should be attached to the created users. + Multiple values can be provided by separating them with commas + - ![Provision Lease](/images/platform/dynamic-secrets/lease-values-aws-iam.png) - - + + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. + + Allowed template variables are + + - `{{randomUsername}}`: Random username string + - `{{unixTimestamp}}`: Current Unix timestamp + + + + + After submitting the form, you will see a dynamic secret created in the dashboard. + ![Dynamic Secret](../../../images/platform/dynamic-secrets/dynamic-secret.png) + + + + Once you've successfully configured the dynamic secret, you're ready to generate on-demand credentials. + To do this, simply click on the 'Generate' button which appears when hovering over the dynamic secret item. + Alternatively, you can initiate the creation of a new lease by selecting 'New Lease' from the dynamic secret lease list section. + + ![Dynamic Secret](/images/platform/dynamic-secrets/dynamic-secret-generate.png) + ![Dynamic Secret](/images/platform/dynamic-secrets/dynamic-secret-lease-empty.png) + + When generating these secrets, it's important to specify a Time-to-Live (TTL) duration. This will dictate how long the credentials are valid for. + + ![Provision Lease](/images/platform/dynamic-secrets/provision-lease.png) + + + Ensure that the TTL for the lease falls within the maximum TTL defined when configuring the dynamic secret in step 4. + + + Once you click the `Submit` button, a new secret lease will be generated and the credentials for it will be shown to you. + + ![Provision Lease](/images/platform/dynamic-secrets/lease-values-aws-iam.png) + + + + + + Infisical will use the provided **Access Key ID** and **Secret Key** to connect to your AWS instance. + + + Navigate to the Secret Overview dashboard and select the environment in which you would like to add a dynamic secret to. + + + ![Add Dynamic Secret Button](../../../images/platform/dynamic-secrets/add-dynamic-secret-button.png) + + + ![Dynamic Secret Modal](../../../images/platform/dynamic-secrets/dynamic-secret-modal-aws-iam.png) + + + ![Dynamic Secret Setup Modal](../../../images/platform/dynamic-secrets/dynamic-secret-setup-modal-aws-iam-access-key.png) + + Name by which you want the secret to be referenced + + + + Default time-to-live for a generated secret (it is possible to modify this value after a secret is generated) + + + + Maximum time-to-live for a generated secret + + + + Select *Access Key* method. + + + + The managing AWS IAM User Access Key + + + + The managing AWS IAM User Secret Key + + + + [IAM AWS Path](https://aws.amazon.com/blogs/security/optimize-aws-administration-with-iam-paths/) to scope created IAM User resource access. + + + + The AWS data center region. + + + + The IAM Policy ARN of the [AWS Permissions Boundary](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_boundaries.html) to attach to IAM users created in the role. + + + + The AWS IAM groups that should be assigned to the created users. Multiple values can be provided by separating them with commas + + + + The AWS IAM managed policies that should be attached to the created users. Multiple values can be provided by separating them with commas + + + + The AWS IAM inline policy that should be attached to the created users. + Multiple values can be provided by separating them with commas + + + + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. + + Allowed template variables are + + - `{{randomUsername}}`: Random username string + - `{{unixTimestamp}}`: Current Unix timestamp + + + + + + After submitting the form, you will see a dynamic secret created in the dashboard. + ![Dynamic Secret](../../../images/platform/dynamic-secrets/dynamic-secret.png) + + + + Once you've successfully configured the dynamic secret, you're ready to generate on-demand credentials. + To do this, simply click on the 'Generate' button which appears when hovering over the dynamic secret item. + Alternatively, you can initiate the creation of a new lease by selecting 'New Lease' from the dynamic secret lease list section. + + ![Dynamic Secret](/images/platform/dynamic-secrets/dynamic-secret-generate.png) + ![Dynamic Secret](/images/platform/dynamic-secrets/dynamic-secret-lease-empty.png) + + When generating these secrets, it's important to specify a Time-to-Live (TTL) duration. This will dictate how long the credentials are valid for. + + ![Provision Lease](/images/platform/dynamic-secrets/provision-lease.png) + + + Ensure that the TTL for the lease falls within the maximum TTL defined when configuring the dynamic secret in step 4. + + + Once you click the `Submit` button, a new secret lease will be generated and the credentials for it will be shown to you. + + ![Provision Lease](/images/platform/dynamic-secrets/lease-values-aws-iam.png) + + + + + ## Audit or Revoke Leases + Once you have created one or more leases, you will be able to access them by clicking on the respective dynamic secret item on the dashboard. This will allow you to see the lease details and delete the lease ahead of its expiration time. ![Provision Lease](/images/platform/dynamic-secrets/lease-data.png) ## Renew Leases + To extend the life of the generated dynamic secret lease past its initial time to live, simply click on the **Renew** button as illustrated below. ![Provision Lease](/images/platform/dynamic-secrets/dynamic-secret-lease-renew.png) - Lease renewals cannot exceed the maximum TTL set when configuring the dynamic secret + Lease renewals cannot exceed the maximum TTL set when configuring the dynamic + secret diff --git a/docs/documentation/platform/dynamic-secrets/cassandra.mdx b/docs/documentation/platform/dynamic-secrets/cassandra.mdx index 628432bea..56b7ec336 100644 --- a/docs/documentation/platform/dynamic-secrets/cassandra.mdx +++ b/docs/documentation/platform/dynamic-secrets/cassandra.mdx @@ -80,11 +80,27 @@ The above configuration allows user creation and granting permissions. ![Modify CQL Statements Modal](../../../images/platform/dynamic-secrets/modify-cql-statements.png) - Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. - Allowed template variables are - - `{{randomUsername}}`: Random username string - - `{{unixTimestamp}}`: Current Unix timestamp + Allowed template variables are + - `{{randomUsername}}`: Random username string + - `{{unixTimestamp}}`: Current Unix timestamp + - `{{identity.name}}`: Name of the identity that is generating the secret + - `{{random N}}`: Random string of N characters + + Allowed template functions are + - `truncate`: Truncates a string to a specified length + - `replace`: Replaces a substring with another value + + Examples: + ``` + {{randomUsername}} // 3POnzeFyK9gW2nioK0q2gMjr6CZqsRiX + {{unixTimestamp}} // 17490641580 + {{identity.name}} // testuser + {{random-5}} // x9k2m + {{truncate identity.name 4}} // test + {{replace identity.name 'user' 'replace'}} // testreplace + ``` If you want to provide specific privileges for the generated dynamic credentials, you can modify the CQL statement to your needs. This is useful if you want to only give access to a specific key-space(s). diff --git a/docs/documentation/platform/dynamic-secrets/elastic-search.mdx b/docs/documentation/platform/dynamic-secrets/elastic-search.mdx index 6c5028bb2..06d1102e2 100644 --- a/docs/documentation/platform/dynamic-secrets/elastic-search.mdx +++ b/docs/documentation/platform/dynamic-secrets/elastic-search.mdx @@ -87,13 +87,29 @@ The port that your Elasticsearch instance is running on. _(Example: 9200)_ A CA may be required if your DB requires it for incoming connections. This is often the case when connecting to a managed service. - - Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. + + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. - Allowed template variables are - - `{{randomUsername}}`: Random username string - - `{{unixTimestamp}}`: Current Unix timestamp - + Allowed template variables are + - `{{randomUsername}}`: Random username string + - `{{unixTimestamp}}`: Current Unix timestamp + - `{{identity.name}}`: Name of the identity that is generating the secret + - `{{random N}}`: Random string of N characters + + Allowed template functions are + - `truncate`: Truncates a string to a specified length + - `replace`: Replaces a substring with another value + + Examples: + ``` + {{randomUsername}} // 3POnzeFyK9gW2nioK0q2gMjr6CZqsRiX + {{unixTimestamp}} // 17490641580 + {{identity.name}} // testuser + {{random-5}} // x9k2m + {{truncate identity.name 4}} // test + {{replace identity.name 'user' 'replace'}} // testreplace + ``` + ![Dynamic Secret Setup Modal](../../../images/platform/dynamic-secrets/dynamic-secret-input-modal-elastic-search.png) diff --git a/docs/documentation/platform/dynamic-secrets/kubernetes.mdx b/docs/documentation/platform/dynamic-secrets/kubernetes.mdx index d6d051ff7..87c5b3e89 100644 --- a/docs/documentation/platform/dynamic-secrets/kubernetes.mdx +++ b/docs/documentation/platform/dynamic-secrets/kubernetes.mdx @@ -33,125 +33,6 @@ This feature is ideal for scenarios where you need to: - Maintain a secure audit trail of cluster access - Manage access to multiple Kubernetes clusters -## Prerequisites - -- A Kubernetes cluster with a service account -- Cluster access token with permissions to create service account tokens -- (Optional) [Gateway](/documentation/platform/gateways/overview) for private cluster access - -## RBAC Configuration - -Before you can start generating dynamic service account tokens, you'll need to configure the appropriate permissions in your Kubernetes cluster. This involves setting up Role-Based Access Control (RBAC) to allow the creation and management of service account tokens. - -The RBAC configuration serves a crucial security purpose: it creates a dedicated service account with minimal permissions that can only create and manage service account tokens. This follows the principle of least privilege, ensuring that the token generation process is secure and controlled. - -The following RBAC configuration creates the necessary permissions for generating service account tokens: - -```yaml rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: tokenrequest -rules: - - apiGroups: [""] - resources: - - "serviceaccounts/token" - - "serviceaccounts" - verbs: - - "create" - - "get" ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: tokenrequest -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: tokenrequest -subjects: - - kind: ServiceAccount - name: infisical-token-requester - namespace: default -``` - -```bash -kubectl apply -f rbac.yaml -``` - -This configuration: - -1. Creates a `ClusterRole` named `tokenrequest` that allows: - - Creating and getting service account tokens - - Getting service account information -2. Creates a `ClusterRoleBinding` that binds the role to a service account named `infisical-token-requester` in the `default` namespace - -You can customize the service account name and namespace according to your needs. - -## Obtaining the Cluster Token - -After setting up the RBAC configuration, you need to obtain a token for the service account that will be used to create dynamic secrets. Here's how to get the token: - -1. Create a service account in your Kubernetes cluster that will be used to create service account tokens: - -```yaml infisical-service-account.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: infisical-token-requester - namespace: default -``` - -```bash -kubectl apply -f infisical-service-account.yaml -``` - -2. Create a long-lived service account token using this configuration file: - -```yaml service-account-token.yaml -apiVersion: v1 -kind: Secret -type: kubernetes.io/service-account-token -metadata: - name: infisical-token-requester-token - annotations: - kubernetes.io/service-account.name: "infisical-token-requester" -``` - -```bash -kubectl apply -f service-account-token.yaml -``` - -3. Link the secret to the service account: - -```bash -kubectl patch serviceaccount infisical-token-requester -p '{"secrets": [{"name": "infisical-token-requester-token"}]}' -n default -``` - -4. Retrieve the token: - -```bash -kubectl get secret infisical-token-requester-token -n default -o=jsonpath='{.data.token}' | base64 --decode -``` - -This token will be used as the "Cluster Token" in the dynamic secret configuration. - -## Obtaining the Cluster URL - -The cluster URL is the address of your Kubernetes API server. The simplest way to find it is to use the `kubectl cluster-info` command: - -```bash -kubectl cluster-info -``` - -This command works for all Kubernetes environments (managed services like GKE, EKS, AKS, or self-hosted clusters) and will show you the Kubernetes control plane address, which is your cluster URL. - - - Make sure the cluster URL is accessible from where you're running Infisical. - If you're using a private cluster, you'll need to configure a [Gateway](/documentation/platform/gateways/overview) to - access it. - - ## Set up Dynamic Secrets with Kubernetes @@ -164,6 +45,371 @@ This command works for all Kubernetes environments (managed services like GKE, E ![Dynamic Secret Modal](/images/platform/dynamic-secrets/dynamic-secret-modal-kubernetes.png) + + Before proceeding with the setup, you'll need to make two key decisions: + + 1. **Credential Type**: How you want to manage service accounts + - **Static**: Use an existing service account with predefined permissions + - **Dynamic**: Create temporary service accounts with specific role assignments + + 2. **Authentication Method**: How you want to authenticate with the cluster + - **Token (API)**: Use a service account token for direct API access + - **Gateway**: Use an Infisical Gateway deployed in your cluster + + + + Static credentials generate service account tokens for a predefined service account. This is useful when you want to: + - Generate tokens for an existing service account + - Maintain consistent permissions across token generations + - Use a service account that already has the necessary RBAC permissions + + ### Prerequisites + + - A Kubernetes cluster with a service account + - Cluster access token with permissions to create service account tokens + - (Optional) [Gateway](/documentation/platform/gateways/overview) for private cluster access + + ### Authentication Setup + + Choose your authentication method: + + + + This method uses a service account token to authenticate with the Kubernetes cluster. It's suitable when: + - You want to use a specific service account token that you've created + - You're working with a public cluster or have network access to the cluster's API server + - You want to explicitly control which service account is used for operations + + + With Token (API) authentication, Infisical uses the provided service account token + to make API calls to your Kubernetes cluster. This token must have the necessary + permissions to generate tokens for the target service account. + + + 1. Create a service account: + ```yaml infisical-service-account.yaml + apiVersion: v1 + kind: ServiceAccount + metadata: + name: infisical-token-requester + namespace: default + ``` + + ```bash + kubectl apply -f infisical-service-account.yaml + ``` + + 2. Set up RBAC permissions: + ```yaml rbac.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: tokenrequest + rules: + - apiGroups: [""] + resources: + - "serviceaccounts/token" + - "serviceaccounts" + verbs: + - "create" + - "get" + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: tokenrequest + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: tokenrequest + subjects: + - kind: ServiceAccount + name: infisical-token-requester + namespace: default + ``` + + ```bash + kubectl apply -f rbac.yaml + ``` + + 3. Create and obtain the token: + ```yaml service-account-token.yaml + apiVersion: v1 + kind: Secret + type: kubernetes.io/service-account-token + metadata: + name: infisical-token-requester-token + annotations: + kubernetes.io/service-account.name: "infisical-token-requester" + ``` + + ```bash + kubectl apply -f service-account-token.yaml + kubectl patch serviceaccount infisical-token-requester -p '{"secrets": [{"name": "infisical-token-requester-token"}]}' -n default + kubectl get secret infisical-token-requester-token -n default -o=jsonpath='{.data.token}' | base64 --decode + ``` + + + This method uses an Infisical Gateway deployed in your Kubernetes cluster. It's ideal when: + - You want to avoid storing static service account tokens + - You prefer to use the Gateway's pre-configured service account + - You want centralized management of cluster operations + + + With Gateway authentication, Infisical communicates with the Gateway, which then + uses its own service account to make API calls to the Kubernetes API server. + The Gateway's service account must have the necessary permissions to generate + tokens for the target service account. + + + + When using Gateway authentication, the Gateway will access the Kubernetes API server + using its internal cluster URL (typically https://kubernetes.default.svc) and TLS configuration. + You don't need to specify these values separately in the dynamic secret configuration. + + + 1. Deploy the Infisical Gateway in your cluster + 2. Set up RBAC permissions for the Gateway's service account: + ```yaml rbac.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: tokenrequest + rules: + - apiGroups: [""] + resources: + - "serviceaccounts/token" + - "serviceaccounts" + verbs: + - "create" + - "get" + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: tokenrequest + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: tokenrequest + subjects: + - kind: ServiceAccount + name: infisical-gateway + namespace: infisical + ``` + + ```bash + kubectl apply -f rbac.yaml + ``` + + + + + + + Dynamic credentials create a temporary service account, assign it to a defined role/cluster-role, and generate a service account token. This is useful when you want to: + - Create temporary service accounts with specific permissions + - Automatically clean up service accounts after token expiration + - Assign different roles to different users or applications + - Maintain strict control over service account permissions + - Support multiple namespaces with a single dynamic secret configuration + + ### Prerequisites + + - A Kubernetes cluster with a service account + - Cluster access token with permissions to create service accounts and manage RBAC + - (Optional) [Gateway](/documentation/platform/gateways/overview) for private cluster access + + ### Namespace Support + + When configuring a dynamic secret, you can specify multiple allowed namespaces as a comma-separated list. During lease creation, you can then specify which namespace to use from this allowed list. This provides flexibility while maintaining security by: + + - Allowing a single dynamic secret configuration to support multiple namespaces + - Restricting service account creation to only the specified allowed namespaces + - Enabling fine-grained control over which namespaces can be used for each lease + + For example, if you configure a dynamic secret with allowed namespaces "default,kube-system,monitoring", you can create leases that use any of these namespaces while preventing access to other namespaces in your cluster. + + ### Authentication Setup + + Choose your authentication method: + + + + This method uses a service account token to authenticate with the Kubernetes cluster. It's suitable when: + - You want to use a specific service account token that you've created + - You're working with a public cluster or have network access to the cluster's API server + - You want to explicitly control which service account is used for operations + + + With Token (API) authentication, Infisical uses the provided service account token + to make API calls to your Kubernetes cluster. This token must have the necessary + permissions to create and manage service accounts, their tokens, and RBAC resources. + + + 1. Create a service account: + ```yaml service-account.yaml + apiVersion: v1 + kind: ServiceAccount + metadata: + name: infisical-token-requester + namespace: default + --- + apiVersion: v1 + kind: Secret + type: kubernetes.io/service-account-token + metadata: + name: infisical-token-requester-token + annotations: + kubernetes.io/service-account.name: "infisical-token-requester" + ``` + + ```bash + kubectl apply -f service-account.yaml + ``` + + 2. Set up RBAC permissions: + ```yaml rbac.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: tokenrequest + rules: + - apiGroups: [""] + resources: + - "serviceaccounts/token" + - "serviceaccounts" + verbs: + - "create" + - "get" + - "delete" + - apiGroups: ["rbac.authorization.k8s.io"] + resources: + - "rolebindings" + - "clusterrolebindings" + verbs: + - "create" + - "delete" + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: tokenrequest + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: tokenrequest + subjects: + - kind: ServiceAccount + name: infisical-token-requester + namespace: default + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: infisical-dynamic-role-binding-sa + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: infisical-dynamic-role + subjects: + - kind: ServiceAccount + name: infisical-token-requester + namespace: default + ``` + + ```bash + kubectl apply -f rbac.yaml + ``` + + + This method uses an Infisical Gateway deployed in your Kubernetes cluster. It's ideal when: + - You want to avoid storing static service account tokens + - You prefer to use the Gateway's pre-configured service account + - You want centralized management of cluster operations + + + With Gateway authentication, Infisical communicates with the Gateway, which then + uses its own service account to make API calls to the Kubernetes API server. + The Gateway's service account must have the necessary permissions to create and + manage service accounts, their tokens, and RBAC resources. + + + + When using Gateway authentication, the Gateway will access the Kubernetes API server + using its internal cluster URL (typically https://kubernetes.default.svc) and TLS configuration. + You don't need to specify these values separately in the dynamic secret configuration. + + + 1. Deploy the Infisical Gateway in your cluster + 2. Set up RBAC permissions for the Gateway's service account: + ```yaml rbac.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: tokenrequest + rules: + - apiGroups: [""] + resources: + - "serviceaccounts/token" + - "serviceaccounts" + verbs: + - "create" + - "get" + - "delete" + - apiGroups: ["rbac.authorization.k8s.io"] + resources: + - "rolebindings" + - "clusterrolebindings" + verbs: + - "create" + - "delete" + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: tokenrequest + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: tokenrequest + subjects: + - kind: ServiceAccount + name: infisical-gateway + namespace: infisical + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: infisical-dynamic-role-binding-sa + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: infisical-dynamic-role + subjects: + - kind: ServiceAccount + name: infisical-gateway + namespace: infisical + ``` + + ```bash + kubectl apply -f rbac.yaml + ``` + + + + + In Kubernetes RBAC, a service account can only create role bindings for resources that it has access to. + This means that if you want to create dynamic service accounts with access to certain resources, + the service account creating these bindings (either the token requester or Gateway service account) + must also have access to those same resources. For example, if you want to create dynamic service + accounts that can access secrets, the token requester service account must also have access to secrets. + + + + + + Name by which you want the secret to be referenced @@ -178,56 +424,82 @@ This command works for all Kubernetes environments (managed services like GKE, E Select a gateway for private cluster access. If not specified, the Internet Gateway will be used. - Kubernetes API server URL (e.g., https://kubernetes.default.svc) + Kubernetes API server URL (e.g., https://kubernetes.default.svc). Not required when using Gateway authentication as the Gateway will use its internal cluster URL. - Whether to enable SSL verification for the Kubernetes API server connection. + Whether to enable SSL verification for the Kubernetes API server connection. Not required when using Gateway authentication as the Gateway will use its internal TLS configuration. - Custom CA certificate for the Kubernetes API server. Leave blank to use the system/public CA. + Custom CA certificate for the Kubernetes API server. Leave blank to use the system/public CA. Not required when using Gateway authentication as the Gateway will use its internal TLS configuration. + + + Choose between Token (API) or Gateway authentication. If using Gateway, the Gateway must be deployed in your Kubernetes cluster. - Token with permissions to create service account tokens + Token with permissions to create service accounts and manage RBAC (required when using Token authentication) - - Name of the service account to generate tokens for - - - Kubernetes namespace where the service account exists + + Choose between Static (predefined service account) or Dynamic (temporary service accounts with role assignments) + + + + + Name of the service account to generate tokens for + + + Kubernetes namespace where the service account exists + + + + + + Kubernetes namespace(s) where the service accounts will be created. You can specify multiple namespaces as a comma-separated list (e.g., "default,kube-system"). During lease creation, you can specify which namespace to use from this allowed list. + + + Type of role to assign (ClusterRole or Role) + + + Name of the role to assign to the temporary service account + + + + Optional list of audiences to include in the generated token - ![Dynamic Secret Setup Modal](../../../images/platform/dynamic-secrets/dynamic-secret-setup-modal-kubernetes.png) + ![Dynamic Secret Setup Modal](../../../images/platform/dynamic-secrets/dynamic-secret-setup-modal-kubernetes-1.png) + ![Dynamic Secret Setup Modal](../../../images/platform/dynamic-secrets/dynamic-secret-setup-modal-kubernetes-2.png) After submitting the form, you will see a dynamic secret created in the dashboard. - - Once you've successfully configured the dynamic secret, you're ready to generate on-demand service account tokens. - To do this, simply click on the 'Generate' button which appears when hovering over the dynamic secret item. - Alternatively, you can initiate the creation of a new lease by selecting 'New Lease' from the dynamic secret lease list section. - - ![Dynamic Secret](/images/platform/dynamic-secrets/dynamic-secret-generate.png) - ![Dynamic Secret](/images/platform/dynamic-secrets/dynamic-secret-lease-empty.png) - - When generating these secrets, it's important to specify a Time-to-Live (TTL) duration. This will dictate how long the credentials are valid for. - - ![Provision Lease](/images/platform/dynamic-secrets/provision-lease.png) - - - Ensure that the TTL for the lease fall within the maximum TTL defined when configuring the dynamic secret. - - - Once you click the `Submit` button, a new secret lease will be generated and the service account token will be shown to you. - - ![Provision Lease](/images/platform/dynamic-secrets/kubernetes-lease-value.png) - - +## Generate and Manage Tokens + +Once you've successfully configured the dynamic secret, you're ready to generate on-demand service account tokens. +To do this, simply click on the 'Generate' button which appears when hovering over the dynamic secret item. +Alternatively, you can initiate the creation of a new lease by selecting 'New Lease' from the dynamic secret lease list section. + +![Dynamic Secret](/images/platform/dynamic-secrets/dynamic-secret-generate.png) +![Dynamic Secret](/images/platform/dynamic-secrets/dynamic-secret-lease-empty.png) + +When generating these secrets, it's important to specify a Time-to-Live (TTL) duration. This will dictate how long the credentials are valid for. + +![Provision Lease](/images/platform/dynamic-secrets/provision-lease.png) + + + Ensure that the TTL for the lease fall within the maximum TTL defined when + configuring the dynamic secret. + + +Once you click the `Submit` button, a new secret lease will be generated and the service account token will be shown to you. + +![Provision Lease](/images/platform/dynamic-secrets/kubernetes-lease-value.png) + ## Audit or Revoke Leases Once you have created one or more leases, you will be able to access them by clicking on the respective dynamic secret item on the dashboard. diff --git a/docs/documentation/platform/dynamic-secrets/ldap.mdx b/docs/documentation/platform/dynamic-secrets/ldap.mdx index 1a3eca404..ddaaf9101 100644 --- a/docs/documentation/platform/dynamic-secrets/ldap.mdx +++ b/docs/documentation/platform/dynamic-secrets/ldap.mdx @@ -123,13 +123,29 @@ The Infisical LDAP dynamic secret allows you to generate user credentials on dem changetype: delete ``` - + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. Allowed template variables are - `{{randomUsername}}`: Random username string - `{{unixTimestamp}}`: Current Unix timestamp - + - `{{identity.name}}`: Name of the identity that is generating the secret + - `{{random N}}`: Random string of N characters + + Allowed template functions are + - `truncate`: Truncates a string to a specified length + - `replace`: Replaces a substring with another value + + Examples: + ``` + {{randomUsername}} // 3POnzeFyK9gW2nioK0q2gMjr6CZqsRiX + {{unixTimestamp}} // 17490641580 + {{identity.name}} // testuser + {{random-5}} // x9k2m + {{truncate identity.name 4}} // test + {{replace identity.name 'user' 'replace'}} // testreplace + ``` + diff --git a/docs/documentation/platform/dynamic-secrets/mongo-atlas.mdx b/docs/documentation/platform/dynamic-secrets/mongo-atlas.mdx index 5d27d16e2..f167a0641 100644 --- a/docs/documentation/platform/dynamic-secrets/mongo-atlas.mdx +++ b/docs/documentation/platform/dynamic-secrets/mongo-atlas.mdx @@ -63,13 +63,29 @@ Create a project scoped API Key with the required permission in your Mongo Atlas ![Modify Scope Modal](../../../images/platform/dynamic-secrets/advanced-option-atlas.png) - - Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. + + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. - Allowed template variables are - - `{{randomUsername}}`: Random username string - - `{{unixTimestamp}}`: Current Unix timestamp - + Allowed template variables are + - `{{randomUsername}}`: Random username string + - `{{unixTimestamp}}`: Current Unix timestamp + - `{{identity.name}}`: Name of the identity that is generating the secret + - `{{random N}}`: Random string of N characters + + Allowed template functions are + - `truncate`: Truncates a string to a specified length + - `replace`: Replaces a substring with another value + + Examples: + ``` + {{randomUsername}} // 3POnzeFyK9gW2nioK0q2gMjr6CZqsRiX + {{unixTimestamp}} // 17490641580 + {{identity.name}} // testuser + {{random-5}} // x9k2m + {{truncate identity.name 4}} // test + {{replace identity.name 'user' 'replace'}} // testreplace + ``` + List that contains clusters, MongoDB Atlas Data Lakes, and MongoDB Atlas Streams Instances that this database user can access. If omitted, MongoDB Cloud grants the database user access to all the clusters, MongoDB Atlas Data Lakes, and MongoDB Atlas Streams Instances in the project. diff --git a/docs/documentation/platform/dynamic-secrets/mongo-db.mdx b/docs/documentation/platform/dynamic-secrets/mongo-db.mdx index f71922473..7c4ebb568 100644 --- a/docs/documentation/platform/dynamic-secrets/mongo-db.mdx +++ b/docs/documentation/platform/dynamic-secrets/mongo-db.mdx @@ -66,12 +66,28 @@ Create a user with the required permission in your MongoDB instance. This user w A CA may be required if your DB requires it for incoming connections. - + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. Allowed template variables are - `{{randomUsername}}`: Random username string - `{{unixTimestamp}}`: Current Unix timestamp + - `{{identity.name}}`: Name of the identity that is generating the secret + - `{{random N}}`: Random string of N characters + + Allowed template functions are + - `truncate`: Truncates a string to a specified length + - `replace`: Replaces a substring with another value + + Examples: + ``` + {{randomUsername}} // 3POnzeFyK9gW2nioK0q2gMjr6CZqsRiX + {{unixTimestamp}} // 17490641580 + {{identity.name}} // testuser + {{random-5}} // x9k2m + {{truncate identity.name 4}} // test + {{replace identity.name 'user' 'replace'}} // testreplace + ``` ![Dynamic Secret Setup Modal](../../../images/platform/dynamic-secrets/dynamic-secret-mongodb.png) diff --git a/docs/documentation/platform/dynamic-secrets/mssql.mdx b/docs/documentation/platform/dynamic-secrets/mssql.mdx index 0a73bf129..6b6cef982 100644 --- a/docs/documentation/platform/dynamic-secrets/mssql.mdx +++ b/docs/documentation/platform/dynamic-secrets/mssql.mdx @@ -9,7 +9,6 @@ The Infisical MS SQL dynamic secret allows you to generate Microsoft SQL server Create a user with the required permission in your SQL instance. This user will be used to create new accounts on-demand. - ## Set up Dynamic Secrets with MS SQL @@ -27,104 +26,123 @@ Create a user with the required permission in your SQL instance. This user will Name by which you want the secret to be referenced - - Default time-to-live for a generated secret (it is possible to modify this value after a secret is generated) - + + Default time-to-live for a generated secret (it is possible to modify this value after a secret is generated) + - - Maximum time-to-live for a generated secret - + + Maximum time-to-live for a generated secret + - - List of key/value metadata pairs - + + List of key/value metadata pairs + - - Choose the service you want to generate dynamic secrets for. This must be selected as **MS SQL**. - + + Choose the service you want to generate dynamic secrets for. This must be selected as **MS SQL**. + - - Database host - + + Database host + - - Database port - + + Database port + - - Username that will be used to create dynamic secrets - + + Username that will be used to create dynamic secrets + - - Password that will be used to create dynamic secrets - + + Password that will be used to create dynamic secrets + - - Name of the database for which you want to create dynamic secrets - + + Name of the database for which you want to create dynamic secrets + - - A CA may be required if your DB requires it for incoming connections. AWS RDS instances with default settings will requires a CA which can be downloaded [here](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.SSL.html#UsingWithRDS.SSL.CertificatesAllRegions). - + + A CA may be required if your DB requires it for incoming connections. AWS RDS instances with default settings will requires a CA which can be downloaded [here](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.SSL.html#UsingWithRDS.SSL.CertificatesAllRegions). + - ![Dynamic Secret Setup Modal](../../../images/platform/dynamic-secrets/dynamic-secret-setup-modal-mssql.png) + ![Dynamic Secret Setup Modal](../../../images/platform/dynamic-secrets/dynamic-secret-setup-modal-mssql.png) ![Modify SQL Statements Modal](../../../images/platform/dynamic-secrets/modify-sql-statements-mssql.png) - - Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. + + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. + + Allowed template variables are + - `{{randomUsername}}`: Random username string + - `{{unixTimestamp}}`: Current Unix timestamp + - `{{identity.name}}`: Name of the identity that is generating the secret + - `{{random N}}`: Random string of N characters + + Allowed template functions are + - `truncate`: Truncates a string to a specified length + - `replace`: Replaces a substring with another value + + Examples: + ``` + {{randomUsername}} // 3POnzeFyK9gW2nioK0q2gMjr6CZqsRiX + {{unixTimestamp}} // 17490641580 + {{identity.name}} // testuser + {{random-5}} // x9k2m + {{truncate identity.name 4}} // test + {{replace identity.name 'user' 'replace'}} // testreplace + ``` + - Allowed template variables are - - `{{randomUsername}}`: Random username string - - `{{unixTimestamp}}`: Current Unix timestamp - - - If you want to provide specific privileges for the generated dynamic credentials, you can modify the SQL statement to your needs. This is useful if you want to only give access to a specific table(s). - After submitting the form, you will see a dynamic secret created in the dashboard. - - If this step fails, you may have to add the CA certificate. - + + If this step fails, you may have to add the CA certificate. + + + ![Dynamic Secret](../../../images/platform/dynamic-secrets/dynamic-secret.png) - ![Dynamic Secret](../../../images/platform/dynamic-secrets/dynamic-secret.png) Once you've successfully configured the dynamic secret, you're ready to generate on-demand credentials. To do this, simply click on the 'Generate' button which appears when hovering over the dynamic secret item. Alternatively, you can initiate the creation of a new lease by selecting 'New Lease' from the dynamic secret lease list section. - ![Dynamic Secret](/images/platform/dynamic-secrets/dynamic-secret-generate.png) - ![Dynamic Secret](/images/platform/dynamic-secrets/dynamic-secret-lease-empty.png) + ![Dynamic Secret](/images/platform/dynamic-secrets/dynamic-secret-generate.png) + ![Dynamic Secret](/images/platform/dynamic-secrets/dynamic-secret-lease-empty.png) - When generating these secrets, it's important to specify a Time-to-Live (TTL) duration. This will dictate how long the credentials are valid for. + When generating these secrets, it's important to specify a Time-to-Live (TTL) duration. This will dictate how long the credentials are valid for. - ![Provision Lease](/images/platform/dynamic-secrets/provision-lease.png) + ![Provision Lease](/images/platform/dynamic-secrets/provision-lease.png) - - Ensure that the TTL for the lease fall within the maximum TTL defined when configuring the dynamic secret. - + + Ensure that the TTL for the lease fall within the maximum TTL defined when configuring the dynamic secret. + - Once you click the `Submit` button, a new secret lease will be generated and the credentials for it will be shown to you. + Once you click the `Submit` button, a new secret lease will be generated and the credentials for it will be shown to you. + + ![Provision Lease](/images/platform/dynamic-secrets/lease-values.png) - ![Provision Lease](/images/platform/dynamic-secrets/lease-values.png) ## Audit or Revoke Leases + Once you have created one or more leases, you will be able to access them by clicking on the respective dynamic secret item on the dashboard. This will allow you to see the expiration time of the lease or delete the lease before it's set time to live. ![Provision Lease](/images/platform/dynamic-secrets/lease-data.png) ## Renew Leases + To extend the life of the generated dynamic secret leases past its initial time to live, simply click on the **Renew** button as illustrated below. ![Provision Lease](/images/platform/dynamic-secrets/dynamic-secret-lease-renew.png) - Lease renewals cannot exceed the maximum TTL set when configuring the dynamic secret + Lease renewals cannot exceed the maximum TTL set when configuring the dynamic + secret diff --git a/docs/documentation/platform/dynamic-secrets/mysql.mdx b/docs/documentation/platform/dynamic-secrets/mysql.mdx index 6f708ebba..33d11a4fc 100644 --- a/docs/documentation/platform/dynamic-secrets/mysql.mdx +++ b/docs/documentation/platform/dynamic-secrets/mysql.mdx @@ -69,15 +69,28 @@ Create a user with the required permission in your SQL instance. This user will ![Modify SQL Statements Modal](../../../images/platform/dynamic-secrets/modify-sql-statement-mysql.png) - - Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. + + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. - Allowed template variables are - - `{{randomUsername}}`: Random username string - - `{{unixTimestamp}}`: Current Unix timestamp - - - If you want to provide specific privileges for the generated dynamic credentials, you can modify the SQL statement to your needs. This is useful if you want to only give access to a specific table(s). + Allowed template variables are + - `{{randomUsername}}`: Random username string + - `{{unixTimestamp}}`: Current Unix timestamp + - `{{identity.name}}`: Name of the identity that is generating the secret + - `{{random N}}`: Random string of N characters + + Allowed template functions are + - `truncate`: Truncates a string to a specified length + - `replace`: Replaces a substring with another value + + Examples: + ``` + {{randomUsername}} // 3POnzeFyK9gW2nioK0q2gMjr6CZqsRiX + {{unixTimestamp}} // 17490641580 + {{identity.name}} // testuser + {{random-5}} // x9k2m + {{truncate identity.name 4}} // test + {{replace identity.name 'user' 'replace'}} // testreplace + ``` diff --git a/docs/documentation/platform/dynamic-secrets/oracle.mdx b/docs/documentation/platform/dynamic-secrets/oracle.mdx index 02b379b98..3c83f3359 100644 --- a/docs/documentation/platform/dynamic-secrets/oracle.mdx +++ b/docs/documentation/platform/dynamic-secrets/oracle.mdx @@ -71,15 +71,28 @@ Create a user with the required permission in your SQL instance. This user will ![Modify SQL Statements Modal](../../../images/platform/dynamic-secrets/modify-sql-statement-oracle.png) - - Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. + + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. - Allowed template variables are - - `{{randomUsername}}`: Random username string - - `{{unixTimestamp}}`: Current Unix timestamp - - - If you want to provide specific privileges for the generated dynamic credentials, you can modify the SQL statement to your needs. This is useful if you want to only give access to a specific table(s). + Allowed template variables are + - `{{randomUsername}}`: Random username string + - `{{unixTimestamp}}`: Current Unix timestamp + - `{{identity.name}}`: Name of the identity that is generating the secret + - `{{random N}}`: Random string of N characters + + Allowed template functions are + - `truncate`: Truncates a string to a specified length + - `replace`: Replaces a substring with another value + + Examples: + ``` + {{randomUsername}} // 3POnzeFyK9gW2nioK0q2gMjr6CZqsRiX + {{unixTimestamp}} // 17490641580 + {{identity.name}} // testuser + {{random-5}} // x9k2m + {{truncate identity.name 4}} // test + {{replace identity.name 'user' 'replace'}} // testreplace + ``` diff --git a/docs/documentation/platform/dynamic-secrets/postgresql.mdx b/docs/documentation/platform/dynamic-secrets/postgresql.mdx index f13c9c762..974066e14 100644 --- a/docs/documentation/platform/dynamic-secrets/postgresql.mdx +++ b/docs/documentation/platform/dynamic-secrets/postgresql.mdx @@ -72,12 +72,28 @@ Create a user with the required permission in your SQL instance. This user will ![Modify SQL Statements Modal](../../../images/platform/dynamic-secrets/modify-sql-statements.png) - - Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. + + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. - Allowed template variables are - - `{{randomUsername}}`: Random username string - - `{{unixTimestamp}}`: Current Unix timestamp + Allowed template variables are + - `{{randomUsername}}`: Random username string + - `{{unixTimestamp}}`: Current Unix timestamp + - `{{identity.name}}`: Name of the identity that is generating the secret + - `{{random N}}`: Random string of N characters + + Allowed template functions are + - `truncate`: Truncates a string to a specified length + - `replace`: Replaces a substring with another value + + Examples: + ``` + {{randomUsername}} // 3POnzeFyK9gW2nioK0q2gMjr6CZqsRiX + {{unixTimestamp}} // 17490641580 + {{identity.name}} // testuser + {{random-5}} // x9k2m + {{truncate identity.name 4}} // test + {{replace identity.name 'user' 'replace'}} // testreplace + ``` If you want to provide specific privileges for the generated dynamic credentials, you can modify the SQL statement to your needs. This is useful if you want to only give access to a specific table(s). diff --git a/docs/documentation/platform/dynamic-secrets/rabbit-mq.mdx b/docs/documentation/platform/dynamic-secrets/rabbit-mq.mdx index 09c04e61b..be41901b7 100644 --- a/docs/documentation/platform/dynamic-secrets/rabbit-mq.mdx +++ b/docs/documentation/platform/dynamic-secrets/rabbit-mq.mdx @@ -66,12 +66,28 @@ The port that the RabbitMQ management plugin is listening on. This is `15672` by -Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. -Allowed template variables are -- `{{randomUsername}}`: Random username string -- `{{unixTimestamp}}`: Current Unix timestamp - + Allowed template variables are + - `{{randomUsername}}`: Random username string + - `{{unixTimestamp}}`: Current Unix timestamp + - `{{identity.name}}`: Name of the identity that is generating the secret + - `{{random N}}`: Random string of N characters + + Allowed template functions are + - `truncate`: Truncates a string to a specified length + - `replace`: Replaces a substring with another value + + Examples: + ``` + {{randomUsername}} // 3POnzeFyK9gW2nioK0q2gMjr6CZqsRiX + {{unixTimestamp}} // 17490641580 + {{identity.name}} // testuser + {{random-5}} // x9k2m + {{truncate identity.name 4}} // test + {{replace identity.name 'user' 'replace'}} // testreplace + ``` + A CA may be required if your DB requires it for incoming connections. This is often the case when connecting to a managed service. diff --git a/docs/documentation/platform/dynamic-secrets/redis.mdx b/docs/documentation/platform/dynamic-secrets/redis.mdx index b3e585204..8e28c3cb2 100644 --- a/docs/documentation/platform/dynamic-secrets/redis.mdx +++ b/docs/documentation/platform/dynamic-secrets/redis.mdx @@ -57,12 +57,28 @@ Create a user with the required permission in your Redis instance. This user wil ![Modify Redis Statements Modal](/images/platform/dynamic-secrets/modify-redis-statement.png) - - Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. + + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. - Allowed template variables are - - `{{randomUsername}}`: Random username string - - `{{unixTimestamp}}`: Current Unix timestamp + Allowed template variables are + - `{{randomUsername}}`: Random username string + - `{{unixTimestamp}}`: Current Unix timestamp + - `{{identity.name}}`: Name of the identity that is generating the secret + - `{{random N}}`: Random string of N characters + + Allowed template functions are + - `truncate`: Truncates a string to a specified length + - `replace`: Replaces a substring with another value + + Examples: + ``` + {{randomUsername}} // 3POnzeFyK9gW2nioK0q2gMjr6CZqsRiX + {{unixTimestamp}} // 17490641580 + {{identity.name}} // testuser + {{random-5}} // x9k2m + {{truncate identity.name 4}} // test + {{replace identity.name 'user' 'replace'}} // testreplace + ``` If you want to provide specific privileges for the generated dynamic credentials, you can modify the Redis statement to your needs. This is useful if you want to only give access to a specific table(s). diff --git a/docs/documentation/platform/dynamic-secrets/sap-ase.mdx b/docs/documentation/platform/dynamic-secrets/sap-ase.mdx index 2737ab084..6da572f35 100644 --- a/docs/documentation/platform/dynamic-secrets/sap-ase.mdx +++ b/docs/documentation/platform/dynamic-secrets/sap-ase.mdx @@ -64,13 +64,29 @@ The Infisical SAP ASE dynamic secret allows you to generate SAP ASE database cre ![Modify SQL Statements Modal](../../../images/platform/dynamic-secrets/sap-ase/dynamic-secret-sap-ase-statements.png) - - Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. + + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. - Allowed template variables are - - `{{randomUsername}}`: Random username string - - `{{unixTimestamp}}`: Current Unix timestamp - + Allowed template variables are + - `{{randomUsername}}`: Random username string + - `{{unixTimestamp}}`: Current Unix timestamp + - `{{identity.name}}`: Name of the identity that is generating the secret + - `{{random N}}`: Random string of N characters + + Allowed template functions are + - `truncate`: Truncates a string to a specified length + - `replace`: Replaces a substring with another value + + Examples: + ``` + {{randomUsername}} // 3POnzeFyK9gW2nioK0q2gMjr6CZqsRiX + {{unixTimestamp}} // 17490641580 + {{identity.name}} // testuser + {{random-5}} // x9k2m + {{truncate identity.name 4}} // test + {{replace identity.name 'user' 'replace'}} // testreplace + ``` + If you want to provide specific privileges for the generated dynamic credentials, you can modify the SQL statement to your needs. diff --git a/docs/documentation/platform/dynamic-secrets/sap-hana.mdx b/docs/documentation/platform/dynamic-secrets/sap-hana.mdx index 597d69803..8ccd842f2 100644 --- a/docs/documentation/platform/dynamic-secrets/sap-hana.mdx +++ b/docs/documentation/platform/dynamic-secrets/sap-hana.mdx @@ -64,12 +64,28 @@ The Infisical SAP HANA dynamic secret allows you to generate SAP HANA database c ![Modify SQL Statements Modal](../../../images/platform/dynamic-secrets/modify-sap-hana-sql-statements.png) - - Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. + + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. - Allowed template variables are - - `{{randomUsername}}`: Random username string - - `{{unixTimestamp}}`: Current Unix timestamp + Allowed template variables are + - `{{randomUsername}}`: Random username string + - `{{unixTimestamp}}`: Current Unix timestamp + - `{{identity.name}}`: Name of the identity that is generating the secret + - `{{random N}}`: Random string of N characters + + Allowed template functions are + - `truncate`: Truncates a string to a specified length + - `replace`: Replaces a substring with another value + + Examples: + ``` + {{randomUsername}} // 3POnzeFyK9gW2nioK0q2gMjr6CZqsRiX + {{unixTimestamp}} // 17490641580 + {{identity.name}} // testuser + {{random-5}} // x9k2m + {{truncate identity.name 4}} // test + {{replace identity.name 'user' 'replace'}} // testreplace + ``` diff --git a/docs/documentation/platform/dynamic-secrets/snowflake.mdx b/docs/documentation/platform/dynamic-secrets/snowflake.mdx index 86378bbf6..f0bbaa08c 100644 --- a/docs/documentation/platform/dynamic-secrets/snowflake.mdx +++ b/docs/documentation/platform/dynamic-secrets/snowflake.mdx @@ -78,12 +78,28 @@ Infisical's Snowflake dynamic secrets allow you to generate Snowflake user crede ![Modify SQL Statements Modal](/images/platform/dynamic-secrets/snowflake/dynamic-secret-snowflake-sql-statements.png) - Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. + Specifies a template for generating usernames. This field allows customization of how usernames are automatically created. - Allowed template variables are - - `{{randomUsername}}`: Random username string - - `{{unixTimestamp}}`: Current Unix timestamp - + Allowed template variables are + - `{{randomUsername}}`: Random username string + - `{{unixTimestamp}}`: Current Unix timestamp + - `{{identity.name}}`: Name of the identity that is generating the secret + - `{{random N}}`: Random string of N characters + + Allowed template functions are + - `truncate`: Truncates a string to a specified length + - `replace`: Replaces a substring with another value + + Examples: + ``` + {{randomUsername}} // 3POnzeFyK9gW2nioK0q2gMjr6CZqsRiX + {{unixTimestamp}} // 17490641580 + {{identity.name}} // testuser + {{random-5}} // x9k2m + {{truncate identity.name 4}} // test + {{replace identity.name 'user' 'replace'}} // testreplace + ``` + If you want to provide specific privileges for the generated dynamic credentials, you can modify the SQL diff --git a/docs/documentation/platform/gateways/gateway-security.mdx b/docs/documentation/platform/gateways/gateway-security.mdx index 83490fd4d..93a7f662f 100644 --- a/docs/documentation/platform/gateways/gateway-security.mdx +++ b/docs/documentation/platform/gateways/gateway-security.mdx @@ -89,22 +89,3 @@ The relay system provides secure tunneling: - Gateways only accept connections to approved resources - Each connection requires explicit project authorization - Resources remain private to their assigned organization - -## Security Measures - -### Certificate Lifecycle -- Certificates have limited validity periods -- Automatic certificate rotation -- Immediate certificate revocation capabilities - -### Monitoring and Verification -1. **Continuous Verification**: - - Regular heartbeat checks - - Certificate chain validation - - Connection state monitoring - -2. **Security Controls**: - - Automatic connection termination on verification failure - - Audit logging of all access attempts - - Machine identity based authentication - diff --git a/docs/documentation/platform/gateways/networking.mdx b/docs/documentation/platform/gateways/networking.mdx new file mode 100644 index 000000000..6acdc1993 --- /dev/null +++ b/docs/documentation/platform/gateways/networking.mdx @@ -0,0 +1,168 @@ +--- +title: "Networking" +description: "Network configuration and firewall requirements for Infisical Gateway" +--- + +The Infisical Gateway requires outbound network connectivity to establish secure communication with Infisical's relay infrastructure. +This page outlines the required ports, protocols, and firewall configurations needed for optimal gateway usage. + +## Network Architecture + +The gateway uses a relay-based architecture to establish secure connections: + +1. **Gateway** connects outbound to **Relay Servers** using UDP/QUIC protocol +2. **Relay Servers** facilitate secure communication between Gateway and Infisical Cloud +3. All traffic is end-to-end encrypted using mutual TLS over QUIC + +## Required Network Connectivity + +### Outbound Connections (Required) + +The gateway requires the following outbound connectivity: + +| Protocol | Destination | Ports | Purpose | +|----------|-------------|-------|---------| +| UDP | Relay Servers | 49152-65535 | Allocated relay communication (TLS) | +| TCP | app.infisical.com / eu.infisical.com | 443 | API communication and relay allocation | + +### Relay Server IP Addresses + +Your firewall must allow outbound connectivity to the following Infisical relay servers on dynamically allocated ports. + + + + ``` + 54.235.197.91:49152-65535 + 18.215.196.229:49152-65535 + 3.222.120.233:49152-65535 + 34.196.115.157:49152-65535 + ``` + + + ``` + 3.125.237.40:49152-65535 + 52.28.157.98:49152-65535 + 3.125.176.90:49152-65535 + ``` + + + Please contact your Infisical account manager for dedicated relay server IP addresses. + + + + + These IP addresses are static and managed by Infisical. Any changes will be communicated with 60-day advance notice. + + +## Protocol Details + +### QUIC over UDP + +The gateway uses QUIC (Quick UDP Internet Connections) for primary communication: + +- **Port 5349**: STUN/TURN over TLS (secure relay communication) +- **Built-in features**: Connection migration, multiplexing, reduced latency +- **Encryption**: TLS 1.3 with certificate pinning + +## Understanding Firewall Behavior with UDP + +Unlike TCP connections, UDP is a stateless protocol, and depending on your organization's firewall configuration, you may need to adjust network rules accordingly. +When the gateway sends UDP packets to a relay server, the return responses need to be allowed back through the firewall. +Modern firewalls handle this through "connection tracking" (also called "stateful inspection"), but the behavior can vary depending on your firewall configuration. + + +### Connection Tracking + +Modern firewalls automatically track UDP connections and allow return responses. This is the preferred configuration as it: +- Automatically handles return responses +- Reduces firewall rule complexity +- Avoids the need for manual IP whitelisting + +In the event that your firewall does not support connection tracking, you will need to whitelist the relay IPs to explicitly define return traffic manually. + +## Common Network Scenarios + +### Corporate Firewalls + +For corporate environments with strict egress filtering: + +1. **Whitelist relay IP addresses** (listed above) +2. **Allow UDP port 5349** outbound +3. **Configure connection tracking** for UDP return traffic +4. **Allow ephemeral port range** 49152-65535 for return traffic if connection tracking is disabled + +### Cloud Environments (AWS/GCP/Azure) + +Configure security groups to allow: +- **Outbound UDP** to relay IPs on port 5349 +- **Outbound HTTPS** to app.infisical.com/eu.infisical.com on port 443 +- **Inbound UDP** on ephemeral ports (if not using stateful rules) + +## Frequently Asked Questions + + +The gateway is designed to handle network interruptions gracefully: + +- **Automatic reconnection**: The gateway will automatically attempt to reconnect to relay servers every 5 seconds if the connection is lost +- **Connection retry logic**: Built-in retry mechanisms handle temporary network outages without manual intervention +- **Multiple relay servers**: If one relay server is unavailable, the gateway can connect to alternative relay servers +- **Persistent sessions**: Existing connections are maintained where possible during brief network interruptions +- **Graceful degradation**: The gateway logs connection issues and continues attempting to restore connectivity + +No manual intervention is typically required during network interruptions. + + + +QUIC (Quick UDP Internet Connections) provides several advantages over traditional TCP for gateway communication: + +- **Faster connection establishment**: QUIC combines transport and security handshakes, reducing connection setup time +- **Built-in encryption**: TLS 1.3 is integrated into the protocol, ensuring all traffic is encrypted by default +- **Connection migration**: QUIC connections can survive IP address changes (useful for NAT rebinding) +- **Reduced head-of-line blocking**: Multiple data streams can be multiplexed without blocking each other +- **Better performance over unreliable networks**: Advanced congestion control and packet loss recovery +- **Lower latency**: Optimized for real-time communication between gateway and cloud services + +While TCP is stateful and easier for firewalls to track, QUIC's performance benefits outweigh the additional firewall configuration requirements. + + + +No inbound ports need to be opened. The gateway only makes outbound connections: + +- **Outbound UDP** to relay servers on ports 49152-65535 +- **Outbound HTTPS** to Infisical API endpoints +- **Return responses** are handled by connection tracking or explicit IP whitelisting + +This design maintains security by avoiding the need for inbound firewall rules that could expose your network to external threats. + + + +If your firewall has strict UDP restrictions: + +1. **Work with your network team** to allow outbound UDP to the specific relay IP addresses +2. **Use explicit IP whitelisting** if connection tracking is disabled +3. **Consider network policy exceptions** for the gateway host +4. **Monitor firewall logs** to identify which specific rules are blocking traffic + +The gateway requires UDP connectivity to function - TCP-only configurations are not supported. + + + +The gateway connects to **one relay server at a time**: + +- **Single active connection**: Only one relay connection is established per gateway instance +- **Automatic failover**: If the current relay becomes unavailable, the gateway will connect to an alternative relay +- **Load distribution**: Different gateway instances may connect to different relay servers for load balancing +- **No manual selection**: The Infisical API automatically assigns the optimal relay server based on availability and proximity + +You should whitelist all relay IP addresses to ensure proper failover functionality. + + +No, relay servers cannot decrypt any traffic passing through them: + +- **End-to-end encryption**: All traffic between the gateway and Infisical Cloud is encrypted using mutual TLS with certificate pinning +- **Relay acts as a tunnel**: The relay server only forwards encrypted packets - it has no access to encryption keys +- **No data storage**: Relay servers do not store any traffic or network-identifiable information +- **Certificate isolation**: Each organization has its own private PKI system, ensuring complete tenant isolation + +The relay infrastructure is designed as a secure forwarding mechanism, similar to a VPN tunnel, where the relay provider cannot see the contents of the traffic flowing through it. + \ No newline at end of file diff --git a/docs/documentation/platform/gateways/overview.mdx b/docs/documentation/platform/gateways/overview.mdx index ae4a3c7ad..127e544b7 100644 --- a/docs/documentation/platform/gateways/overview.mdx +++ b/docs/documentation/platform/gateways/overview.mdx @@ -32,7 +32,7 @@ For detailed installation instructions, refer to the Infisical [CLI Installation To function, the Gateway must authenticate with Infisical. This requires a machine identity configured with the appropriate permissions to create and manage a Gateway. Once authenticated, the Gateway establishes a secure connection with Infisical to allow your private resources to be reachable. -### Deployment process +### Get started @@ -89,18 +89,208 @@ Once authenticated, the Gateway establishes a secure connection with Infisical t helm repo update ``` - ### Create a Kubernetes Secret with the gateway token + ### Create a Kubernetes Secret containing gateway environment variables - Create a new Kubernetes secret containing the gateway token as the `TOKEN` key. You can optionally also set the `INFISICAL_API_URL` key to your Infisical instance URL. By default, `INFISICAL_API_URL` is set to `https://app.infisical.com`. + The gateway supports all identity authentication methods through the use of environment variables. + The environment variables must be set in the `infisical-gateway-environment` Kubernetes secret. - ```bash - kubectl create secret generic infisical-gateway-environment --from-literal=TOKEN= - ``` - - - The secret name is `infisical-gateway-environment` by default. The `TOKEN` key is required, and the `INFISICAL_API_URL` key is optional. - + #### Supported authentication methods + + + + The Universal Auth method is a simple and secure way to authenticate with Infisical. It requires a client ID and a client secret to authenticate with Infisical. + + + + + Your machine identity client ID. + + + Your machine identity client secret. + + + The authentication method to use. Must be `universal-auth` when using Universal Auth. + + + + + ```bash + kubectl create secret generic infisical-gateway-environment --from-literal=INFISICAL_AUTH_METHOD=universal-auth --from-literal=INFISICAL_UNIVERSAL_AUTH_CLIENT_ID= --from-literal=INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET= + ``` + + + + The Native Kubernetes method is used to authenticate with Infisical when running in a Kubernetes environment. It requires a service account token to authenticate with Infisical. + + + + + Your machine identity ID. + + + Path to the Kubernetes service account token to use. Default: `/var/run/secrets/kubernetes.io/serviceaccount/token`. + + + The authentication method to use. Must be `kubernetes` when using Native Kubernetes. + + + + + + ```bash + kubectl create secret generic infisical-gateway-environment --from-literal=INFISICAL_AUTH_METHOD=kubernetes --from-literal=INFISICAL_MACHINE_IDENTITY_ID= + ``` + + + + The Native Azure method is used to authenticate with Infisical when running in an Azure environment. + + + + + Your machine identity ID. + + + The authentication method to use. Must be `azure` when using Native Azure. + + + + + ```bash + kubectl create secret generic infisical-gateway-environment --from-literal=INFISICAL_AUTH_METHOD=azure --from-literal=INFISICAL_MACHINE_IDENTITY_ID= + ``` + + + The Native GCP ID Token method is used to authenticate with Infisical when running in a GCP environment. + + + + + Your machine identity ID. + + + The authentication method to use. Must be `gcp-id-token` when using Native GCP ID Token. + + + + + ```bash + kubectl create secret generic infisical-gateway-environment --from-literal=INFISICAL_AUTH_METHOD=gcp-id-token --from-literal=INFISICAL_MACHINE_IDENTITY_ID= + ``` + + + + The GCP IAM method is used to authenticate with Infisical with a GCP service account key. + + + + + Your machine identity ID. + + + Path to your GCP service account key file _(Must be in JSON format!)_ + + + The authentication method to use. Must be `gcp-iam` when using GCP IAM. + + + + + ```bash + kubectl create secret generic infisical-gateway-environment --from-literal=INFISICAL_AUTH_METHOD=gcp-iam --from-literal=INFISICAL_MACHINE_IDENTITY_ID= --from-literal=INFISICAL_GCP_SERVICE_ACCOUNT_KEY_FILE_PATH= + ``` + + + + + The AWS IAM method is used to authenticate with Infisical with an AWS IAM role while running in an AWS environment like EC2, Lambda, etc. + + + + + Your machine identity ID. + + + The authentication method to use. Must be `aws-iam` when using Native AWS IAM. + + + + + ```bash + kubectl create secret generic infisical-gateway-environment --from-literal=INFISICAL_AUTH_METHOD=aws-iam --from-literal=INFISICAL_MACHINE_IDENTITY_ID= + ``` + + + + The OIDC Auth method is used to authenticate with Infisical via identity tokens with OIDC. + + + + + Your machine identity ID. + + + The OIDC JWT from the identity provider. + + + The authentication method to use. Must be `oidc-auth` when using OIDC Auth. + + + + + ```bash + kubectl create secret generic infisical-gateway-environment --from-literal=INFISICAL_AUTH_METHOD=oidc-auth --from-literal=INFISICAL_MACHINE_IDENTITY_ID= --from-literal=INFISICAL_JWT= + ``` + + + + The JWT Auth method is used to authenticate with Infisical via a JWT token. + + + + + The JWT token to use for authentication. + + + Your machine identity ID. + + + The authentication method to use. Must be `jwt-auth` when using JWT Auth. + + + + + ```bash + kubectl create secret generic infisical-gateway-environment --from-literal=INFISICAL_AUTH_METHOD=jwt-auth --from-literal=INFISICAL_JWT= --from-literal=INFISICAL_MACHINE_IDENTITY_ID= + ``` + + + You can use the `INFISICAL_TOKEN` environment variable to authenticate with Infisical with a raw machine identity access token. + + + + + The machine identity access token to use for authentication. + + + + + ```bash + kubectl create secret generic infisical-gateway-environment --from-literal=INFISICAL_TOKEN= + ``` + + + + + #### Other environment variables + + + + The API URL to use for the gateway. By default, `INFISICAL_API_URL` is set to `https://app.infisical.com`. + + + ### Install the Infisical Gateway Helm Chart ```bash @@ -128,7 +318,7 @@ Once authenticated, the Gateway establishes a secure connection with Infisical t - + For development or testing, you can run the Gateway directly. Log in with your machine identity and start the Gateway in one command: ```bash infisical gateway --token $(infisical login --method=universal-auth --client-id=<> --client-secret=<> --plain) diff --git a/docs/documentation/platform/identities/kubernetes-auth.mdx b/docs/documentation/platform/identities/kubernetes-auth.mdx index 9daff1e81..e357ba75e 100644 --- a/docs/documentation/platform/identities/kubernetes-auth.mdx +++ b/docs/documentation/platform/identities/kubernetes-auth.mdx @@ -52,10 +52,11 @@ Infisical is able to authenticate and interact with the TokenReview API by using In the following steps, we explore how to create and use identities for your applications in Kubernetes to access the Infisical API using the Kubernetes Auth authentication method. + + - - + **When to use this option**: Choose this approach when you want centralized authentication management. Only one service account needs special permissions, and your application service accounts remain unchanged. @@ -126,41 +127,91 @@ In the following steps, we explore how to create and use identities for your app ``` Keep this JWT token handy as you will need it for the **Token Reviewer JWT** field when configuring the Kubernetes Auth authentication method for the identity in step 2. + - - + + + **When to use this option**: Choose this approach to eliminate long-lived tokens. This option simplifies Infisical configuration but requires each application service account to have elevated permissions. + - - **When to use this option**: Choose this approach to eliminate long-lived tokens. This option simplifies Infisical configuration but requires each application service account to have elevated permissions. - + The self-validation method eliminates the need for a separate long-lived reviewer JWT by using the same token for both authentication and validation. Instead of creating a dedicated reviewer service account, you'll grant the necessary permissions to each application service account. - The self-validation method eliminates the need for a separate long-lived reviewer JWT by using the same token for both authentication and validation. Instead of creating a dedicated reviewer service account, you'll grant the necessary permissions to each application service account. + For each service account that needs to authenticate with Infisical, add the `system:auth-delegator` role: - For each service account that needs to authenticate with Infisical, add the `system:auth-delegator` role: + ```yaml client-role-binding.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: infisical-client-binding-[your-app-name] + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator + subjects: + - kind: ServiceAccount + name: [your-app-service-account] + namespace: [your-app-namespace] + ``` - ```yaml client-role-binding.yaml - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: infisical-client-binding-[your-app-name] - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator - subjects: - - kind: ServiceAccount - name: [your-app-service-account] - namespace: [your-app-namespace] - ``` + ``` + kubectl apply -f client-role-binding.yaml + ``` - ``` - kubectl apply -f client-role-binding.yaml - ``` + When configuring Kubernetes Auth in Infisical, leave the **Token Reviewer JWT** field empty. Infisical will use the client's own token for validation. + + + + **When to use this option**: Choose this approach when you have a gateway deployed in your Kubernetes Cluster and wish to eliminate long-lived tokens. This approach simplifies Infisical Kubernetes Auth configuration, and only one service account will need to have the elevated `system:auth-delegator` ClusterRole binding. + - When configuring Kubernetes Auth in Infisical, leave the **Token Reviewer JWT** field empty. Infisical will use the client's own token for validation. - - - + + **Note:** Gateway is a paid feature. - **Infisical Cloud users:** Gateway is + available under the **Enterprise Tier**. - **Self-Hosted Infisical:** Please + contact [sales@infisical.com](mailto:sales@infisical.com) to purchase an + enterprise license. + + + + + To deploy a gateway in your Kubernetes cluster, follow our [Gateway deployment guide using helm](/documentation/platform/gateways/overview). + + + + To grant the gateway the `system:auth-delegator` ClusterRole binding, you can use the following command: + + ```yaml gateway-role-binding.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: infisical-token-reviewer-role-binding + namespace: default # Replace with your namespace if not default + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator + subjects: + - kind: ServiceAccount + name: infisical-gateway # The name of the gateway service account + namespace: default # Replace with your namespace if not default + ``` + + ```bash + kubectl apply -f gateway-role-binding.yaml + ``` + + + The gateway service account name is `infisical-gateway` by default if deployed using Helm. + + + + + To configure your Kubernetes Auth method to use the gateway as the token reviewer, set the `Review Method` to "Gateway as Reviewer", and select the gateway you want to use as the token reviewer. + + ![identities organization create kubernetes auth method](/images/platform/identities/identities-kubernetes-auth-gateway-as-reviewer.png) + + + + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. diff --git a/docs/documentation/platform/identities/oidc-auth/azure.mdx b/docs/documentation/platform/identities/oidc-auth/azure.mdx new file mode 100644 index 000000000..a9f244794 --- /dev/null +++ b/docs/documentation/platform/identities/oidc-auth/azure.mdx @@ -0,0 +1,157 @@ +--- +title: Azure +description: "Learn how to authenticate Azure pipelines with Infisical using OpenID Connect (OIDC)." +--- + +**OIDC Auth** is a platform-agnostic JWT-based authentication method that can be used to authenticate from any platform or environment using an identity provider with OpenID Connect. + +## Diagram + +The following sequence diagram illustrates the OIDC Auth workflow for authenticating Azure pipelines with Infisical. + +```mermaid +sequenceDiagram + participant Client as Azure Pipeline + participant Idp as Identity Provider + participant Infis as Infisical + + Client->>Idp: Step 1: Request identity token + Idp-->>Client: Return JWT with verifiable claims + + Note over Client,Infis: Step 2: Login Operation + Client->>Infis: Send signed JWT to /api/v1/auth/oidc-auth/login + + Note over Infis,Idp: Step 3: Query verification + Infis->>Idp: Request JWT public key using OIDC Discovery + Idp-->>Infis: Return public key + + Note over Infis: Step 4: JWT validation + Infis->>Client: Return short-lived access token + + Note over Client,Infis: Step 5: Access Infisical API with Token + Client->>Infis: Make authenticated requests using the short-lived access token +``` + +## Concept + +At a high-level, Infisical authenticates a client by verifying the JWT and checking that it meets specific requirements (e.g. it is issued by a trusted identity provider) at the `/api/v1/auth/oidc-auth/login` endpoint. If successful, +then Infisical returns a short-lived access token that can be used to make authenticated requests to the Infisical API. + +To be more specific: + +1. The Azure pipeline requests an identity token from Azure's identity provider. +2. The fetched identity token is sent to Infisical at the `/api/v1/auth/oidc-auth/login` endpoint. +3. Infisical fetches the public key that was used to sign the identity token from Azure's identity provider using OIDC Discovery. +4. Infisical validates the JWT using the public key provided by the identity provider and checks that the subject, audience, and claims of the token matches with the set criteria. +5. If all is well, Infisical returns a short-lived access token that the Azure pipeline can use to make authenticated requests to the Infisical API. + + + Infisical needs network-level access to Azure's identity provider endpoints. + + +## Guide + +In the following steps, we explore how to create and use identities to access the Infisical API using the OIDC Auth authentication method. + + + + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. + + ![identities organization](/images/platform/identities/identities-org.png) + + When creating an identity, you specify an organization level [role](/documentation/platform/role-based-access-controls) for it to assume; you can configure roles in Organization Settings > Access Control > Organization Roles. + + ![identities organization create](/images/platform/identities/identities-org-create.png) + + Now input a few details for your new identity. Here's some guidance for each field: + + - Name (required): A friendly name for the identity. + - Role (required): A role from the **Organization Roles** tab for the identity to assume. The organization role assigned will determine what organization level resources this identity can have access to. + + Once you've created an identity, you'll be redirected to a page where you can manage the identity. + + ![identities page](/images/platform/identities/identities-page.png) + + Since the identity has been configured with Universal Auth by default, you should re-configure it to use OIDC Auth instead. To do this, press to edit the **Authentication** section, + remove the existing Universal Auth configuration, and add a new OIDC Auth configuration onto the identity. + + ![identities page remove default auth](/images/platform/identities/identities-page-remove-default-auth.png) + + ![identities create oidc auth method](/images/platform/identities/identities-org-create-oidc-auth-method.png) + + Restrict access by configuring the Subject, Audiences, and Claims fields + + Here's some more guidance on each field: + -
**OIDC Discovery URL**: The URL used to retrieve the OpenID Connect configuration from the identity provider. This is used to fetch the public keys needed to verify the JWT. For Azure, set this to `https://login.microsoftonline.com/{tenant-id}/v2.0` (replace `{tenant-id}` with your Azure AD tenant ID).
+ -
**Issuer**: The value of the `iss` claim that the token must match. For Azure, this should be `https://login.microsoftonline.com/{tenant-id}/v2.0`.
+ - **Subject**: This must match the `sub` claim in the JWT. + - **Audiences**: Values that must match the `aud` claim. + - **Claims**: Additional claims that must be present. Refer to [Azure DevOps docs](https://learn.microsoft.com/en-us/azure/devops/pipelines/library/connect-to-azure?view=azure-devops#workload-identity-federation) for available claims. + - **Access Token TTL**: Lifetime of the issued token (in seconds), e.g., `2592000` (30 days) + - **Access Token Max TTL**: Maximum allowed lifetime of the token + - **Access Token Max Number of Uses**: Max times the token can be used (`0` = unlimited) + - **Access Token Trusted IPs**: List of allowed IP ranges (defaults to `0.0.0.0/0`) + + If you are unsure about what to configure for the subject, audience, and claims fields, you can inspect the JWT token from your Azure DevOps pipeline by adding a debug step that outputs the token claims. + The `subject`, `audiences`, and `claims` fields support glob pattern matching; however, we highly recommend using hardcoded values whenever possible. + + + To enable the identity to access project-level resources such as secrets within a specific project, you should add it to that project. + + To do this, head over to the project you want to add the identity to and go to Project Settings > Access Control > Machine Identities and press **Add identity**. + + Next, select the identity you want to add to the project and the project level role you want to allow it to assume. The project role assigned will determine what project level resources this identity can have access to. + + ![identities project](/images/platform/identities/identities-project.png) + + ![identities project create](/images/platform/identities/identities-project-create.png) + + + In Azure DevOps, to authenticate with Infisical using OIDC, you must configure a service connection that enables workload identity federation. + + Once set up, the OIDC token can be fetched automatically within the pipeline job context. Here's an example: + + ```yaml + trigger: + - main + + pool: + vmImage: ubuntu-latest + + steps: + - task: AzureCLI@2 + displayName: 'Retrieve secrets from Infisical using OIDC' + inputs: + azureSubscription: 'your-azure-service-connection-name' + scriptType: 'bash' + scriptLocation: 'inlineScript' + addSpnToEnvironment: true + inlineScript: | + # Get OIDC access token + OIDC_TOKEN=$(az account get-access-token --resource "api://AzureADTokenExchange" --query accessToken -o tsv) + + [ -z "$OIDC_TOKEN" ] && { echo "Failed to get access token"; exit 1; } + + # Exchange for Infisical access token + ACCESS_TOKEN=$(curl -s -X POST "/api/v1/auth/oidc-auth/login" \ + -H "Content-Type: application/json" \ + -d "{\"identityId\":\"{your-identity-id}\",\"jwt\":\"$OIDC_TOKEN\"}" \ + | jq -r '.accessToken') + + # Fetch secrets + curl -s -H "Authorization: Bearer $ACCESS_TOKEN" \ + "/api/v3/secrets/raw?environment={your-environment-slug}&workspaceSlug={your-workspace-slug}" + ``` + + Make sure the service connection is properly configured for workload identity federation and linked to your Azure AD app registration with appropriate claims. + + + Each identity access token has a time-to-live (TTL) which you can infer from the response of the login operation; + the default TTL is `7200` seconds which can be adjusted. + + If an identity access token expires, it can no longer authenticate with the Infisical API. In this case, + a new access token should be obtained by performing another login operation. + + + + diff --git a/docs/documentation/platform/identities/oidc-auth/spire.mdx b/docs/documentation/platform/identities/oidc-auth/spire.mdx new file mode 100644 index 000000000..b402a1d10 --- /dev/null +++ b/docs/documentation/platform/identities/oidc-auth/spire.mdx @@ -0,0 +1,177 @@ +--- +title: SPIFFE/SPIRE +description: "Learn how to authenticate SPIRE workloads with Infisical using OpenID Connect (OIDC)." +--- + +**OIDC Auth** is a platform-agnostic JWT-based authentication method that can be used to authenticate from any platform or environment using an identity provider with OpenID Connect. + +## Diagram + +The following sequence diagram illustrates the OIDC Auth workflow for authenticating SPIRE workloads with Infisical. + +```mermaid +sequenceDiagram + participant Client as SPIRE Workload + participant Agent as SPIRE Agent + participant Server as SPIRE Server + participant Infis as Infisical + + Client->>Agent: Step 1: Request JWT-SVID + Agent->>Server: Validate workload and fetch signing key + Server-->>Agent: Return signing material + Agent-->>Client: Return JWT-SVID with verifiable claims + + Note over Client,Infis: Step 2: Login Operation + Client->>Infis: Send JWT-SVID to /api/v1/auth/oidc-auth/login + + Note over Infis,Server: Step 3: Query verification + Infis->>Server: Request JWT public key using OIDC Discovery + Server-->>Infis: Return public key + + Note over Infis: Step 4: JWT validation + Infis->>Client: Return short-lived access token + + Note over Client,Infis: Step 5: Access Infisical API with Token + Client->>Infis: Make authenticated requests using the short-lived access token +``` + +## Concept + +At a high-level, Infisical authenticates a SPIRE workload by verifying the JWT-SVID and checking that it meets specific requirements (e.g. it is issued by a trusted SPIRE server) at the `/api/v1/auth/oidc-auth/login` endpoint. If successful, +then Infisical returns a short-lived access token that can be used to make authenticated requests to the Infisical API. + +To be more specific: + +1. The SPIRE workload requests a JWT-SVID from the local SPIRE Agent. +2. The SPIRE Agent validates the workload's identity and requests signing material from the SPIRE Server. +3. The SPIRE Agent returns a JWT-SVID containing the workload's SPIFFE ID and other claims. +4. The JWT-SVID is sent to Infisical at the `/api/v1/auth/oidc-auth/login` endpoint. +5. Infisical fetches the public key that was used to sign the JWT-SVID from the SPIRE Server using OIDC Discovery. +6. Infisical validates the JWT-SVID using the public key provided by the SPIRE Server and checks that the subject, audience, and claims of the token matches with the set criteria. +7. If all is well, Infisical returns a short-lived access token that the workload can use to make authenticated requests to the Infisical API. + +Infisical needs network-level access to the SPIRE Server's OIDC Discovery endpoint. + +## Prerequisites + +Before following this guide, ensure you have: + +- A running SPIRE deployment with both SPIRE Server and SPIRE Agent configured +- OIDC Discovery Provider deployed alongside your SPIRE Server +- Workload registration entries created in SPIRE for the workloads that need to access Infisical +- Network connectivity between Infisical and your OIDC Discovery Provider endpoint + +For detailed SPIRE setup instructions, refer to the [SPIRE documentation](https://spiffe.io/docs/latest/spire-about/). + +## OIDC Discovery Provider Setup + +To enable JWT-SVID verification with Infisical, you need to deploy the OIDC Discovery Provider alongside your SPIRE Server. The OIDC Discovery Provider runs as a separate service that exposes the necessary OIDC endpoints. + +In Kubernetes deployments, this is typically done by adding an `oidc-discovery-provider` container to your SPIRE Server StatefulSet: + +```yaml +- name: spire-oidc + image: ghcr.io/spiffe/oidc-discovery-provider:1.12.2 + args: + - -config + - /run/spire/oidc/config/oidc-discovery-provider.conf + ports: + - containerPort: 443 + name: spire-oidc-port +``` + +The OIDC Discovery Provider will expose the OIDC Discovery endpoint at `https:///.well-known/openid_configuration`, which Infisical will use to fetch the public keys for JWT-SVID verification. + +For detailed setup instructions, refer to the [SPIRE OIDC Discovery Provider documentation](https://github.com/spiffe/spire/tree/main/support/oidc-discovery-provider). + +## Guide + +In the following steps, we explore how to create and use identities to access the Infisical API using the OIDC Auth authentication method with SPIFFE/SPIRE. + + + + To create an identity, head to your Organization Settings > Access Control > Identities and press **Create identity**. + + ![identities organization](/images/platform/identities/identities-org.png) + + When creating an identity, you specify an organization level [role](/documentation/platform/role-based-access-controls) for it to assume; you can configure roles in Organization Settings > Access Control > Organization Roles. + + ![identities organization create](/images/platform/identities/identities-org-create.png) + + Now input a few details for your new identity. Here's some guidance for each field: + + - Name (required): A friendly name for the identity. + - Role (required): A role from the **Organization Roles** tab for the identity to assume. The organization role assigned will determine what organization level resources this identity can have access to. + + Once you've created an identity, you'll be redirected to a page where you can manage the identity. + + ![identities page](/images/platform/identities/identities-page.png) + + Since the identity has been configured with Universal Auth by default, you should re-configure it to use OIDC Auth instead. To do this, press to edit the **Authentication** section, + remove the existing Universal Auth configuration, and add a new OIDC Auth configuration onto the identity. + + ![identities page remove default auth](/images/platform/identities/identities-page-remove-default-auth.png) + + ![identities create oidc auth method](/images/platform/identities/identities-org-create-oidc-auth-method.png) + + Restrict access by configuring the Subject, Audiences, and Claims fields + + Here's some more guidance on each field: + - OIDC Discovery URL: The URL used to retrieve the OpenID Connect configuration from the SPIRE Server. This will be used to fetch the public key needed for verifying the provided JWT-SVID. This should be set to your SPIRE Server's OIDC Discovery endpoint, typically `https://:/.well-known/openid_configuration` + - Issuer: The unique identifier of the SPIRE Server issuing the JWT-SVID. This value is used to verify the iss (issuer) claim in the JWT-SVID to ensure the token is issued by a trusted SPIRE Server. This should match your SPIRE Server's configured issuer, typically `https://:` + - CA Certificate: The PEM-encoded CA certificate for establishing secure communication with the SPIRE Server endpoints. This should contain the CA certificate that signed your SPIRE Server's TLS certificate. + - Subject: The expected SPIFFE ID that is the subject of the JWT-SVID. The format of the sub field for SPIRE JWT-SVIDs follows the SPIFFE ID format: `spiffe:///`. For example: `spiffe://example.org/workload/api-server` + - Audiences: A list of intended recipients for the JWT-SVID. This value is checked against the aud (audience) claim in the token. When workloads request JWT-SVIDs from SPIRE, they specify an audience (e.g., `infisical` or your service name). Configure this to match what your workloads use. + - Claims: Additional information or attributes that should be present in the JWT-SVID for it to be valid. Standard SPIRE JWT-SVID claims include `sub` (SPIFFE ID), `aud` (audience), `exp` (expiration), and `iat` (issued at). You can also configure custom claims if your SPIRE Server includes additional metadata. + - Access Token TTL (default is `2592000` equivalent to 30 days): The lifetime for an access token in seconds. This value will be referenced at renewal time. + - Access Token Max TTL (default is `2592000` equivalent to 30 days): The maximum lifetime for an access token in seconds. This value will be referenced at renewal time. + - Access Token Max Number of Uses (default is `0`): The maximum number of times that an access token can be used; a value of `0` implies infinite number of uses. + - Access Token Trusted IPs: The IPs or CIDR ranges that access tokens can be used from. By default, each token is given the `0.0.0.0/0`, allowing usage from any network address. + SPIRE JWT-SVIDs contain standard claims like `sub` (SPIFFE ID), `aud` (audience), `exp`, and `iat`. The audience is typically specified when requesting the JWT-SVID (e.g., `spire-agent api fetch jwt -audience infisical`). + The `subject`, `audiences`, and `claims` fields support glob pattern matching; however, we highly recommend using hardcoded SPIFFE IDs whenever possible for better security. + + + To enable the identity to access project-level resources such as secrets within a specific project, you should add it to that project. + + To do this, head over to the project you want to add the identity to and go to Project Settings > Access Control > Machine Identities and press **Add identity**. + + Next, select the identity you want to add to the project and the project level role you want to allow it to assume. The project role assigned will determine what project level resources this identity can have access to. + + ![identities project](/images/platform/identities/identities-project.png) + + ![identities project create](/images/platform/identities/identities-project-create.png) + + + Here's an example of how a workload can use its JWT-SVID to authenticate with Infisical and retrieve secrets: + + ```bash + #!/bin/bash + + # Obtain JWT-SVID from SPIRE Agent + JWT_SVID=$(spire-agent api fetch jwt -audience infisical -socketPath /run/spire/sockets/agent.sock | grep -A1 "token(" | tail -1) + + # Authenticate with Infisical using the JWT-SVID + ACCESS_TOKEN=$(curl -s -X POST \ + -H "Content-Type: application/json" \ + -d "{\"identityId\":\"\",\"jwt\":\"$JWT_SVID\"}" \ + https://app.infisical.com/api/v1/auth/oidc-auth/login | jq -r '.accessToken') + + # Use the access token to retrieve secrets + curl -s -H "Authorization: Bearer $ACCESS_TOKEN" \ + "https://app.infisical.com/api/v3/secrets/raw?workspaceSlug=&environment=&secretPath=/" + ``` + + + Each identity access token has a time-to-live (TTL) which you can infer from the response of the login operation; + the default TTL is `7200` seconds which can be adjusted. + + If an identity access token expires, it can no longer authenticate with the Infisical API. In this case, + a new access token should be obtained by performing another login operation. + + + + JWT-SVIDs from SPIRE have their own expiration time (typically short-lived). Ensure your application handles both JWT-SVID renewal from SPIRE and access token renewal from Infisical appropriately. + + + + \ No newline at end of file diff --git a/docs/documentation/platform/pit-recovery.mdx b/docs/documentation/platform/pit-recovery.mdx index 448faddbb..24d0071a7 100644 --- a/docs/documentation/platform/pit-recovery.mdx +++ b/docs/documentation/platform/pit-recovery.mdx @@ -4,38 +4,130 @@ description: "Learn how to rollback secrets and configurations to any snapshot w --- - Point-in-Time Recovery is a paid feature. - - If you're using Infisical Cloud, then it is available under the **Pro Tier**. If you're self-hosting Infisical, - then you should contact sales@infisical.com to purchase an enterprise license to use it. + Point-in-Time Recovery is a paid feature. If you're using Infisical Cloud, + then it is available under the **Pro Tier**. If you're self-hosting Infisical, + then you should contact sales@infisical.com to purchase an enterprise license + to use it. Infisical's point-in-time recovery functionality allows secrets to be rolled back to any point in time for any given [folder](./folder) or [environment](/documentation/platform/project#project-environments). -Every time a secret is updated, a new snapshot is taken – capturing the state of the folder and environment at that point of time. -## Snapshots + + + ## Understanding Commits -Similar to Git, a commit (also known as snapshot) in Infisical is the state of your project's secrets at a specific point in time scoped to -an environment and [folder](./folder) within it. + Similar to Git, a commit in Infisical represents a snapshot of changes made to your project's resources at a specific point in time. Each commit is scoped to an environment and [folder](./folder) within it. Unlike the legacy snapshot system, the new commits interface provides granular tracking of individual changes, allowing you to see exactly what was modified, added, or removed in each commit. -To view a list of snapshots for the current folder, press the **Commits** button. + ### Accessing Commits -![PIT commits](../../images/platform/pit-recovery/pit-recovery-commits.png) + From your secrets management interface, you can access the commits functionality by clicking the **Commits Button**. This button is located in the top-right area of your secrets view and shows the number of commits for the current folder (e.g., "4 Commits"). -This opens up a sidebar from which you can select to view a particular snapshot: + ![Commits Button](../../images/platform/pit-recovery/pit-recovery-revamp/pit-commits-button.png) -![PIT snapshots](../../images/platform/pit-recovery/pit-recovery-commits-drawer.png) + ### Commits List View -## Rolling back + The commits page displays a comprehensive chronological history of all changes made to your environment and folders: -After pressing on a snapshot from the sidebar, you can view it and roll back the state -of the folder to that point in time by pressing the **Rollback** button. + ![Commits List View](../../images/platform/pit-recovery/pit-recovery-revamp/pit-commits-history.png) -![PIT snapshot](../../images/platform/pit-recovery/pit-recovery-rollback.png) + - **Chronological Sorting**: Commits are grouped by date + - **Commit Information**: Each commit shows: + - Commit message + - Author information + - Relative timestamp + - Unique commit hash identifier + - **Search Functionality**: Use the search bar to quickly find specific commits + - **Sorting Options**: Sort commits by various criteria using the sort controls -Rolling back secrets to a past snapshot creates a creates a snapshot at the top of the stack and updates secret versions. + ### Detailed Commit Inspection - -Rollbacks are localized to not affect other folders within the same environment. This means each [folder](./folder) maintains its own independent history of changes, offering precise and isolated control over rollback actions. -Put differently, every [folder](./folder) possesses a distinct and separate timeline, providing granular control when managing your secrets. - \ No newline at end of file + Clicking on any commit from the list opens a detailed view showing the list of changes made in that commit. + + ![Detailed Commit Inspection](../../images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes.png) + + #### Change Categories + + The commit changes details can be grouped into the following categories: + + **Folder Changes** + - Shows folder additions, modifications, or deletions + - Displays the folder properties changes in JSON format, including: + - Folder name + - Folder description + + **Secret Changes** + - Lists all secrets that were added, updated, or removed + - Shows the complete secret configuration including: + - Secret key and value + - Comments, tags and metadata + - Encoding settings (e.g., skipMultilineEncoding) + - Values are displayed with appropriate masking for security + + **Visual Indicators** + - Green "+" indicators show additions + - Red "-" indicators show deletions + - Modified content shows both old and new states + + ### Restoration Options + + Each commit provides two distinct restoration methods accessible via the **Restore Options** dropdown: + + ![Restore Options](../../images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes-options.png) + + #### Revert changes + This option provides surgical precision for undoing specific modifications: + + - **Granular Control**: Reverts only the specific changes introduced in that individual commit + - **Selective Restoration**: Preserves all other changes made after the commit + - **Targeted Undo**: Perfect for reversing a specific problematic change without affecting other work + - **Minimal Impact**: Only affects the resources that were modified in that particular commit + - **Use Case**: Ideal when you want to undo a specific change while keeping all other modifications intact + + #### Roll back to this commit + This option performs a complete restoration to the selected point in time: + + ![Rollback](../../images/platform/pit-recovery/pit-recovery-revamp/pit-commit-restore.png) + + - **Complete State Restoration**: Returns the entire folder to its exact state at the time of this commit + - **Restore All Child Folders**: If enabled, it'll also restore all nested folders to their exact state at the time of this commit. + - **Destructive Operation**: Discards ALL changes made after the selected commit + - **New Commit Creation**: Creates a new commit representing this rollback operation + - **Use Case**: Ideal when you want to completely undo a series of changes and return to a known good state + + **Warning**: This operation will undo all modifications made after the selected commit, which may include multiple secrets and configuration changes. + + + + + The snapshots interface is deprecated and will be removed in a future version. Please use the new Commits interface for more granular point-in-time recovery operations. + + + ## Snapshots + + Similar to Git, a commit (also known as snapshot) in Infisical is the state of your project's secrets at a specific point in time scoped to + an environment and [folder](./folder) within it. + + To view a list of snapshots for the current folder, press the **Commits** button. + + ![PIT commits](../../images/platform/pit-recovery/pit-recovery-commits.png) + + This opens up a sidebar from which you can select to view a particular snapshot: + + ![PIT snapshots](../../images/platform/pit-recovery/pit-recovery-commits-drawer.png) + + ## Rolling back + + After pressing on a snapshot from the sidebar, you can view it and roll back the state + of the folder to that point in time by pressing the **Rollback** button. + + ![PIT snapshot](../../images/platform/pit-recovery/pit-recovery-rollback.png) + + Rolling back secrets to a past snapshot creates a snapshot at the top of the stack and updates secret versions. + + + Rollbacks are localized to not affect other folders within the same environment. This means each [folder](./folder) maintains its own independent history of changes, offering precise and isolated control over rollback actions. + Put differently, every [folder](./folder) possesses a distinct and separate timeline, providing granular control when managing your secrets. + + + + diff --git a/docs/documentation/platform/webhooks.mdx b/docs/documentation/platform/webhooks.mdx index 92d3ff8b8..e6e71b9b0 100644 --- a/docs/documentation/platform/webhooks.mdx +++ b/docs/documentation/platform/webhooks.mdx @@ -27,7 +27,7 @@ If the signature in the header matches the signature that you generated, then yo ```json { - "event": "secret.modified", + "event": "secrets.modified", "project": { "workspaceId": "the workspace id", "environment": "project environment", diff --git a/docs/documentation/setup/networking.mdx b/docs/documentation/setup/networking.mdx index 4a666b73c..6de27c3c0 100644 --- a/docs/documentation/setup/networking.mdx +++ b/docs/documentation/setup/networking.mdx @@ -4,33 +4,36 @@ sidebarTitle: "Networking" description: "Network configuration details for Infisical Cloud" --- -## Overview - When integrating your infrastructure with Infisical Cloud, you may need to configure network access controls. This page provides the IP addresses that Infisical uses to communicate with your services. -## Egress IP Addresses +## Infisical IP Addresses -Infisical Cloud operates from two regions: US and EU. If your infrastructure has strict network policies, you may need to allow traffic from Infisical by adding the following IP addresses to your ingress rules. These are the egress IPs Infisical uses when making outbound requests to your services. +Infisical Cloud operates from multiple regions. If your infrastructure has strict network policies, you may need to allow traffic from Infisical by adding the following IP addresses to your ingress rules. These are the IP addresses that Infisical uses when making outbound requests to your services. -### US Region + + + ``` + 3.213.63.16 + 54.164.68.7 + ``` + + + + ``` + 3.77.89.19 + 3.125.209.189 + ``` + + + + For dedicated Infisical deployments, please contact your account manager for the specific IP addresses used in your dedicated environment. + + -To allow connections from Infisical US, add these IP addresses to your ingress rules: + +These IP addresses are static and managed by Infisical. Any changes will be communicated with 60-day advance notice. + -- `3.213.63.16` -- `54.164.68.7` +## What These IP Addresses Are Used For -### EU Region - -To allow connections from Infisical EU, add these IP addresses to your ingress rules: - -- `3.77.89.19` -- `3.125.209.189` - -## Common Use Cases - -You may need to allow Infisical’s egress IPs if your services require inbound connections for: - -- Secret rotation - When Infisical needs to send requests to your systems to automatically rotate credentials -- Dynamic secrets - When Infisical generates and manages temporary credentials for your cloud services -- Secret integrations - When syncing secrets with third-party services like Azure Key Vault -- Native authentication with machine identities - When using methods like Kubernetes authentication +These IP addresses represent the source IPs you'll see when Infisical Cloud makes connections to your infrastructure. All outbound traffic from Infisical Cloud originates from these IP addresses, ensuring predictable source IP addresses for your firewall rules. diff --git a/docs/images/app-connections/github-radar/self-hosted-github-radar-step-7.png b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-7.png index 414b0231b..f7646855d 100644 Binary files a/docs/images/app-connections/github-radar/self-hosted-github-radar-step-7.png and b/docs/images/app-connections/github-radar/self-hosted-github-radar-step-7.png differ diff --git a/docs/images/platform/access-controls/abac-policy-k8s-format.png b/docs/images/platform/access-controls/abac-policy-k8s-format.png new file mode 100644 index 000000000..0aff7830a Binary files /dev/null and b/docs/images/platform/access-controls/abac-policy-k8s-format.png differ diff --git a/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-aws-iam-access-key.png b/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-aws-iam-access-key.png new file mode 100644 index 000000000..439208d83 Binary files /dev/null and b/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-aws-iam-access-key.png differ diff --git a/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-aws-iam-assume-role.png b/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-aws-iam-assume-role.png new file mode 100644 index 000000000..e3be4b5f1 Binary files /dev/null and b/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-aws-iam-assume-role.png differ diff --git a/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-aws-iam.png b/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-aws-iam.png deleted file mode 100644 index 0ba6aa172..000000000 Binary files a/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-aws-iam.png and /dev/null differ diff --git a/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-kubernetes-1.png b/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-kubernetes-1.png new file mode 100644 index 000000000..dffbedad3 Binary files /dev/null and b/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-kubernetes-1.png differ diff --git a/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-kubernetes-2.png b/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-kubernetes-2.png new file mode 100644 index 000000000..cc5e56001 Binary files /dev/null and b/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-kubernetes-2.png differ diff --git a/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-kubernetes.png b/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-kubernetes.png deleted file mode 100644 index 011dfadc7..000000000 Binary files a/docs/images/platform/dynamic-secrets/dynamic-secret-setup-modal-kubernetes.png and /dev/null differ diff --git a/docs/images/platform/identities/identities-kubernetes-auth-gateway-as-reviewer.png b/docs/images/platform/identities/identities-kubernetes-auth-gateway-as-reviewer.png new file mode 100644 index 000000000..30ac12545 Binary files /dev/null and b/docs/images/platform/identities/identities-kubernetes-auth-gateway-as-reviewer.png differ diff --git a/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes-options.png b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes-options.png new file mode 100644 index 000000000..60d9dff01 Binary files /dev/null and b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes-options.png differ diff --git a/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes.png b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes.png new file mode 100644 index 000000000..56357e9ef Binary files /dev/null and b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-changes.png differ diff --git a/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-restore.png b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-restore.png new file mode 100644 index 000000000..3ed22c63b Binary files /dev/null and b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commit-restore.png differ diff --git a/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-button.png b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-button.png new file mode 100644 index 000000000..273760ca5 Binary files /dev/null and b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-button.png differ diff --git a/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-history.png b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-history.png new file mode 100644 index 000000000..3832209f4 Binary files /dev/null and b/docs/images/platform/pit-recovery/pit-recovery-revamp/pit-commits-history.png differ diff --git a/docs/images/secret-syncs/gcp-secret-manager/gcp-secret-manager-destination.png b/docs/images/secret-syncs/gcp-secret-manager/gcp-secret-manager-destination.png index c50b6232a..1841b4b6d 100644 Binary files a/docs/images/secret-syncs/gcp-secret-manager/gcp-secret-manager-destination.png and b/docs/images/secret-syncs/gcp-secret-manager/gcp-secret-manager-destination.png differ diff --git a/docs/integrations/secret-syncs/1password.mdx b/docs/integrations/secret-syncs/1password.mdx index a33f54c8d..6e2b96b4a 100644 --- a/docs/integrations/secret-syncs/1password.mdx +++ b/docs/integrations/secret-syncs/1password.mdx @@ -46,7 +46,7 @@ description: "Learn how to configure a 1Password Sync for Infisical." - **Overwrite Destination Secrets**: Removes any secrets at the destination endpoint not present in Infisical. - **Import Secrets (Prioritize Infisical)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Infisical over 1Password when keys conflict. - **Import Secrets (Prioritize 1Password)**: Imports secrets from the destination endpoint before syncing, prioritizing values from 1Password over Infisical when keys conflict. - - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name. + - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name and `{{environment}}` for the environment. We highly recommend using a Key Schema to ensure that Infisical only manages the specific keys you intend, keeping everything else untouched. diff --git a/docs/integrations/secret-syncs/aws-parameter-store.mdx b/docs/integrations/secret-syncs/aws-parameter-store.mdx index 11f0c94ad..abc52d971 100644 --- a/docs/integrations/secret-syncs/aws-parameter-store.mdx +++ b/docs/integrations/secret-syncs/aws-parameter-store.mdx @@ -40,7 +40,7 @@ description: "Learn how to configure an AWS Parameter Store Sync for Infisical." - **Overwrite Destination Secrets**: Removes any secrets at the destination endpoint not present in Infisical. - **Import Secrets (Prioritize Infisical)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Infisical over Parameter Store when keys conflict. - **Import Secrets (Prioritize AWS Parameter Store)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Parameter Store over Infisical when keys conflict. - - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name. + - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name and `{{environment}}` for the environment. We highly recommend using a Key Schema to ensure that Infisical only manages the specific keys you intend, keeping everything else untouched. diff --git a/docs/integrations/secret-syncs/aws-secrets-manager.mdx b/docs/integrations/secret-syncs/aws-secrets-manager.mdx index f7654eeae..91c606b0a 100644 --- a/docs/integrations/secret-syncs/aws-secrets-manager.mdx +++ b/docs/integrations/secret-syncs/aws-secrets-manager.mdx @@ -43,7 +43,7 @@ description: "Learn how to configure an AWS Secrets Manager Sync for Infisical." - **Overwrite Destination Secrets**: Removes any secrets at the destination endpoint not present in Infisical. - **Import Secrets (Prioritize Infisical)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Infisical over Secrets Manager when keys conflict. - **Import Secrets (Prioritize AWS Secrets Manager)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Secrets Manager over Infisical when keys conflict. - - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name. + - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name and `{{environment}}` for the environment. We highly recommend using a Key Schema to ensure that Infisical only manages the specific keys you intend, keeping everything else untouched. diff --git a/docs/integrations/secret-syncs/azure-app-configuration.mdx b/docs/integrations/secret-syncs/azure-app-configuration.mdx index ee47504bc..f4aaa7edd 100644 --- a/docs/integrations/secret-syncs/azure-app-configuration.mdx +++ b/docs/integrations/secret-syncs/azure-app-configuration.mdx @@ -48,7 +48,7 @@ description: "Learn how to configure an Azure App Configuration Sync for Infisic - **Overwrite Destination Secrets**: Removes any secrets at the destination endpoint not present in Infisical. - **Import Secrets (Prioritize Infisical)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Infisical over Secrets Manager when keys conflict. - **Import Secrets (Prioritize Azure App Configuration)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Secrets Manager over Infisical when keys conflict. - - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name. + - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name and `{{environment}}` for the environment. We highly recommend using a Key Schema to ensure that Infisical only manages the specific keys you intend, keeping everything else untouched. diff --git a/docs/integrations/secret-syncs/azure-key-vault.mdx b/docs/integrations/secret-syncs/azure-key-vault.mdx index 609ba8b8d..d19a0162e 100644 --- a/docs/integrations/secret-syncs/azure-key-vault.mdx +++ b/docs/integrations/secret-syncs/azure-key-vault.mdx @@ -51,7 +51,7 @@ description: "Learn how to configure a Azure Key Vault Sync for Infisical." - **Overwrite Destination Secrets**: Removes any secrets at the destination endpoint not present in Infisical. - **Import Secrets (Prioritize Infisical)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Infisical over Secrets Manager when keys conflict. - **Import Secrets (Prioritize Azure Key Vault)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Secrets Manager over Infisical when keys conflict. - - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name. + - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name and `{{environment}}` for the environment. We highly recommend using a Key Schema to ensure that Infisical only manages the specific keys you intend, keeping everything else untouched. diff --git a/docs/integrations/secret-syncs/camunda.mdx b/docs/integrations/secret-syncs/camunda.mdx index df57a5b7d..0e977aa27 100644 --- a/docs/integrations/secret-syncs/camunda.mdx +++ b/docs/integrations/secret-syncs/camunda.mdx @@ -39,7 +39,7 @@ description: "Learn how to configure a Camunda Sync for Infisical." - **Overwrite Destination Secrets**: Removes any secrets at the destination endpoint not present in Infisical. - **Import Secrets (Prioritize Infisical)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Infisical over Camunda when keys conflict. - **Import Secrets (Prioritize Camunda)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Camunda over Infisical when keys conflict. - - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name. + - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name and `{{environment}}` for the environment. We highly recommend using a Key Schema to ensure that Infisical only manages the specific keys you intend, keeping everything else untouched. diff --git a/docs/integrations/secret-syncs/databricks.mdx b/docs/integrations/secret-syncs/databricks.mdx index 225bad5b1..e11537420 100644 --- a/docs/integrations/secret-syncs/databricks.mdx +++ b/docs/integrations/secret-syncs/databricks.mdx @@ -46,7 +46,7 @@ description: "Learn how to configure a Databricks Sync for Infisical." Databricks does not support importing secrets. - - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name. + - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name and `{{environment}}` for the environment. We highly recommend using a Key Schema to ensure that Infisical only manages the specific keys you intend, keeping everything else untouched. diff --git a/docs/integrations/secret-syncs/gcp-secret-manager.mdx b/docs/integrations/secret-syncs/gcp-secret-manager.mdx index ace63787d..0be08a9a9 100644 --- a/docs/integrations/secret-syncs/gcp-secret-manager.mdx +++ b/docs/integrations/secret-syncs/gcp-secret-manager.mdx @@ -34,6 +34,9 @@ description: "Learn how to configure a GCP Secret Manager Sync for Infisical." - **GCP Connection**: The GCP Connection to authenticate with. - **Project**: The GCP project to sync with. + - **Scope**: The GCP project scope that secrets should be synced to: + - **Global**: Secrets will be synced globally; available to all project regions. + - **Region**: Secrets will be synced to the specified region. 5. Configure the **Sync Options** to specify how secrets should be synced, then click **Next**. ![Configure Options](/images/secret-syncs/gcp-secret-manager/gcp-secret-manager-options.png) @@ -42,7 +45,7 @@ description: "Learn how to configure a GCP Secret Manager Sync for Infisical." - **Overwrite Destination Secrets**: Removes any secrets at the destination endpoint not present in Infisical. - **Import Secrets (Prioritize Infisical)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Infisical over GCP Secret Manager when keys conflict. - **Import Secrets (Prioritize GCP Secret Manager)**: Imports secrets from the destination endpoint before syncing, prioritizing values from GCP Secret Manager over Infisical when keys conflict. - - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name. + - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name and `{{environment}}` for the environment. We highly recommend using a Key Schema to ensure that Infisical only manages the specific keys you intend, keeping everything else untouched. diff --git a/docs/integrations/secret-syncs/github.mdx b/docs/integrations/secret-syncs/github.mdx index 7786567cc..14b2d9a7f 100644 --- a/docs/integrations/secret-syncs/github.mdx +++ b/docs/integrations/secret-syncs/github.mdx @@ -62,7 +62,7 @@ description: "Learn how to configure a GitHub Sync for Infisical." GitHub does not support importing secrets. - - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name. + - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name and `{{environment}}` for the environment. We highly recommend using a Key Schema to ensure that Infisical only manages the specific keys you intend, keeping everything else untouched. diff --git a/docs/integrations/secret-syncs/hashicorp-vault.mdx b/docs/integrations/secret-syncs/hashicorp-vault.mdx index 48e4d8dfd..fae2e0962 100644 --- a/docs/integrations/secret-syncs/hashicorp-vault.mdx +++ b/docs/integrations/secret-syncs/hashicorp-vault.mdx @@ -54,7 +54,7 @@ description: "Learn how to configure a Hashicorp Vault Sync for Infisical." - **Overwrite Destination Secrets**: Removes any secrets at the destination endpoint not present in Infisical. - **Import Secrets (Prioritize Infisical)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Infisical over Hashicorp Vault when keys conflict. - **Import Secrets (Prioritize Hashicorp Vault)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Hashicorp Vault over Infisical when keys conflict. - - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name. + - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name and `{{environment}}` for the environment. We highly recommend using a Key Schema to ensure that Infisical only manages the specific keys you intend, keeping everything else untouched. diff --git a/docs/integrations/secret-syncs/humanitec.mdx b/docs/integrations/secret-syncs/humanitec.mdx index ec36bd4da..f252724fb 100644 --- a/docs/integrations/secret-syncs/humanitec.mdx +++ b/docs/integrations/secret-syncs/humanitec.mdx @@ -55,7 +55,7 @@ description: "Learn how to configure a Humanitec Sync for Infisical." Humanitec does not support importing secrets. - - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name. + - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name and `{{environment}}` for the environment. We highly recommend using a Key Schema to ensure that Infisical only manages the specific keys you intend, keeping everything else untouched. diff --git a/docs/integrations/secret-syncs/oci-vault.mdx b/docs/integrations/secret-syncs/oci-vault.mdx index 00b7120e7..396b4d13f 100644 --- a/docs/integrations/secret-syncs/oci-vault.mdx +++ b/docs/integrations/secret-syncs/oci-vault.mdx @@ -57,7 +57,7 @@ description: "Learn how to configure an Oracle Cloud Infrastructure Vault Sync f - **Overwrite Destination Secrets**: Removes any secrets at the destination endpoint not present in Infisical. - **Import Secrets (Prioritize Infisical)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Infisical over OCI Vault when keys conflict. - **Import Secrets (Prioritize OCI Vault)**: Imports secrets from the destination endpoint before syncing, prioritizing values from OCI Vault over Infisical when keys conflict. - - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name. + - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name and `{{environment}}` for the environment. We highly recommend using a Key Schema to ensure that Infisical only manages the specific keys you intend, keeping everything else untouched. diff --git a/docs/integrations/secret-syncs/overview.mdx b/docs/integrations/secret-syncs/overview.mdx index 65ec4e6cf..937c8d826 100644 --- a/docs/integrations/secret-syncs/overview.mdx +++ b/docs/integrations/secret-syncs/overview.mdx @@ -101,6 +101,10 @@ Key Schemas transform your secret keys by applying a prefix, suffix, or format p Any destination secrets which do not match the schema will not get deleted or updated by Infisical. +Key Schemas use handlebars syntax to define dynamic values. Here's a full list of available variables: +- `{{secretKey}}` - The key of the secret +- `{{environment}}` - The environment which the secret is in (e.g. dev, staging, prod) + **Example:** - Infisical key: `SECRET_1` - Schema: `INFISICAL_{{secretKey}}` diff --git a/docs/integrations/secret-syncs/teamcity.mdx b/docs/integrations/secret-syncs/teamcity.mdx index 3482101ca..52f2c1bac 100644 --- a/docs/integrations/secret-syncs/teamcity.mdx +++ b/docs/integrations/secret-syncs/teamcity.mdx @@ -48,7 +48,7 @@ description: "Learn how to configure a TeamCity Sync for Infisical." Infisical only syncs secrets from within the target scope; inherited secrets will not be imported. - - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name. + - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name and `{{environment}}` for the environment. We highly recommend using a Key Schema to ensure that Infisical only manages the specific keys you intend, keeping everything else untouched. diff --git a/docs/integrations/secret-syncs/terraform-cloud.mdx b/docs/integrations/secret-syncs/terraform-cloud.mdx index d2f762ef1..c48b87609 100644 --- a/docs/integrations/secret-syncs/terraform-cloud.mdx +++ b/docs/integrations/secret-syncs/terraform-cloud.mdx @@ -56,7 +56,7 @@ description: "Learn how to configure a Terraform Cloud Sync for Infisical." Terraform Cloud does not support importing secrets. - - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name. + - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name and `{{environment}}` for the environment. We highly recommend using a Key Schema to ensure that Infisical only manages the specific keys you intend, keeping everything else untouched. diff --git a/docs/integrations/secret-syncs/vercel.mdx b/docs/integrations/secret-syncs/vercel.mdx index c903d3faa..74cffbc11 100644 --- a/docs/integrations/secret-syncs/vercel.mdx +++ b/docs/integrations/secret-syncs/vercel.mdx @@ -43,7 +43,7 @@ description: "Learn how to configure a Vercel Sync for Infisical." - **Overwrite Destination Secrets**: Removes any secrets at the destination endpoint not present in Infisical. - **Import Secrets (Prioritize Infisical)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Infisical over Vercel when keys conflict. - **Import Secrets (Prioritize Vercel)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Vercel over Infisical when keys conflict. - - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name. + - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name and `{{environment}}` for the environment. We highly recommend using a Key Schema to ensure that Infisical only manages the specific keys you intend, keeping everything else untouched. diff --git a/docs/integrations/secret-syncs/windmill.mdx b/docs/integrations/secret-syncs/windmill.mdx index e98a2c7b6..c0757ef37 100644 --- a/docs/integrations/secret-syncs/windmill.mdx +++ b/docs/integrations/secret-syncs/windmill.mdx @@ -44,7 +44,7 @@ description: "Learn how to configure a Windmill Sync for Infisical." - **Overwrite Destination Secrets**: Removes any secrets at the destination endpoint not present in Infisical. - **Import Secrets (Prioritize Infisical)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Infisical over Windmill when keys conflict. - **Import Secrets (Prioritize Windmill)**: Imports secrets from the destination endpoint before syncing, prioritizing values from Windmill over Infisical when keys conflict. - - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name. + - **Key Schema**: Template that determines how secret names are transformed when syncing, using `{{secretKey}}` as a placeholder for the original secret name and `{{environment}}` for the environment. We highly recommend using a Key Schema to ensure that Infisical only manages the specific keys you intend, keeping everything else untouched. diff --git a/docs/internals/bug-bounty.mdx b/docs/internals/bug-bounty.mdx deleted file mode 100644 index fddd41683..000000000 --- a/docs/internals/bug-bounty.mdx +++ /dev/null @@ -1,80 +0,0 @@ ---- -title: "Bug bounty program" -description: " Learn about our bug bounty program and how to report vulnerabilities." ---- - -The Infisical Bug Bounty Program is our way of recognizing and rewarding the work of security researchers who help keep our platform secure. By reporting vulnerabilities or potential risks, you help us protect secrets, infrastructure, and the organizations who rely on us. - -We value reports that help identify vulnerabilities that affect the integrity of secrets, prevent unauthorized access to environments, or expose flaws in our authentication or authorization flows. - -### How to Report - -- Send reports to **security@infisical.com** with clear steps to reproduce, impact, and (if possible) a proof-of-concept. -- You will receive follow ups from our team if we deam your report to be a legitimate vulnerability or need further clarification. We do not respond to spam, auto generated reports, inaccurate claims, or submissions that are clearly out of scope. - - -### What's in Scope? - -- Vulnerabilities in our cloud-hosted platform (e.g., `app.infisical.com`, `eu.infisical.com`) -- Security issues in the open source Infisical codebase, as maintained in our official GitHub repository -- Authentication bypass, privilege escalation, or access to secrets/data without authorization - -### Reward Guidelines - -Bounties are based on severity, impact, and exploitability, as well as whether the report introduces a new vulnerability class or helps improve an existing fix. - -| Severity | Examples | Typical Reward (USD currency) | -| --- | --- | --- | -| **Critical** | Full unauthorized access to secrets, authentication bypass, cross-tenant access, RCE, full compromise, etc | $2,000 - $5,000 | -| **High** | Privilege escalation, project-level access without authorization, persistent DoS | $750 - $2,000 | -| **Medium** | Info disclosure, scoped DoS (e.g. ReDoS with auth), or minor access control issues | $100 - $1,000 | -| **Low / Informational** | Missing headers, CSP warnings, theoretical flaws, self-hosting misconfigurations | Recognition only | - - -We may award lower amounts for: -- Duplicate class vulnerabilities already under review -- Patch bypasses of previously rewarded issues -- Vulnerabilities requiring unrealistic attacker conditions - -All final reward amounts are determined at Infisical's discretion based on impact, report quality, and how actionable the issue is. - - -### Out of Scope - -- Social engineering or phishing (including email hyperlink injection without code execution) -- Rate limiting issues on non-sensitive endpoints -- Denial-of-service attacks that require authentication and don't impact core service availability -- Findings based on outdated or forked code not maintained by the Infisical team -- Vulnerabilities in third-party dependencies unless they result in a direct risk to Infisical users - - -### Responsible Disclosure - -We ask that researchers: - -- Avoid accessing data that isn't yours -- Do not publicly disclose without coordination -- Use testing accounts where possible -- Give us a reasonable window to investigate and patch before going public - -Researchers can also spin up our [self-hosted version of Infisical](/self-hosting/overview) to test for vulnerabilities locally. - -### Program Conduct and Enforcement - -We value professional and collaborative interaction with security researchers. To maintain the integrity of our bug bounty program, we expect all participants to adhere to the following guidelines: - -- Maintain professional communication in all interactions -- Do not threaten public disclosure of vulnerabilities before we've had reasonable time to investigate and address the issue -- Do not attempt to extort or coerce compensation through threats -- Follow the responsible disclosure process outlined in this document -- Do not use automated scanning tools without prior permission - -Violations of these guidelines may result in: - -1. **Warning**: For minor violations, we may issue a warning explaining the violation and requesting compliance with program guidelines. -2. **Temporary Ban**: Repeated minor violations or more serious violations may result in a temporary suspension from the program. -3. **Permanent Ban**: Severe violations such as threats, extortion attempts, or unauthorized public disclosure will result in permanent removal from the Infisical Bug Bounty Program. - -We reserve the right to reject reports, withhold bounties, and remove participants from the program at our discretion for conduct that undermines the collaborative spirit of security research. - -Infisical is committed to working respectfully with security researchers who follow these guidelines, and we strive to recognize and reward valuable contributions that help protect our platform and users. diff --git a/docs/internals/permissions/project-permissions.mdx b/docs/internals/permissions/project-permissions.mdx index 98f3bfeb2..da9351188 100644 --- a/docs/internals/permissions/project-permissions.mdx +++ b/docs/internals/permissions/project-permissions.mdx @@ -176,6 +176,13 @@ Supports conditions and permission inversion | `read` | View secret versions and snapshots | | `create` | Roll back secrets to snapshots | +#### Subject: `commits` + +| Action | Description | +| -------- | ---------------------------------- | +| `read` | View commits and changes across folders | +| `perform-rollback` | Roll back commits changes and restore folders to previous state| + #### Subject: `secret-approval` | Action | Description | diff --git a/docs/internals/security.mdx b/docs/internals/security.mdx index 219c32287..85138be9c 100644 --- a/docs/internals/security.mdx +++ b/docs/internals/security.mdx @@ -117,7 +117,3 @@ Whether or not Infisical or your employees can access data in the Infisical inst It should be noted that, even on Infisical Cloud, it is physically impossible for employees of Infisical to view the values of secrets if users have not explicitly granted Infisical access to their project (i.e. opted out of zero-knowledge). Please email security@infisical.com if you have any specific inquiries about employee data and security policies. - -## Bug Bounty Program -We run a [Bug Bounty Program](/internals/bug-bounty) to recognize and reward security researchers who help make Infisical more secure. -If you've found a vulnerability, please review the program details for scope, disclosure guidelines, and reward tiers. \ No newline at end of file diff --git a/docs/mint.json b/docs/mint.json index 3fcce4fb1..17ab395d4 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -233,7 +233,8 @@ "group": "Gateway", "pages": [ "documentation/platform/gateways/overview", - "documentation/platform/gateways/gateway-security" + "documentation/platform/gateways/gateway-security", + "documentation/platform/gateways/networking" ] }, "documentation/platform/project-templates", @@ -340,10 +341,12 @@ "group": "OIDC Auth", "pages": [ "documentation/platform/identities/oidc-auth/general", + "documentation/platform/identities/oidc-auth/azure", "documentation/platform/identities/oidc-auth/github", "documentation/platform/identities/oidc-auth/circleci", "documentation/platform/identities/oidc-auth/gitlab", - "documentation/platform/identities/oidc-auth/terraform-cloud" + "documentation/platform/identities/oidc-auth/terraform-cloud", + "documentation/platform/identities/oidc-auth/spire" ] }, @@ -825,8 +828,7 @@ "api-reference/endpoints/workspaces/delete-workspace", "api-reference/endpoints/workspaces/get-workspace", "api-reference/endpoints/workspaces/update-workspace", - "api-reference/endpoints/workspaces/secret-snapshots", - "api-reference/endpoints/workspaces/rollback-snapshot" + "api-reference/endpoints/workspaces/secret-snapshots" ] }, { @@ -926,6 +928,12 @@ { "group": "Dynamic Secrets", "pages": [ + { + "group": "Kubernetes", + "pages": [ + "api-reference/endpoints/dynamic-secrets/kubernetes/create-lease" + ] + }, "api-reference/endpoints/dynamic-secrets/create", "api-reference/endpoints/dynamic-secrets/update", "api-reference/endpoints/dynamic-secrets/delete", @@ -1871,7 +1879,6 @@ }, "internals/components", "internals/security", - "internals/bug-bounty", "internals/service-tokens" ] }, diff --git a/docs/self-hosting/deployment-options/kubernetes-helm.mdx b/docs/self-hosting/deployment-options/kubernetes-helm.mdx index e0e5ba75d..f8a2a6864 100644 --- a/docs/self-hosting/deployment-options/kubernetes-helm.mdx +++ b/docs/self-hosting/deployment-options/kubernetes-helm.mdx @@ -29,7 +29,7 @@ description: "Learn how to use Helm chart to install Infisical on your Kubernete infisical: image: repository: infisical/infisical - tag: "v0.46.2-postgres" #<-- update + tag: "<>" #<-- select tag from Dockerhub from the above link pullPolicy: IfNotPresent ``` diff --git a/frontend/src/components/auth/CodeInputStep.tsx b/frontend/src/components/auth/CodeInputStep.tsx index 09958fafd..f992c8da6 100644 --- a/frontend/src/components/auth/CodeInputStep.tsx +++ b/frontend/src/components/auth/CodeInputStep.tsx @@ -78,11 +78,14 @@ export default function CodeInputStep({ const resendVerificationEmail = async () => { setIsResendingVerificationEmail(true); setIsLoading(true); - await mutateAsync({ email }); - setTimeout(() => { - setIsLoading(false); - setIsResendingVerificationEmail(false); - }, 2000); + try { + await mutateAsync({ email }); + } finally { + setTimeout(() => { + setIsLoading(false); + setIsResendingVerificationEmail(false); + }, 1000); + } }; return ( diff --git a/frontend/src/components/navigation/SecretDashboardPathBreadcrumb.tsx b/frontend/src/components/navigation/SecretDashboardPathBreadcrumb.tsx index 584a3f6c7..31a626866 100644 --- a/frontend/src/components/navigation/SecretDashboardPathBreadcrumb.tsx +++ b/frontend/src/components/navigation/SecretDashboardPathBreadcrumb.tsx @@ -14,13 +14,15 @@ type Props = { selectedPathSegmentIndex: number; environmentSlug: string; projectId: string; + disableCopy?: boolean; }; export const SecretDashboardPathBreadcrumb = ({ secretPathSegments, selectedPathSegmentIndex, environmentSlug, - projectId + projectId, + disableCopy }: Props) => { const [, isCopying, setIsCopying] = useTimedReset({ initialState: false @@ -32,7 +34,7 @@ export const SecretDashboardPathBreadcrumb = ({ return (
- {isLastItem ? ( + {isLastItem && !disableCopy ? (
( +
+ {displayName}{" "} + + {locationId} + +
+); + export const GcpSyncFields = () => { const { control, setValue } = useFormContext< TSecretSyncForm & { destination: SecretSync.GCPSecretManager } >(); const connectionId = useWatch({ name: "connection.id", control }); + const projectId = useWatch({ name: "destinationConfig.projectId", control }); + const selectedScope = useWatch({ name: "destinationConfig.scope", control }); const { data: projects, isPending } = useGcpConnectionListProjects(connectionId, { enabled: Boolean(connectionId) }); + const { data: locations, isPending: areLocationsPending } = useGcpConnectionListProjectLocations( + { connectionId, projectId }, + { + enabled: Boolean(connectionId) && Boolean(projectId) + } + ); + useEffect(() => { - setValue("destinationConfig.scope", GcpSyncScope.Global); + if (!selectedScope) setValue("destinationConfig.scope", GcpSyncScope.Global); }, []); return ( @@ -33,6 +62,7 @@ export const GcpSyncFields = () => { { setValue("destinationConfig.projectId", ""); + setValue("destinationConfig.locationId", ""); }} /> { isLoading={isPending && Boolean(connectionId)} isDisabled={!connectionId} value={projects?.find((project) => project.id === value) ?? null} - onChange={(option) => - onChange((option as SingleValue)?.id ?? null) - } + onChange={(option) => { + setValue("destinationConfig.locationId", ""); + onChange((option as SingleValue)?.id ?? null); + }} options={projects} placeholder="Select a GCP project..." getOptionLabel={(option) => option.name} @@ -71,6 +102,76 @@ export const GcpSyncFields = () => { )} /> + ( + +

+ Specify how Infisical should sync secrets to GCP. The following options are + available: +

+
    + {Object.values(GCP_SYNC_SCOPES).map(({ name, description }) => { + return ( +
  • +

    + {name}: {description} +

    +
    • + ); + })} +
+
+ } + tooltipClassName="max-w-lg" + label="Scope" + > + + + )} + /> + {selectedScope === GcpSyncScope.Region && ( + ( + + option.locationId === value) ?? null} + onChange={(option) => + onChange((option as SingleValue)?.locationId ?? null) + } + options={locations} + placeholder="Select a region..." + getOptionValue={(option) => option.locationId} + formatOptionLabel={formatOptionLabel} + /> + + )} + /> + )} ); }; diff --git a/frontend/src/components/secret-syncs/forms/SecretSyncOptionsFields/SecretSyncOptionsFields.tsx b/frontend/src/components/secret-syncs/forms/SecretSyncOptionsFields/SecretSyncOptionsFields.tsx index 4b419bdcc..0f6382edc 100644 --- a/frontend/src/components/secret-syncs/forms/SecretSyncOptionsFields/SecretSyncOptionsFields.tsx +++ b/frontend/src/components/secret-syncs/forms/SecretSyncOptionsFields/SecretSyncOptionsFields.tsx @@ -132,7 +132,27 @@ export const SecretSyncOptionsFields = ({ hideInitialSync }: Props) => { render={({ field: { value, onChange }, fieldState: { error } }) => ( + + When a secret is synced, values will be injected into the key schema before it + reaches the destination. This is useful for organization. + + +
+ Available keys: +
    +
  • + {"{{secretKey}}"} - The key of the secret +
    • +
  • + {"{{environment}}"} - The environment which the secret is in + (e.g. dev, staging, prod) +
    • +
+
+
+ } isError={Boolean(error)} isOptional errorText={error?.message} diff --git a/frontend/src/components/secret-syncs/forms/SecretSyncReviewFields/GcpSyncReviewFields.tsx b/frontend/src/components/secret-syncs/forms/SecretSyncReviewFields/GcpSyncReviewFields.tsx index 000478f5e..bebb1ffe4 100644 --- a/frontend/src/components/secret-syncs/forms/SecretSyncReviewFields/GcpSyncReviewFields.tsx +++ b/frontend/src/components/secret-syncs/forms/SecretSyncReviewFields/GcpSyncReviewFields.tsx @@ -3,12 +3,23 @@ import { useFormContext } from "react-hook-form"; import { GenericFieldLabel } from "@app/components/secret-syncs"; import { TSecretSyncForm } from "@app/components/secret-syncs/forms/schemas"; import { SecretSync } from "@app/hooks/api/secretSyncs"; +import { GcpSyncScope } from "@app/hooks/api/secretSyncs/types/gcp-sync"; export const GcpSyncReviewFields = () => { const { watch } = useFormContext< TSecretSyncForm & { destination: SecretSync.GCPSecretManager } >(); - const projectId = watch("destinationConfig.projectId"); + const destinationConfig = watch("destinationConfig"); - return {projectId}; + return ( + <> + {destinationConfig.projectId} + + {destinationConfig.scope} + + {destinationConfig.scope === GcpSyncScope.Region && ( + {destinationConfig.locationId} + )} + + ); }; diff --git a/frontend/src/components/secret-syncs/forms/schemas/base-secret-sync-schema.ts b/frontend/src/components/secret-syncs/forms/schemas/base-secret-sync-schema.ts index 75a5b68c1..1d925eb07 100644 --- a/frontend/src/components/secret-syncs/forms/schemas/base-secret-sync-schema.ts +++ b/frontend/src/components/secret-syncs/forms/schemas/base-secret-sync-schema.ts @@ -13,11 +13,27 @@ export const BaseSecretSyncSchema = - !val || /^(?:[a-zA-Z0-9_\-/]*)(?:\{\{secretKey\}\})(?:[a-zA-Z0-9_\-/]*)$/.test(val), + (val) => { + if (!val) return true; + + const allowedOptionalPlaceholders = ["{{environment}}"]; + + const allowedPlaceholdersRegexPart = ["{{secretKey}}", ...allowedOptionalPlaceholders] + .map((p) => p.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\$&")) // Escape regex special characters + .join("|"); + + const allowedContentRegex = new RegExp( + `^([a-zA-Z0-9_\\-/]|${allowedPlaceholdersRegexPart})*$` + ); + const contentIsValid = allowedContentRegex.test(val); + + const secretKeyCount = (val.match(/\{\{secretKey\}\}/g) || []).length; + + return contentIsValid && secretKeyCount === 1; + }, { message: - "Key schema must include one {{secretKey}} and only contain letters, numbers, dashes, underscores, slashes, and the {{secretKey}} placeholder." + "Key schema must include exactly one {{secretKey}} placeholder. It can also include {{environment}} placeholders. Only alphanumeric characters (a-z, A-Z, 0-9), dashes (-), underscores (_), and slashes (/) are allowed besides the placeholders." } ) }); diff --git a/frontend/src/components/secret-syncs/forms/schemas/gcp-sync-destination-schema.ts b/frontend/src/components/secret-syncs/forms/schemas/gcp-sync-destination-schema.ts index 4225c6619..fffae153e 100644 --- a/frontend/src/components/secret-syncs/forms/schemas/gcp-sync-destination-schema.ts +++ b/frontend/src/components/secret-syncs/forms/schemas/gcp-sync-destination-schema.ts @@ -7,9 +7,16 @@ import { GcpSyncScope } from "@app/hooks/api/secretSyncs/types/gcp-sync"; export const GcpSyncDestinationSchema = BaseSecretSyncSchema().merge( z.object({ destination: z.literal(SecretSync.GCPSecretManager), - destinationConfig: z.object({ - scope: z.literal(GcpSyncScope.Global), - projectId: z.string().min(1, "Project ID required") - }) + destinationConfig: z.discriminatedUnion("scope", [ + z.object({ + scope: z.literal(GcpSyncScope.Global), + projectId: z.string().min(1, "Project ID required") + }), + z.object({ + scope: z.literal(GcpSyncScope.Region), + projectId: z.string().min(1, "Project ID required"), + locationId: z.string().min(1, "Region required") + }) + ]) }) ); diff --git a/frontend/src/const/routes.ts b/frontend/src/const/routes.ts index f5b462c3c..871985ce1 100644 --- a/frontend/src/const/routes.ts +++ b/frontend/src/const/routes.ts @@ -75,6 +75,18 @@ export const ROUTE_PATHS = Object.freeze({ "/secret-manager/$projectId/secrets/$envSlug", "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/secrets/$envSlug" ), + RollbackPreviewPage: setRoute( + "/secret-manager/$projectId/commits/$environment/$folderId/$commitId/restore", + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId/restore" + ), + CommitDetailsPage: setRoute( + "/secret-manager/$projectId/commits/$environment/$folderId/$commitId", + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId/$commitId" + ), + CommitsPage: setRoute( + "/secret-manager/$projectId/commits/$environment/$folderId", + "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/commits/$environment/$folderId" + ), OverviewPage: setRoute( "/secret-manager/$projectId/overview", "/_authenticate/_inject-org-details/_org-layout/secret-manager/$projectId/_secret-manager-layout/overview" diff --git a/frontend/src/context/ProjectPermissionContext/types.ts b/frontend/src/context/ProjectPermissionContext/types.ts index 2cbc5bd69..f9b2a828a 100644 --- a/frontend/src/context/ProjectPermissionContext/types.ts +++ b/frontend/src/context/ProjectPermissionContext/types.ts @@ -152,6 +152,11 @@ export enum PermissionConditionOperators { $ELEMENTMATCH = "$elemMatch" } +export enum ProjectPermissionCommitsActions { + Read = "read", + PerformRollback = "perform-rollback" +} + export type IdentityManagementSubjectFields = { identityId: string; }; @@ -229,6 +234,7 @@ export enum ProjectPermissionSub { Cmek = "cmek", SecretSyncs = "secret-syncs", Kmip = "kmip", + Commits = "commits", SecretScanningDataSources = "secret-scanning-data-sources", SecretScanningFindings = "secret-scanning-findings", SecretScanningConfigs = "secret-scanning-configs" @@ -367,6 +373,7 @@ export type ProjectPermissionSet = | [ProjectPermissionCmekActions, ProjectPermissionSub.Cmek] | [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms] | [ProjectPermissionKmipActions, ProjectPermissionSub.Kmip] + | [ProjectPermissionCommitsActions, ProjectPermissionSub.Commits] | [ ProjectPermissionSecretScanningDataSourceActions, ProjectPermissionSub.SecretScanningDataSources diff --git a/frontend/src/helpers/secretSyncs.ts b/frontend/src/helpers/secretSyncs.ts index ba4ba3ae6..98a519e33 100644 --- a/frontend/src/helpers/secretSyncs.ts +++ b/frontend/src/helpers/secretSyncs.ts @@ -4,6 +4,7 @@ import { SecretSyncImportBehavior, SecretSyncInitialSyncBehavior } from "@app/hooks/api/secretSyncs"; +import { GcpSyncScope } from "@app/hooks/api/secretSyncs/types/gcp-sync"; import { HumanitecSyncScope } from "@app/hooks/api/secretSyncs/types/humanitec-sync"; export const SECRET_SYNC_MAP: Record = { @@ -129,3 +130,14 @@ export const HUMANITEC_SYNC_SCOPES: Record< "Infisical will sync secrets as environment level shared values to the specified Humanitec application environment." } }; + +export const GCP_SYNC_SCOPES: Record = { + [GcpSyncScope.Global]: { + name: "Global", + description: "Secrets will be synced globally; being available in all project regions." + }, + [GcpSyncScope.Region]: { + name: "Region", + description: "Secrets will be synced to the specified region." + } +}; diff --git a/frontend/src/hooks/api/appConnections/gcp/queries.tsx b/frontend/src/hooks/api/appConnections/gcp/queries.tsx index a88860006..c8535aae1 100644 --- a/frontend/src/hooks/api/appConnections/gcp/queries.tsx +++ b/frontend/src/hooks/api/appConnections/gcp/queries.tsx @@ -3,12 +3,14 @@ import { useQuery, UseQueryOptions } from "@tanstack/react-query"; import { apiRequest } from "@app/config/request"; import { appConnectionKeys } from "../queries"; -import { TGcpProject } from "./types"; +import { TGcpLocation, TGcpProject, TListProjectLocations } from "./types"; const gcpConnectionKeys = { all: [...appConnectionKeys.all, "gcp"] as const, listProjects: (connectionId: string) => - [...gcpConnectionKeys.all, "projects", connectionId] as const + [...gcpConnectionKeys.all, "projects", connectionId] as const, + listProjectLocations: ({ projectId, connectionId }: TListProjectLocations) => + [...gcpConnectionKeys.all, "project-locations", connectionId, projectId] as const }; export const useGcpConnectionListProjects = ( @@ -35,3 +37,29 @@ export const useGcpConnectionListProjects = ( ...options }); }; + +export const useGcpConnectionListProjectLocations = ( + { connectionId, projectId }: TListProjectLocations, + options?: Omit< + UseQueryOptions< + TGcpLocation[], + unknown, + TGcpLocation[], + ReturnType + >, + "queryKey" | "queryFn" + > +) => { + return useQuery({ + queryKey: gcpConnectionKeys.listProjectLocations({ connectionId, projectId }), + queryFn: async () => { + const { data } = await apiRequest.get( + `/api/v1/app-connections/gcp/${connectionId}/secret-manager-project-locations`, + { params: { projectId } } + ); + + return data; + }, + ...options + }); +}; diff --git a/frontend/src/hooks/api/appConnections/gcp/types.ts b/frontend/src/hooks/api/appConnections/gcp/types.ts index 2af4eee9d..c5b594446 100644 --- a/frontend/src/hooks/api/appConnections/gcp/types.ts +++ b/frontend/src/hooks/api/appConnections/gcp/types.ts @@ -2,3 +2,13 @@ export type TGcpProject = { id: string; name: string; }; + +export type TListProjectLocations = { + connectionId: string; + projectId: string; +}; + +export type TGcpLocation = { + displayName: string; + locationId: string; +}; diff --git a/frontend/src/hooks/api/auditLogs/constants.tsx b/frontend/src/hooks/api/auditLogs/constants.tsx index ca5a38e0b..696f5745f 100644 --- a/frontend/src/hooks/api/auditLogs/constants.tsx +++ b/frontend/src/hooks/api/auditLogs/constants.tsx @@ -52,6 +52,7 @@ export const eventToNameMap: { [K in EventType]: string } = { [EventType.CREATE_WEBHOOK]: "Create webhook", [EventType.UPDATE_WEBHOOK_STATUS]: "Update webhook status", [EventType.DELETE_WEBHOOK]: "Delete webhook", + [EventType.WEBHOOK_TRIGGERED]: "Webhook event", [EventType.GET_SECRET_IMPORTS]: "List secret imports", [EventType.CREATE_SECRET_IMPORT]: "Create secret import", [EventType.UPDATE_SECRET_IMPORT]: "Update secret import", diff --git a/frontend/src/hooks/api/auditLogs/enums.tsx b/frontend/src/hooks/api/auditLogs/enums.tsx index 67f0bac99..159a61808 100644 --- a/frontend/src/hooks/api/auditLogs/enums.tsx +++ b/frontend/src/hooks/api/auditLogs/enums.tsx @@ -65,6 +65,7 @@ export enum EventType { CREATE_WEBHOOK = "create-webhook", UPDATE_WEBHOOK_STATUS = "update-webhook-status", DELETE_WEBHOOK = "delete-webhook", + WEBHOOK_TRIGGERED = "webhook-triggered", GET_SECRET_IMPORTS = "get-secret-imports", CREATE_SECRET_IMPORT = "create-secret-import", UPDATE_SECRET_IMPORT = "update-secret-import", diff --git a/frontend/src/hooks/api/auditLogs/types.tsx b/frontend/src/hooks/api/auditLogs/types.tsx index 8774e1a73..006a3d7ca 100644 --- a/frontend/src/hooks/api/auditLogs/types.tsx +++ b/frontend/src/hooks/api/auditLogs/types.tsx @@ -427,6 +427,16 @@ interface DeleteWebhookEvent { }; } +export interface WebhookTriggeredEvent { + type: EventType.WEBHOOK_TRIGGERED; + metadata: { + webhookId: string; + status: string; + type: string; + payload: { [k: string]: string | null }; + }; +} + interface GetSecretImportsEvent { type: EventType.GET_SECRET_IMPORTS; metadata: { @@ -891,6 +901,7 @@ export type Event = | CreateWebhookEvent | UpdateWebhookStatusEvent | DeleteWebhookEvent + | WebhookTriggeredEvent | GetSecretImportsEvent | CreateSecretImportEvent | UpdateSecretImportEvent diff --git a/frontend/src/hooks/api/dynamicSecret/types.ts b/frontend/src/hooks/api/dynamicSecret/types.ts index f9aa6d4d0..2357a83c0 100644 --- a/frontend/src/hooks/api/dynamicSecret/types.ts +++ b/frontend/src/hooks/api/dynamicSecret/types.ts @@ -37,6 +37,11 @@ export enum DynamicSecretProviders { Vertica = "vertica" } +export enum KubernetesDynamicSecretCredentialType { + Static = "static", + Dynamic = "dynamic" +} + export enum SqlProviders { Postgres = "postgres", MySql = "mysql2", @@ -44,6 +49,11 @@ export enum SqlProviders { MsSQL = "mssql" } +export enum DynamicSecretAwsIamAuth { + AssumeRole = "assume-role", + AccessKey = "access-key" +} + export type TDynamicSecretProvider = | { type: DynamicSecretProviders.SqlDatabase; @@ -78,15 +88,26 @@ export type TDynamicSecretProvider = } | { type: DynamicSecretProviders.AwsIam; - inputs: { - accessKey: string; - secretAccessKey: string; - region: string; - awsPath?: string; - policyDocument?: string; - userGroups?: string; - policyArns?: string; - }; + inputs: + | { + method: DynamicSecretAwsIamAuth.AccessKey; + accessKey: string; + secretAccessKey: string; + region: string; + awsPath?: string; + policyDocument?: string; + userGroups?: string; + policyArns?: string; + } + | { + method: DynamicSecretAwsIamAuth.AssumeRole; + roleArn: string; + region: string; + awsPath?: string; + policyDocument?: string; + userGroups?: string; + policyArns?: string; + }; } | { type: DynamicSecretProviders.Redis; @@ -267,17 +288,32 @@ export type TDynamicSecretProvider = } | { type: DynamicSecretProviders.Kubernetes; - inputs: { - url: string; - clusterToken: string; - ca?: string; - serviceAccountName: string; - credentialType: "dynamic" | "static"; - namespace: string; - gatewayId?: string; - sslEnabled: boolean; - audiences: string[]; - }; + inputs: + | { + url?: string; + clusterToken?: string; + ca?: string; + serviceAccountName: string; + credentialType: KubernetesDynamicSecretCredentialType.Static; + namespace: string; + gatewayId?: string; + sslEnabled: boolean; + audiences: string[]; + authMethod: string; + } + | { + url?: string; + clusterToken?: string; + ca?: string; + credentialType: KubernetesDynamicSecretCredentialType.Dynamic; + namespace: string; + gatewayId?: string; + sslEnabled: boolean; + audiences: string[]; + roleType: string; + role: string; + authMethod: string; + }; } | { type: DynamicSecretProviders.Vertica; diff --git a/frontend/src/hooks/api/dynamicSecretLease/mutation.ts b/frontend/src/hooks/api/dynamicSecretLease/mutation.ts index 1a95a3ab0..e7051bd2b 100644 --- a/frontend/src/hooks/api/dynamicSecretLease/mutation.ts +++ b/frontend/src/hooks/api/dynamicSecretLease/mutation.ts @@ -2,6 +2,7 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import { apiRequest } from "@app/config/request"; +import { DynamicSecretProviders } from "../dynamicSecret/types"; import { dynamicSecretLeaseKeys } from "./queries"; import { TCreateDynamicSecretLeaseDTO, @@ -19,6 +20,14 @@ export const useCreateDynamicSecretLease = () => { TCreateDynamicSecretLeaseDTO >({ mutationFn: async (dto) => { + if (dto.provider === DynamicSecretProviders.Kubernetes) { + const { data } = await apiRequest.post<{ lease: TDynamicSecretLease; data: unknown }>( + "/api/v1/dynamic-secrets/leases/kubernetes", + dto + ); + return data; + } + const { data } = await apiRequest.post<{ lease: TDynamicSecretLease; data: unknown }>( "/api/v1/dynamic-secrets/leases", dto diff --git a/frontend/src/hooks/api/dynamicSecretLease/types.ts b/frontend/src/hooks/api/dynamicSecretLease/types.ts index 76bedc8b3..51ee64c4b 100644 --- a/frontend/src/hooks/api/dynamicSecretLease/types.ts +++ b/frontend/src/hooks/api/dynamicSecretLease/types.ts @@ -1,3 +1,5 @@ +import { DynamicSecretProviders } from "../dynamicSecret/types"; + export enum DynamicSecretLeaseStatus { FailedDeletion = "Failed to delete" } @@ -13,12 +15,20 @@ export type TDynamicSecretLease = { updatedAt: string; }; +export type TDynamicSecretKubernetesLeaseConfig = { + namespace?: string; +}; + +export type TDynamicSecretLeaseConfig = TDynamicSecretKubernetesLeaseConfig; + export type TCreateDynamicSecretLeaseDTO = { dynamicSecretName: string; projectSlug: string; ttl?: string; path: string; environmentSlug: string; + config?: TDynamicSecretLeaseConfig; + provider: DynamicSecretProviders; }; export type TRenewDynamicSecretLeaseDTO = { diff --git a/frontend/src/hooks/api/folderCommits/index.tsx b/frontend/src/hooks/api/folderCommits/index.tsx new file mode 100644 index 000000000..ffdec812a --- /dev/null +++ b/frontend/src/hooks/api/folderCommits/index.tsx @@ -0,0 +1,2 @@ +export { useGetFolderCommitHistory, useGetFolderCommitsCount } from "./queries"; +export * from "./types"; diff --git a/frontend/src/hooks/api/folderCommits/queries.tsx b/frontend/src/hooks/api/folderCommits/queries.tsx new file mode 100644 index 000000000..8d0883c5b --- /dev/null +++ b/frontend/src/hooks/api/folderCommits/queries.tsx @@ -0,0 +1,294 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; + +import { apiRequest } from "@app/config/request"; + +import { CommitHistoryItem, CommitWithChanges, RollbackPreview } from "./types"; + +export const commitKeys = { + count: ({ + workspaceId, + environment, + directory + }: { + workspaceId: string; + environment: string; + directory?: string; + }) => [{ workspaceId, environment, directory }, "folder-commits-count"] as const, + + history: ({ + workspaceId, + environment, + directory + }: { + workspaceId: string; + environment: string; + directory?: string; + }) => [{ workspaceId, environment, directory }, "folder-commits"] as const, + + details: ({ workspaceId, commitId }: { workspaceId: string; commitId: string }) => + [{ workspaceId, commitId }, "commit-details"] as const, + + rollbackPreview: ({ + folderId, + commitId, + envSlug, + projectId, + deepRollback + }: { + folderId: string; + commitId: string; + envSlug: string; + projectId: string; + deepRollback: boolean; + }) => [{ folderId, commitId, envSlug, projectId, deepRollback }, "rollback-preview"] as const +}; + +const fetchFolderCommitsCount = async ({ + workspaceId, + environment, + directory +}: { + workspaceId: string; + environment: string; + directory?: string; +}) => { + const res = await apiRequest.get<{ count: number; folderId: string }>( + "/api/v1/pit/commits/count", + { + params: { + environment, + path: directory, + projectId: workspaceId + } + } + ); + return res.data; +}; + +const fetchFolderCommitHistory = async ( + workspaceId: string, + environment: string, + directory: string, + offset: number = 0, + limit: number = 20, + search?: string, + sort: "asc" | "desc" = "desc" +): Promise<{ + commits: CommitHistoryItem[]; + total: number; + hasMore: boolean; +}> => { + const res = await apiRequest.get<{ + commits: CommitHistoryItem[]; + total: number; + hasMore: boolean; + }>("/api/v1/pit/commits", { + params: { + environment, + path: directory, + projectId: workspaceId, + offset, + limit, + search, + sort + } + }); + return res.data; +}; + +export const fetchCommitDetails = async (workspaceId: string, commitId: string) => { + const { data } = await apiRequest.get( + `/api/v1/pit/commits/${commitId}/changes`, + { + params: { + projectId: workspaceId + } + } + ); + return data; +}; + +export const fetchRollbackPreview = async ( + folderId: string, + commitId: string, + envSlug: string, + workspaceId: string, + deepRollback: boolean, + secretPath: string +): Promise => { + const { data } = await apiRequest.get( + `/api/v1/pit/commits/${commitId}/compare`, + { + params: { + folderId, + environment: envSlug, + deepRollback, + secretPath, + projectId: workspaceId + } + } + ); + return data; +}; + +const fetchRollback = async ( + folderId: string, + commitId: string, + workspaceId: string, + deepRollback: boolean, + message?: string, + envSlug?: string +) => { + const { data } = await apiRequest.post<{ success: boolean }>( + `/api/v1/pit/commits/${commitId}/rollback`, + { + folderId, + deepRollback, + message, + environment: envSlug, + projectId: workspaceId + } + ); + return data; +}; + +const fetchRevert = async (commitId: string, workspaceId: string) => { + const { data } = await apiRequest.post<{ success: boolean; message: string }>( + `/api/v1/pit/commits/${commitId}/revert`, + { + projectId: workspaceId + } + ); + return data; +}; + +export const useCommitRevert = ({ + commitId, + projectId, + environment, + directory +}: { + commitId: string; + projectId: string; + environment: string; + directory: string; +}) => { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: () => fetchRevert(commitId, projectId), + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: [ + commitKeys.details({ workspaceId: projectId, commitId }), + commitKeys.history({ workspaceId: projectId, environment, directory }), + commitKeys.count({ workspaceId: projectId, environment, directory }) + ] + }); + } + }); +}; + +export const useCommitRollback = ({ + workspaceId, + commitId, + folderId, + deepRollback, + environment, + directory, + envSlug +}: { + workspaceId: string; + commitId: string; + folderId: string; + deepRollback: boolean; + environment: string; + directory: string; + envSlug: string; +}) => { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (message: string) => + fetchRollback(folderId, commitId, workspaceId, deepRollback, message, envSlug), + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: [ + commitKeys.details({ workspaceId, commitId }), + commitKeys.history({ workspaceId, environment, directory }), + commitKeys.count({ workspaceId, environment, directory }) + ] + }); + } + }); +}; + +export const useGetFolderCommitsCount = ({ + workspaceId, + environment, + directory, + isPaused +}: { + workspaceId: string; + environment: string; + directory: string; + isPaused?: boolean; +}) => + useQuery({ + enabled: Boolean(workspaceId && environment) && !isPaused, + queryKey: commitKeys.count({ workspaceId, environment, directory }), + queryFn: () => fetchFolderCommitsCount({ workspaceId, environment, directory }) + }); + +export const useGetFolderCommitHistory = ({ + workspaceId, + environment, + directory, + offset = 0, + limit = 20, + search, + sort = "desc" +}: { + workspaceId: string; + environment: string; + directory: string; + offset?: number; + limit?: number; + search?: string; + sort?: "asc" | "desc"; +}) => { + return useQuery({ + queryKey: [ + commitKeys.history({ workspaceId, environment, directory }), + offset, + limit, + search, + sort + ], + queryFn: () => + fetchFolderCommitHistory(workspaceId, environment, directory, offset, limit, search, sort), + enabled: Boolean(workspaceId && environment) + }); +}; + +export const useGetCommitDetails = (workspaceId: string, commitId: string) => { + return useQuery({ + queryKey: commitKeys.details({ workspaceId, commitId }), + queryFn: () => fetchCommitDetails(workspaceId, commitId), + enabled: Boolean(workspaceId) && Boolean(commitId) + }); +}; + +export const useGetRollbackPreview = ( + folderId: string, + commitId: string, + envSlug: string, + projectId: string, + deepRollback: boolean, + secretPath: string +) => { + return useQuery({ + queryKey: commitKeys.rollbackPreview({ folderId, commitId, envSlug, projectId, deepRollback }), + queryFn: () => + fetchRollbackPreview(folderId, commitId, envSlug, projectId, deepRollback, secretPath), + enabled: Boolean(folderId) && Boolean(commitId) + }); +}; diff --git a/frontend/src/hooks/api/folderCommits/types.ts b/frontend/src/hooks/api/folderCommits/types.ts new file mode 100644 index 000000000..878e3224d --- /dev/null +++ b/frontend/src/hooks/api/folderCommits/types.ts @@ -0,0 +1,64 @@ +import { CommitType, SecretVersions } from "../types"; + +export type CommitHistoryItem = { + id: string; + commitId: string; + actorMetadata: { + id: string; + name?: string; + }; + actorType: string; + message: string; + folderId: string; + envId: string; + createdAt: string; + updatedAt: string; + isLatest: boolean; +}; + +export type TFolderCommitChanges = { + id: string; + folderCommitId: string; + changeType: CommitType; + isUpdate: boolean; + secretVersionId: string | null; + folderVersionId: string | null; + createdAt: string; + updatedAt: string; + versions: SecretVersions[]; + secretKey?: string; + folderName?: string; + secretVersion?: string; + folderVersion?: string; +}; + +export type FolderReconstructedItem = { + type: string; + id: string; + versionId: string; + folderName?: string; + folderVersion?: number; + secretKey?: string; + secretVersion?: number; +}; + +export type CommitWithChanges = { + changes: CommitHistoryItem & { + changes: TFolderCommitChanges[]; + }; +}; + +export type RollbackChange = { + type: "folder" | "secret"; + id: string; + versionId: string; + changeType: "create" | "update" | "delete"; + commitId: string; +}; + +export type RollbackPreview = { + folderId: string; + folderName: string; + folderPath: string; + changes: RollbackChange[]; +}; diff --git a/frontend/src/hooks/api/groups/queries.tsx b/frontend/src/hooks/api/groups/queries.tsx index eb10085bc..6fbe0ec13 100644 --- a/frontend/src/hooks/api/groups/queries.tsx +++ b/frontend/src/hooks/api/groups/queries.tsx @@ -21,7 +21,27 @@ export const groupKeys = { limit: number; search: string; filter?: EFilterReturnedUsers; - }) => [...groupKeys.forGroupUserMemberships(slug), { offset, limit, search, filter }] as const + }) => [...groupKeys.forGroupUserMemberships(slug), { offset, limit, search, filter }] as const, + specificProjectGroupUserMemberships: ({ + projectId, + slug, + offset, + limit, + search, + filter + }: { + slug: string; + projectId: string; + offset: number; + limit: number; + search: string; + filter?: EFilterReturnedUsers; + }) => + [ + ...groupKeys.forGroupUserMemberships(slug), + projectId, + { offset, limit, search, filter } + ] as const }; export const useGetGroupById = (groupId: string) => { @@ -80,3 +100,51 @@ export const useListGroupUsers = ({ } }); }; + +export const useListProjectGroupUsers = ({ + id, + projectId, + groupSlug, + offset = 0, + limit = 10, + search, + filter +}: { + id: string; + groupSlug: string; + projectId: string; + offset: number; + limit: number; + search: string; + filter?: EFilterReturnedUsers; +}) => { + return useQuery({ + queryKey: groupKeys.specificProjectGroupUserMemberships({ + slug: groupSlug, + projectId, + offset, + limit, + search, + filter + }), + enabled: Boolean(groupSlug), + placeholderData: (previousData) => previousData, + queryFn: async () => { + const params = new URLSearchParams({ + offset: String(offset), + limit: String(limit), + search, + ...(filter && { filter }) + }); + + const { data } = await apiRequest.get<{ users: TGroupUser[]; totalCount: number }>( + `/api/v2/workspace/${projectId}/groups/${id}/users`, + { + params + } + ); + + return data; + } + }); +}; diff --git a/frontend/src/hooks/api/identities/mutations.tsx b/frontend/src/hooks/api/identities/mutations.tsx index c08081f11..a7bd88ce9 100644 --- a/frontend/src/hooks/api/identities/mutations.tsx +++ b/frontend/src/hooks/api/identities/mutations.tsx @@ -843,7 +843,8 @@ export const useAddIdentityKubernetesAuth = () => { accessTokenMaxTTL, accessTokenNumUsesLimit, accessTokenTrustedIps, - gatewayId + gatewayId, + tokenReviewMode }) => { const { data: { identityKubernetesAuth } @@ -860,7 +861,8 @@ export const useAddIdentityKubernetesAuth = () => { accessTokenMaxTTL, accessTokenNumUsesLimit, accessTokenTrustedIps, - gatewayId + gatewayId, + tokenReviewMode } ); @@ -950,7 +952,8 @@ export const useUpdateIdentityKubernetesAuth = () => { accessTokenMaxTTL, accessTokenNumUsesLimit, accessTokenTrustedIps, - gatewayId + gatewayId, + tokenReviewMode }) => { const { data: { identityKubernetesAuth } @@ -967,7 +970,8 @@ export const useUpdateIdentityKubernetesAuth = () => { accessTokenMaxTTL, accessTokenNumUsesLimit, accessTokenTrustedIps, - gatewayId + gatewayId, + tokenReviewMode } ); diff --git a/frontend/src/hooks/api/identities/types.ts b/frontend/src/hooks/api/identities/types.ts index c26466213..6b00eb40c 100644 --- a/frontend/src/hooks/api/identities/types.ts +++ b/frontend/src/hooks/api/identities/types.ts @@ -379,10 +379,16 @@ export type DeleteIdentityAzureAuthDTO = { identityId: string; }; +export enum IdentityKubernetesAuthTokenReviewMode { + Api = "api", + Gateway = "gateway" +} + export type IdentityKubernetesAuth = { identityId: string; kubernetesHost: string; tokenReviewerJwt: string; + tokenReviewMode: IdentityKubernetesAuthTokenReviewMode; allowedNamespaces: string; allowedNames: string; allowedAudience: string; @@ -397,8 +403,9 @@ export type IdentityKubernetesAuth = { export type AddIdentityKubernetesAuthDTO = { organizationId: string; identityId: string; - kubernetesHost: string; + kubernetesHost: string | null; tokenReviewerJwt?: string; + tokenReviewMode: IdentityKubernetesAuthTokenReviewMode; allowedNamespaces: string; allowedNames: string; allowedAudience: string; @@ -415,8 +422,9 @@ export type AddIdentityKubernetesAuthDTO = { export type UpdateIdentityKubernetesAuthDTO = { organizationId: string; identityId: string; - kubernetesHost?: string; + kubernetesHost?: string | null; tokenReviewerJwt?: string | null; + tokenReviewMode?: IdentityKubernetesAuthTokenReviewMode; allowedNamespaces?: string; allowedNames?: string; allowedAudience?: string; diff --git a/frontend/src/hooks/api/roles/mutation.tsx b/frontend/src/hooks/api/roles/mutation.tsx index 8d662f993..6df9aa937 100644 --- a/frontend/src/hooks/api/roles/mutation.tsx +++ b/frontend/src/hooks/api/roles/mutation.tsx @@ -96,7 +96,7 @@ export const useUpdateOrgRole = () => { data: { role } } = await apiRequest.patch(`/api/v1/organization/${orgId}/roles/${id}`, { ...dto, - permissions: permissions?.length ? packRules(permissions) : undefined + permissions: permissions ? packRules(permissions) : undefined }); return role; diff --git a/frontend/src/hooks/api/secretApprovalRequest/types.ts b/frontend/src/hooks/api/secretApprovalRequest/types.ts index 3ac2574b5..241d3c1e2 100644 --- a/frontend/src/hooks/api/secretApprovalRequest/types.ts +++ b/frontend/src/hooks/api/secretApprovalRequest/types.ts @@ -11,7 +11,8 @@ export enum ApprovalStatus { export enum CommitType { DELETE = "delete", UPDATE = "update", - CREATE = "create" + CREATE = "create", + ADD = "add" } export type TSecretApprovalSecChangeData = { diff --git a/frontend/src/hooks/api/secretFolders/queries.tsx b/frontend/src/hooks/api/secretFolders/queries.tsx index b49fbd6cc..9e12f329d 100644 --- a/frontend/src/hooks/api/secretFolders/queries.tsx +++ b/frontend/src/hooks/api/secretFolders/queries.tsx @@ -10,6 +10,7 @@ import { import { apiRequest } from "@app/config/request"; import { dashboardKeys } from "@app/hooks/api/dashboard/queries"; +import { commitKeys } from "../folderCommits/queries"; import { secretSnapshotKeys } from "../secretSnapshots/queries"; import { TCreateFolderDTO, @@ -166,6 +167,9 @@ export const useCreateFolder = () => { queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ workspaceId: projectId, environment, directory: path }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId: projectId, environment, directory: path }) + }); } }); }; @@ -200,6 +204,12 @@ export const useUpdateFolder = () => { queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ workspaceId: projectId, environment, directory: path }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId: projectId, environment, directory: path }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId: projectId, environment, directory: path }) + }); } }); }; @@ -234,6 +244,12 @@ export const useDeleteFolder = () => { queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ workspaceId: projectId, environment, directory: path }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId: projectId, environment, directory: path }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId: projectId, environment, directory: path }) + }); } }); }; @@ -279,6 +295,20 @@ export const useUpdateFolderBatch = () => { directory: folder.path }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ + workspaceId: projectId, + environment: folder.environment, + directory: folder.path + }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ + workspaceId: projectId, + environment: folder.environment, + directory: folder.path + }) + }); }); } }); diff --git a/frontend/src/hooks/api/secretSyncs/types/gcp-sync.ts b/frontend/src/hooks/api/secretSyncs/types/gcp-sync.ts index bda7da6be..b7b9c1db8 100644 --- a/frontend/src/hooks/api/secretSyncs/types/gcp-sync.ts +++ b/frontend/src/hooks/api/secretSyncs/types/gcp-sync.ts @@ -3,15 +3,22 @@ import { SecretSync } from "@app/hooks/api/secretSyncs"; import { TRootSecretSync } from "@app/hooks/api/secretSyncs/types/root-sync"; export enum GcpSyncScope { - Global = "global" + Global = "global", + Region = "region" } export type TGcpSync = TRootSecretSync & { destination: SecretSync.GCPSecretManager; - destinationConfig: { - scope: GcpSyncScope.Global; - projectId: string; - }; + destinationConfig: + | { + scope: GcpSyncScope.Global; + projectId: string; + } + | { + scope: GcpSyncScope.Region; + projectId: string; + locationId: string; + }; connection: { app: AppConnection.GCP; name: string; diff --git a/frontend/src/hooks/api/secrets/mutations.tsx b/frontend/src/hooks/api/secrets/mutations.tsx index 3862d1f8d..82bf623e6 100644 --- a/frontend/src/hooks/api/secrets/mutations.tsx +++ b/frontend/src/hooks/api/secrets/mutations.tsx @@ -3,6 +3,7 @@ import { MutationOptions, useMutation, useQueryClient } from "@tanstack/react-qu import { apiRequest } from "@app/config/request"; import { dashboardKeys } from "@app/hooks/api/dashboard/queries"; +import { commitKeys } from "../folderCommits/queries"; import { secretApprovalRequestKeys } from "../secretApprovalRequest/queries"; import { secretSnapshotKeys } from "../secretSnapshots/queries"; import { secretKeys } from "./queries"; @@ -59,6 +60,12 @@ export const useCreateSecretV3 = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -118,6 +125,12 @@ export const useUpdateSecretV3 = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -164,6 +177,12 @@ export const useDeleteSecretV3 = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -200,6 +219,12 @@ export const useCreateSecretBatch = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -236,6 +261,12 @@ export const useUpdateSecretBatch = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -274,6 +305,12 @@ export const useDeleteSecretBatch = ({ queryClient.invalidateQueries({ queryKey: secretSnapshotKeys.count({ environment, workspaceId, directory: secretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ workspaceId, environment, directory: secretPath }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ workspaceId, environment, directory: secretPath }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId }) }); }, ...options @@ -347,6 +384,20 @@ export const useMoveSecrets = ({ directory: sourceSecretPath }) }); + queryClient.invalidateQueries({ + queryKey: commitKeys.count({ + workspaceId: projectId, + environment: sourceEnvironment, + directory: sourceSecretPath + }) + }); + queryClient.invalidateQueries({ + queryKey: commitKeys.history({ + workspaceId: projectId, + environment: sourceEnvironment, + directory: sourceSecretPath + }) + }); queryClient.invalidateQueries({ queryKey: secretApprovalRequestKeys.count({ workspaceId: projectId }) }); diff --git a/frontend/src/hooks/api/workspace/mutations.tsx b/frontend/src/hooks/api/workspace/mutations.tsx index ea7376d3d..5cd7e3bbc 100644 --- a/frontend/src/hooks/api/workspace/mutations.tsx +++ b/frontend/src/hooks/api/workspace/mutations.tsx @@ -50,10 +50,13 @@ export const useUpdateGroupWorkspaceRole = () => { return groupMembership; }, - onSuccess: (_, { projectId }) => { + onSuccess: (_, { projectId, groupId }) => { queryClient.invalidateQueries({ queryKey: workspaceKeys.getWorkspaceGroupMemberships(projectId) }); + queryClient.invalidateQueries({ + queryKey: workspaceKeys.getWorkspaceGroupMembershipDetails(projectId, groupId) + }); } }); }; diff --git a/frontend/src/hooks/api/workspace/queries.tsx b/frontend/src/hooks/api/workspace/queries.tsx index c040a1267..fa9240101 100644 --- a/frontend/src/hooks/api/workspace/queries.tsx +++ b/frontend/src/hooks/api/workspace/queries.tsx @@ -282,7 +282,8 @@ export const useUpdateProject = () => { newProjectName, newProjectDescription, newSlug, - secretSharing + secretSharing, + showSnapshotsLegacy }) => { const { data } = await apiRequest.patch<{ workspace: Workspace }>( `/api/v1/workspace/${projectID}`, @@ -290,7 +291,8 @@ export const useUpdateProject = () => { name: newProjectName, description: newProjectDescription, slug: newSlug, - secretSharing + secretSharing, + showSnapshotsLegacy } ); return data.workspace; @@ -691,6 +693,21 @@ export const useGetWorkspaceIdentityMembershipDetails = (projectId: string, iden }); }; +export const useGetWorkspaceGroupMembershipDetails = (projectId: string, groupId: string) => { + return useQuery({ + enabled: Boolean(projectId && groupId), + queryKey: workspaceKeys.getWorkspaceGroupMembershipDetails(projectId, groupId), + queryFn: async () => { + const { + data: { groupMembership } + } = await apiRequest.get<{ groupMembership: TGroupMembership }>( + `/api/v2/workspace/${projectId}/groups/${groupId}` + ); + return groupMembership; + } + }); +}; + export const useListWorkspaceGroups = (projectId: string) => { return useQuery({ queryKey: workspaceKeys.getWorkspaceGroupMemberships(projectId), diff --git a/frontend/src/hooks/api/workspace/query-keys.tsx b/frontend/src/hooks/api/workspace/query-keys.tsx index c10616b63..15317d2fd 100644 --- a/frontend/src/hooks/api/workspace/query-keys.tsx +++ b/frontend/src/hooks/api/workspace/query-keys.tsx @@ -36,6 +36,8 @@ export const workspaceKeys = { searchWorkspace: (dto: TSearchProjectsDTO) => ["search-projects", dto] as const, getWorkspaceGroupMemberships: (workspaceId: string) => [{ workspaceId }, "workspace-groups"] as const, + getWorkspaceGroupMembershipDetails: (workspaceId: string, groupId: string) => + [{ workspaceId, groupId }, "workspace-group-membership-details"] as const, getWorkspaceCas: ({ projectSlug }: { projectSlug: string }) => [{ projectSlug }, "workspace-cas"] as const, specificWorkspaceCas: ({ projectSlug, status }: { projectSlug: string; status?: CaStatus }) => diff --git a/frontend/src/hooks/api/workspace/types.ts b/frontend/src/hooks/api/workspace/types.ts index 481bdcc08..fbaea7742 100644 --- a/frontend/src/hooks/api/workspace/types.ts +++ b/frontend/src/hooks/api/workspace/types.ts @@ -39,6 +39,7 @@ export type Workspace = { roles?: TProjectRole[]; hasDeleteProtection: boolean; secretSharing: boolean; + showSnapshotsLegacy: boolean; }; export type WorkspaceEnv = { @@ -79,6 +80,7 @@ export type UpdateProjectDTO = { newProjectDescription?: string; newSlug?: string; secretSharing?: boolean; + showSnapshotsLegacy?: boolean; }; export type UpdatePitVersionLimitDTO = { projectSlug: string; pitVersionLimit: number }; diff --git a/frontend/src/lib/fn/string.ts b/frontend/src/lib/fn/string.ts index 6b2842701..a733308b7 100644 --- a/frontend/src/lib/fn/string.ts +++ b/frontend/src/lib/fn/string.ts @@ -8,6 +8,11 @@ export const formatReservedPaths = (secretPath: string) => { return secretPath; }; +export const parsePathFromReplicatedPath = (secretPath: string) => { + const i = secretPath.indexOf(ReservedFolders.SecretReplication); + return secretPath.slice(0, i); +}; + export const camelCaseToSpaces = (input: string) => { return input.replace(/([a-z])([A-Z])/g, "$1 $2"); }; diff --git a/frontend/src/pages/auth/SelectOrgPage/SelectOrgSection.tsx b/frontend/src/pages/auth/SelectOrgPage/SelectOrgSection.tsx index a736b52a7..ac194cd05 100644 --- a/frontend/src/pages/auth/SelectOrgPage/SelectOrgSection.tsx +++ b/frontend/src/pages/auth/SelectOrgPage/SelectOrgSection.tsx @@ -3,7 +3,7 @@ import { Helmet } from "react-helmet"; import { useTranslation } from "react-i18next"; import { faArrowRight } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { Link, useNavigate } from "@tanstack/react-router"; +import { Link, useNavigate, useRouter } from "@tanstack/react-router"; import axios from "axios"; import { addSeconds, formatISO } from "date-fns"; import { jwtDecode } from "jwt-decode"; @@ -51,6 +51,7 @@ export const SelectOrganizationSection = () => { const [mfaSuccessCallback, setMfaSuccessCallback] = useState<() => void>(() => {}); + const router = useRouter(); const queryParams = new URLSearchParams(window.location.search); const orgId = queryParams.get("org_id"); const callbackPort = queryParams.get("callback_port"); @@ -118,6 +119,8 @@ export const SelectOrganizationSection = () => { }) .finally(() => setIsInitialOrgCheckLoading(false)); + await router.invalidate(); + if (isMfaEnabled) { SecurityClient.setMfaToken(token); if (mfaMethod) { diff --git a/frontend/src/pages/auth/SelectOrgPage/route.tsx b/frontend/src/pages/auth/SelectOrgPage/route.tsx index 445479b3f..27ad4bb94 100644 --- a/frontend/src/pages/auth/SelectOrgPage/route.tsx +++ b/frontend/src/pages/auth/SelectOrgPage/route.tsx @@ -7,7 +7,8 @@ import { SelectOrganizationPage } from "./SelectOrgPage"; export const SelectOrganizationPageQueryParams = z.object({ org_id: z.string().optional().catch(""), callback_port: z.coerce.number().optional().catch(undefined), - is_admin_login: z.boolean().optional().catch(false) + is_admin_login: z.boolean().optional().catch(false), + force: z.boolean().optional() }); export const Route = createFileRoute("/_restrict-login-signup/login/select-organization")({ diff --git a/frontend/src/pages/auth/SignUpInvitePage/SignUpInvitePage.tsx b/frontend/src/pages/auth/SignUpInvitePage/SignUpInvitePage.tsx index a01545537..9cb739736 100644 --- a/frontend/src/pages/auth/SignUpInvitePage/SignUpInvitePage.tsx +++ b/frontend/src/pages/auth/SignUpInvitePage/SignUpInvitePage.tsx @@ -28,8 +28,7 @@ import { import { MfaMethod } from "@app/hooks/api/auth/types"; import { fetchOrganizations } from "@app/hooks/api/organization/queries"; import { ProjectType } from "@app/hooks/api/workspace/types"; - -import { navigateUserToOrg } from "../LoginPage/Login.utils"; +import { isLoggedIn } from "@app/hooks/api/reactQuery"; // eslint-disable-next-line new-cap const client = new jsrp.client(); @@ -71,6 +70,8 @@ export const SignupInvitePage = () => { const { mutateAsync: selectOrganization } = useSelectOrganization(); + const loggedIn = isLoggedIn(); + // Verifies if the information that the users entered (name, workspace) is there, and if the password matched the criteria. const signupErrorCheck = async () => { setIsLoading(true); @@ -242,29 +243,10 @@ export const SignupInvitePage = () => { if (response?.token) { SecurityClient.setSignupToken(response.token); setStep(2); + } else if (loggedIn) { + navigate({ to: "/login/select-organization", search: { force: true } }); } else { - const redirectExistingUser = async () => { - try { - const { token: mfaToken, isMfaEnabled } = await selectOrganization({ - organizationId - }); - - if (isMfaEnabled) { - SecurityClient.setMfaToken(mfaToken); - toggleShowMfa.on(); - setMfaSuccessCallback(() => redirectExistingUser); - return; - } - - // user will be redirected to dashboard - // if not logged in gets kicked out to login - await navigateUserToOrg(navigate, organizationId); - } catch (err) { - navigate({ to: "/login" }); - } - }; - - await redirectExistingUser(); + navigate({ to: "/login" }); } } } catch (err) { diff --git a/frontend/src/pages/auth/VerifyEmailPage/VerifyEmailPage.tsx b/frontend/src/pages/auth/VerifyEmailPage/VerifyEmailPage.tsx index 47012e2cb..9777f564a 100644 --- a/frontend/src/pages/auth/VerifyEmailPage/VerifyEmailPage.tsx +++ b/frontend/src/pages/auth/VerifyEmailPage/VerifyEmailPage.tsx @@ -22,8 +22,12 @@ export const VerifyEmailPage = () => { */ const sendVerificationEmail = async () => { if (email) { - await mutateAsync({ email }); - setStep(2); + try { + await mutateAsync({ email }); + setStep(2); + } catch { + setLoading(false); + } } }; diff --git a/frontend/src/pages/middlewares/restrict-login-signup.tsx b/frontend/src/pages/middlewares/restrict-login-signup.tsx index f85ea44f2..73964868a 100644 --- a/frontend/src/pages/middlewares/restrict-login-signup.tsx +++ b/frontend/src/pages/middlewares/restrict-login-signup.tsx @@ -15,7 +15,8 @@ import { setAuthToken } from "@app/hooks/api/reactQuery"; import { ProjectType } from "@app/hooks/api/workspace/types"; const QueryParamsSchema = z.object({ - callback_port: z.coerce.number().optional().catch(undefined) + callback_port: z.coerce.number().optional().catch(undefined), + force: z.boolean().optional() }); export const AuthConsentWrapper = () => { @@ -71,7 +72,7 @@ export const AuthConsentWrapper = () => { export const Route = createFileRoute("/_restrict-login-signup")({ validateSearch: zodValidator(QueryParamsSchema), search: { - middlewares: [stripSearchParams({ callback_port: undefined })] + middlewares: [stripSearchParams({ callback_port: undefined, force: undefined })] }, beforeLoad: async ({ context, location, search }) => { if (!context.serverConfig.initialized) { @@ -90,6 +91,12 @@ export const Route = createFileRoute("/_restrict-login-signup")({ if (!data) return; setAuthToken(data.token); + + if (location.pathname === "/signupinvite") return; + + // Avoid redirect if on select-organization page with force=true + if (location.pathname.endsWith("select-organization") && search?.force === true) return; + // to do cli login if (search?.callback_port) { if (location.pathname.endsWith("select-organization") || location.pathname.endsWith("login")) diff --git a/frontend/src/pages/organization/AccessManagementPage/AccessManagementPage.tsx b/frontend/src/pages/organization/AccessManagementPage/AccessManagementPage.tsx index 61c4f4411..bb1d5010a 100644 --- a/frontend/src/pages/organization/AccessManagementPage/AccessManagementPage.tsx +++ b/frontend/src/pages/organization/AccessManagementPage/AccessManagementPage.tsx @@ -5,6 +5,7 @@ import { faInfoCircle } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { useNavigate, useSearch } from "@tanstack/react-router"; +import { OrgPermissionGuardBanner } from "@app/components/permissions/OrgPermissionCan"; import { Button, PageHeader, Tab, TabList, TabPanel, Tabs } from "@app/components/v2"; import { ROUTE_PATHS } from "@app/const/routes"; import { @@ -72,6 +73,8 @@ export const AccessManagementPage = () => { } ]; + const hasNoAccess = tabSections.every((tab) => tab.isHidden); + return (
@@ -126,6 +129,7 @@ export const AccessManagementPage = () => { ))}
+ {hasNoAccess && } ); }; diff --git a/frontend/src/pages/organization/AccessManagementPage/components/OrgIdentityTab/components/IdentitySection/IdentityKubernetesAuthForm.tsx b/frontend/src/pages/organization/AccessManagementPage/components/OrgIdentityTab/components/IdentitySection/IdentityKubernetesAuthForm.tsx index 369977a04..708b44a35 100644 --- a/frontend/src/pages/organization/AccessManagementPage/components/OrgIdentityTab/components/IdentitySection/IdentityKubernetesAuthForm.tsx +++ b/frontend/src/pages/organization/AccessManagementPage/components/OrgIdentityTab/components/IdentitySection/IdentityKubernetesAuthForm.tsx @@ -33,20 +33,26 @@ import { useGetIdentityKubernetesAuth, useUpdateIdentityKubernetesAuth } from "@app/hooks/api"; -import { IdentityTrustedIp } from "@app/hooks/api/identities/types"; +import { + IdentityKubernetesAuthTokenReviewMode, + IdentityTrustedIp +} from "@app/hooks/api/identities/types"; import { UsePopUpState } from "@app/hooks/usePopUp"; import { IdentityFormTab } from "./types"; const schema = z .object({ - kubernetesHost: z.string().min(1), + tokenReviewMode: z + .nativeEnum(IdentityKubernetesAuthTokenReviewMode) + .default(IdentityKubernetesAuthTokenReviewMode.Api), + kubernetesHost: z.string().optional().nullable(), tokenReviewerJwt: z.string().optional(), gatewayId: z.string().optional().nullable(), allowedNames: z.string(), allowedNamespaces: z.string(), allowedAudience: z.string(), - caCert: z.string(), + caCert: z.string().optional(), accessTokenTTL: z.string().refine((val) => Number(val) <= 315360000, { message: "Access Token TTL cannot be greater than 315360000" }), @@ -62,7 +68,26 @@ const schema = z ) .min(1) }) - .required(); + .superRefine((data, ctx) => { + if ( + data.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api && + !data.kubernetesHost?.length + ) { + ctx.addIssue({ + path: ["kubernetesHost"], + code: z.ZodIssueCode.custom, + message: "When token review mode is set to API, a Kubernetes host must be provided" + }); + } + + if (data.tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Gateway && !data.gatewayId) { + ctx.addIssue({ + path: ["gatewayId"], + code: z.ZodIssueCode.custom, + message: "When token review mode is set to Gateway, a gateway must be selected" + }); + } + }); export type FormData = z.infer; @@ -100,11 +125,14 @@ export const IdentityKubernetesAuthForm = ({ control, handleSubmit, reset, + watch, + setValue, formState: { isSubmitting } } = useForm({ resolver: zodResolver(schema), defaultValues: { + tokenReviewMode: IdentityKubernetesAuthTokenReviewMode.Api, kubernetesHost: "", tokenReviewerJwt: "", allowedNames: "", @@ -128,6 +156,7 @@ export const IdentityKubernetesAuthForm = ({ useEffect(() => { if (data) { reset({ + tokenReviewMode: data.tokenReviewMode, kubernetesHost: data.kubernetesHost, tokenReviewerJwt: data.tokenReviewerJwt, allowedNames: data.allowedNames, @@ -148,6 +177,7 @@ export const IdentityKubernetesAuthForm = ({ }); } else { reset({ + tokenReviewMode: IdentityKubernetesAuthTokenReviewMode.Api, kubernetesHost: "", tokenReviewerJwt: "", allowedNames: "", @@ -173,6 +203,7 @@ export const IdentityKubernetesAuthForm = ({ accessTokenMaxTTL, accessTokenNumUsesLimit, gatewayId, + tokenReviewMode, accessTokenTrustedIps }: FormData) => { try { @@ -181,7 +212,13 @@ export const IdentityKubernetesAuthForm = ({ if (data) { await updateMutateAsync({ organizationId: orgId, - kubernetesHost, + ...(tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api + ? { + kubernetesHost: kubernetesHost || "" + } + : { + kubernetesHost: null + }), tokenReviewerJwt: tokenReviewerJwt || null, allowedNames, allowedNamespaces, @@ -189,6 +226,7 @@ export const IdentityKubernetesAuthForm = ({ caCert, identityId, gatewayId: gatewayId || null, + tokenReviewMode, accessTokenTTL: Number(accessTokenTTL), accessTokenMaxTTL: Number(accessTokenMaxTTL), accessTokenNumUsesLimit: Number(accessTokenNumUsesLimit), @@ -198,13 +236,20 @@ export const IdentityKubernetesAuthForm = ({ await addMutateAsync({ organizationId: orgId, identityId, - kubernetesHost: kubernetesHost || "", + ...(tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api + ? { + kubernetesHost: kubernetesHost || "" + } + : { + kubernetesHost: null + }), tokenReviewerJwt: tokenReviewerJwt || undefined, allowedNames: allowedNames || "", allowedNamespaces: allowedNamespaces || "", allowedAudience: allowedAudience || "", gatewayId: gatewayId || null, caCert: caCert || "", + tokenReviewMode, accessTokenTTL: Number(accessTokenTTL), accessTokenMaxTTL: Number(accessTokenMaxTTL), accessTokenNumUsesLimit: Number(accessTokenNumUsesLimit), @@ -228,6 +273,8 @@ export const IdentityKubernetesAuthForm = ({ } }; + const tokenReviewMode = watch("tokenReviewMode"); + return (
{ @@ -235,6 +282,7 @@ export const IdentityKubernetesAuthForm = ({ [ "kubernetesHost", "tokenReviewerJwt", + "tokenReviewMode", "gatewayId", "accessTokenTTL", "accessTokenMaxTTL", @@ -253,37 +301,135 @@ export const IdentityKubernetesAuthForm = ({ Advanced - ( - +
+ - - - )} - /> - ( - - - - )} - /> + {(isAllowed) => ( + ( + + +
+ +
+ + + )} + /> + )} + +
+ + ( + + + + )} + /> + + {tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api && ( + ( + + + + )} + /> + )} + + {tokenReviewMode === IdentityKubernetesAuthTokenReviewMode.Api && ( + ( + + + + )} + /> + )} - - {(isAllowed) => ( - ( - - -
- -
- - - )} - /> - )} - - )} /> - ( - -