awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: secrets detection in secret manager

Browse Source
This commit is contained in:
Sheen Capadngan
2025-07-26 03:00:44 +08:00
parent 929822514e
commit c368178cb1
17 changed files with 329 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
backend/src/db/migrations/20250725171821_add-secret-detection-ignore-keys.ts Normal file
View File

@@ -0,0 +1,19 @@
import { Knex } from "knex";
import { TableName } from "../schemas";
export async function up(knex: Knex): Promise<void> {
if (!(await knex.schema.hasColumn(TableName.Project, "secretDetectionIgnoreKeys"))) {
await knex.schema.alterTable(TableName.Project, (t) => {
t.specificType("secretDetectionIgnoreKeys", "text[]");
});
}
}
export async function down(knex: Knex): Promise<void> {
if (await knex.schema.hasColumn(TableName.Project, "secretDetectionIgnoreKeys")) {
await knex.schema.alterTable(TableName.Project, (t) => {
t.dropColumn("secretDetectionIgnoreKeys");
});
}
}

3
backend/src/db/schemas/projects.ts
View File

@@ -30,7 +30,8 @@ export const ProjectsSchema = z.object({
hasDeleteProtection: z.boolean().default(false).nullable().optional(),
secretSharing: z.boolean().default(true),
showSnapshotsLegacy: z.boolean().default(false),
defaultProduct: z.string().nullable().optional()
defaultProduct: z.string().nullable().optional(),
secretDetectionIgnoreKeys: z.string().array().nullable().optional()
});
export type TProjects = z.infer<typeof ProjectsSchema>;

69
backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts
View File

@@ -1,11 +1,20 @@
import { AxiosError } from "axios";
import { exec } from "child_process";
import { join } from "path";
import picomatch from "picomatch";
import RE2 from "re2";
import { readFindingsFile } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import {
createTempFolder,
deleteTempFolder,
readFindingsFile,
writeTextToFile
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
import { BITBUCKET_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION } from "@app/ee/services/secret-scanning-v2/bitbucket";
import { GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION } from "@app/ee/services/secret-scanning-v2/github";
import { getConfig } from "@app/lib/config/env";
import { BadRequestError } from "@app/lib/errors";
import { titleCaseToCamelCase } from "@app/lib/fn";
import { SecretScanningDataSource, SecretScanningFindingSeverity } from "./secret-scanning-v2-enums";
@@ -46,6 +55,19 @@ export function scanDirectory(inputPath: string, outputPath: string, configPath?
});
}
export function scanFile(inputPath: string): Promise<void> {
return new Promise((resolve, reject) => {
const command = `infisical scan --exit-code=77 --source "${inputPath}" --no-git`;
exec(command, (error) => {
if (error && error.code === 77) {
reject(error);
} else {
resolve();
}
});
});
}
export const scanGitRepositoryAndGetFindings = async (
scanPath: string,
findingsPath: string,
@@ -140,3 +162,48 @@ export const parseScanErrorMessage = (err: unknown): string => {
? errorMessage
: `${errorMessage.substring(0, MAX_MESSAGE_LENGTH - 3)}...`;
};
export const scanSecretPolicyViolations = async (
secretPath: string,
secrets: { secretKey: string; secretValue: string }[],
ignoreKeys: string[]
) => {
const appCfg = getConfig();
if (!appCfg.PARAMS_FOLDER_SECRET_DETECTION_ENABLED) {
return;
}
const paramFolderSecretDetectionPaths = appCfg.PARAMS_FOLDER_SECRET_DETECTION_PATHS?.map((el) => el.secretPath) ?? [];
const isPathMatched = paramFolderSecretDetectionPaths.some((pattern) =>
picomatch.isMatch(secretPath, pattern, { strictSlashes: false })
);
if (!isPathMatched) {
return;
}
const tempFolder = await createTempFolder();
try {
let iter = 0;
for await (const secret of secrets) {
if (ignoreKeys.includes(secret.secretKey)) {
// eslint-disable-next-line no-continue
continue;
}
iter += 1;
const secretFilePath = join(tempFolder, `${iter}.txt`);
await writeTextToFile(secretFilePath, `${secret.secretKey}=${secret.secretValue}`);
try {
await scanFile(secretFilePath);
} catch (error) {
throw new BadRequestError({
message: `Secret value detected in ${secret.secretKey}. Please add this instead to the designated secrets path in the project.`,
name: "SecretPolicyViolation"
});
}
}
} finally {
await deleteTempFolder(tempFolder);
}
};

3
backend/src/lib/api-docs/constants.ts
View File

@@ -704,7 +704,8 @@ export const PROJECTS = {
hasDeleteProtection: "Enable or disable delete protection for the project.",
secretSharing: "Enable or disable secret sharing for the project.",
showSnapshotsLegacy: "Enable or disable legacy snapshots for the project.",
defaultProduct: "The default product in which the project will open"
defaultProduct: "The default product in which the project will open",
secretDetectionIgnoreKeys: "The list of secret keys to ignore for secret detection."
},
GET_KEY: {
workspaceId: "The ID of the project to get the key from."

12
backend/src/lib/config/env.ts
View File

@@ -204,6 +204,18 @@ const envSchema = z
WORKFLOW_SLACK_CLIENT_SECRET: zpStr(z.string().optional()),
ENABLE_MSSQL_SECRET_ROTATION_ENCRYPT: zodStrBool.default("true"),
// Special Detection Feature
PARAMS_FOLDER_SECRET_DETECTION_PATHS: zpStr(
z
.string()
.optional()
.transform((val) => {
if (!val) return undefined;
return JSON.parse(val) as { secretPath: string }[];
})
),
PARAMS_FOLDER_SECRET_DETECTION_ENABLED: zodStrBool.default("false"),
// HSM
HSM_LIB_PATH: zpStr(z.string().optional()),
HSM_PIN: zpStr(z.string().optional()),

1
backend/src/server/routes/index.ts
View File

@@ -1231,6 +1231,7 @@ export const registerRoutes = async (
const secretV2BridgeService = secretV2BridgeServiceFactory({
folderDAL,
projectDAL,
secretVersionDAL: secretVersionV2BridgeDAL,
folderCommitService,
secretQueueService,

3
backend/src/server/routes/sanitizedSchemas.ts
View File

@@ -264,7 +264,8 @@ export const SanitizedProjectSchema = ProjectsSchema.pick({
auditLogsRetentionDays: true,
hasDeleteProtection: true,
secretSharing: true,
showSnapshotsLegacy: true
showSnapshotsLegacy: true,
secretDetectionIgnoreKeys: true
});
export const SanitizedTagSchema = SecretTagsSchema.pick({

6
backend/src/server/routes/v1/admin-router.ts
View File

@@ -52,7 +52,8 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => {
defaultAuthOrgAuthEnforced: z.boolean().nullish(),
defaultAuthOrgAuthMethod: z.string().nullish(),
isSecretScanningDisabled: z.boolean(),
kubernetesAutoFetchServiceAccountToken: z.boolean()
kubernetesAutoFetchServiceAccountToken: z.boolean(),
paramsFolderSecretDetectionEnabled: z.boolean()
})
})
}
@@ -67,7 +68,8 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => {
fipsEnabled: crypto.isFipsModeEnabled(),
isMigrationModeOn: serverEnvs.MAINTENANCE_MODE,
isSecretScanningDisabled: serverEnvs.DISABLE_SECRET_SCANNING,
kubernetesAutoFetchServiceAccountToken: serverEnvs.KUBERNETES_AUTO_FETCH_SERVICE_ACCOUNT_TOKEN
kubernetesAutoFetchServiceAccountToken: serverEnvs.KUBERNETES_AUTO_FETCH_SERVICE_ACCOUNT_TOKEN,
paramsFolderSecretDetectionEnabled: serverEnvs.PARAMS_FOLDER_SECRET_DETECTION_ENABLED
}
};
}

6
backend/src/server/routes/v1/project-router.ts
View File

@@ -369,7 +369,8 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
.describe(PROJECTS.UPDATE.slug),
secretSharing: z.boolean().optional().describe(PROJECTS.UPDATE.secretSharing),
showSnapshotsLegacy: z.boolean().optional().describe(PROJECTS.UPDATE.showSnapshotsLegacy),
defaultProduct: z.nativeEnum(ProjectType).optional().describe(PROJECTS.UPDATE.defaultProduct)
defaultProduct: z.nativeEnum(ProjectType).optional().describe(PROJECTS.UPDATE.defaultProduct),
secretDetectionIgnoreKeys: z.array(z.string()).optional().describe(PROJECTS.UPDATE.secretDetectionIgnoreKeys)
}),
response: {
200: z.object({
@@ -392,7 +393,8 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
hasDeleteProtection: req.body.hasDeleteProtection,
slug: req.body.slug,
secretSharing: req.body.secretSharing,
showSnapshotsLegacy: req.body.showSnapshotsLegacy
showSnapshotsLegacy: req.body.showSnapshotsLegacy,
secretDetectionIgnoreKeys: req.body.secretDetectionIgnoreKeys
},
actorAuthMethod: req.permission.authMethod,
actorId: req.permission.id,

11
backend/src/services/project/project-service.ts
View File

@@ -645,7 +645,7 @@ export const projectServiceFactory = ({
const updateProject = async ({ actor, actorId, actorOrgId, actorAuthMethod, update, filter }: TUpdateProjectDTO) => {
const project = await projectDAL.findProjectByFilter(filter);
const { permission } = await permissionService.getProjectPermission({
const { permission, hasRole } = await permissionService.getProjectPermission({
actor,
actorId,
projectId: project.id,
@@ -667,6 +667,12 @@ export const projectServiceFactory = ({
}
}
if (update.secretDetectionIgnoreKeys && !hasRole(ProjectMembershipRole.Admin)) {
throw new ForbiddenRequestError({
message: "Only admins can update secret detection ignore keys"
});
}
const updatedProject = await projectDAL.updateById(project.id, {
name: update.name,
description: update.description,
@@ -676,7 +682,8 @@ export const projectServiceFactory = ({
slug: update.slug,
secretSharing: update.secretSharing,
defaultProduct: update.defaultProduct,
showSnapshotsLegacy: update.showSnapshotsLegacy
showSnapshotsLegacy: update.showSnapshotsLegacy,
secretDetectionIgnoreKeys: update.secretDetectionIgnoreKeys
});
return updatedProject;

1
backend/src/services/project/project-types.ts
View File

@@ -96,6 +96,7 @@ export type TUpdateProjectDTO = {
slug?: string;
secretSharing?: boolean;
showSnapshotsLegacy?: boolean;
secretDetectionIgnoreKeys?: string[];
};
} & Omit<TProjectPermission, "projectId">;

45
backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts
View File

@@ -25,6 +25,7 @@ import {
import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal";
import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal";
import { scanSecretPolicyViolations } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
import { TKeyStoreFactory } from "@app/keystore/keystore";
import { DatabaseErrorCode } from "@app/lib/error-codes";
@@ -38,6 +39,7 @@ import { ActorType } from "../auth/auth-type";
import { TCommitResourceChangeDTO, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service";
import { TKmsServiceFactory } from "../kms/kms-service";
import { KmsDataKey } from "../kms/kms-types";
import { TProjectDALFactory } from "../project/project-dal";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { TReminderServiceFactory } from "../reminder/reminder-types";
import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal";
@@ -88,6 +90,7 @@ import { TSecretVersionV2TagDALFactory } from "./secret-version-tag-dal";
type TSecretV2BridgeServiceFactoryDep = {
secretDAL: TSecretV2BridgeDALFactory;
projectDAL: Pick<TProjectDALFactory, "findById">;
secretVersionDAL: TSecretVersionV2DALFactory;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
secretVersionTagDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
@@ -126,6 +129,7 @@ export type TSecretV2BridgeServiceFactory = ReturnType<typeof secretV2BridgeServ
*/
export const secretV2BridgeServiceFactory = ({
secretDAL,
projectDAL,
projectEnvDAL,
secretTagDAL,
secretVersionDAL,
@@ -295,6 +299,18 @@ export const secretV2BridgeServiceFactory = ({
})
);
const project = await projectDAL.findById(projectId);
await scanSecretPolicyViolations(
secretPath,
[
{
secretKey: inputSecret.secretName,
secretValue: inputSecret.secretValue
}
],
project.secretDetectionIgnoreKeys || []
);
const { nestedReferences, localReferences } = getAllSecretReferences(inputSecret.secretValue);
const allSecretReferences = nestedReferences.concat(
localReferences.map((el) => ({ secretKey: el, secretPath, environment }))
@@ -506,6 +522,20 @@ export const secretV2BridgeServiceFactory = ({
const { secretName, secretValue } = inputSecret;
if (secretValue) {
const project = await projectDAL.findById(projectId);
await scanSecretPolicyViolations(
secretPath,
[
{
secretKey: inputSecret.newSecretName || secretName,
secretValue
}
],
project.secretDetectionIgnoreKeys || []
);
}
const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId
@@ -1585,6 +1615,9 @@ export const secretV2BridgeServiceFactory = ({
if (secrets.length)
throw new BadRequestError({ message: `Secret already exist: ${secrets.map((el) => el.key).join(",")}` });
const project = await projectDAL.findById(projectId);
await scanSecretPolicyViolations(secretPath, inputSecrets, project.secretDetectionIgnoreKeys || []);
// get all tags
const sanitizedTagIds = inputSecrets.flatMap(({ tagIds = [] }) => tagIds);
const tags = sanitizedTagIds.length ? await secretTagDAL.findManyTagsById(projectId, sanitizedTagIds) : [];
@@ -1925,6 +1958,18 @@ export const secretV2BridgeServiceFactory = ({
});
await $validateSecretReferences(projectId, permission, secretReferences, tx);
const project = await projectDAL.findById(projectId);
await scanSecretPolicyViolations(
secretPath,
secretsToUpdate
.filter((el) => el.secretValue)
.map((el) => ({
secretKey: el.newSecretName || el.secretKey,
secretValue: el.secretValue as string
})),
project.secretDetectionIgnoreKeys || []
);
const bulkUpdatedSecrets = await fnSecretBulkUpdate({
folderId,
orgId: actorOrgId,

1
frontend/src/hooks/api/admin/types.ts
View File

@@ -51,6 +51,7 @@ export type TServerConfig = {
invalidatingCache: boolean;
fipsEnabled: boolean;
envOverrides?: Record<string, string>;
paramsFolderSecretDetectionEnabled: boolean;
};
export type TUpdateServerConfigDTO = {

6
frontend/src/hooks/api/workspace/queries.tsx
View File

@@ -281,7 +281,8 @@ export const useUpdateProject = () => {
newProjectDescription,
newSlug,
secretSharing,
showSnapshotsLegacy
showSnapshotsLegacy,
secretDetectionIgnoreKeys
}) => {
const { data } = await apiRequest.patch<{ workspace: Workspace }>(
`/api/v1/workspace/${projectID}`,
@@ -290,7 +291,8 @@ export const useUpdateProject = () => {
description: newProjectDescription,
slug: newSlug,
secretSharing,
showSnapshotsLegacy
showSnapshotsLegacy,
secretDetectionIgnoreKeys
}
);
return data.workspace;

2
frontend/src/hooks/api/workspace/types.ts
View File

@@ -40,6 +40,7 @@ export type Workspace = {
hasDeleteProtection: boolean;
secretSharing: boolean;
showSnapshotsLegacy: boolean;
secretDetectionIgnoreKeys: string[];
};
export type WorkspaceEnv = {
@@ -81,6 +82,7 @@ export type UpdateProjectDTO = {
newSlug?: string;
secretSharing?: boolean;
showSnapshotsLegacy?: boolean;
secretDetectionIgnoreKeys?: string[];
};
export type UpdatePitVersionLimitDTO = { projectSlug: string; pitVersionLimit: number };

6
frontend/src/pages/secret-manager/SettingsPage/components/ProjectGeneralTab/ProjectGeneralTab.tsx
View File

@@ -1,12 +1,17 @@
import { useServerConfig } from "@app/context";
import { AutoCapitalizationSection } from "../AutoCapitalizationSection";
import { BackfillSecretReferenceSecretion } from "../BackfillSecretReferenceSection";
import { EnvironmentSection } from "../EnvironmentSection";
import { PointInTimeVersionLimitSection } from "../PointInTimeVersionLimitSection";
import { SecretDetectionIgnoreKeysSection } from "../SecretDetectionIgnoreKeysSection/SecretDetectionIgnoreKeysSection";
import { SecretSharingSection } from "../SecretSharingSection";
import { SecretSnapshotsLegacySection } from "../SecretSnapshotsLegacySection";
import { SecretTagsSection } from "../SecretTagsSection";
export const SecretSettingsTab = () => {
const { config } = useServerConfig();
return (
<div>
<EnvironmentSection />
@@ -15,6 +20,7 @@ export const SecretSettingsTab = () => {
<SecretSharingSection />
<SecretSnapshotsLegacySection />
<PointInTimeVersionLimitSection />
{config.paramsFolderSecretDetectionEnabled && <SecretDetectionIgnoreKeysSection />}
<BackfillSecretReferenceSecretion />
</div>
);

147
frontend/src/pages/secret-manager/SettingsPage/components/SecretDetectionIgnoreKeysSection/SecretDetectionIgnoreKeysSection.tsx Normal file
View File

@@ -0,0 +1,147 @@
import { useEffect } from "react";
import { Controller, useFieldArray, useForm } from "react-hook-form";
import { faPlus, faTrash } from "@fortawesome/free-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { zodResolver } from "@hookform/resolvers/zod";
import { z } from "zod";
import { createNotification } from "@app/components/notifications";
import { Button, FormControl, IconButton, Input } from "@app/components/v2";
import { useProjectPermission, useWorkspace } from "@app/context";
import { useUpdateProject } from "@app/hooks/api";
import { ProjectMembershipRole } from "@app/hooks/api/roles/types";
const formSchema = z.object({
ignoreKeys: z
.object({
key: z.string().trim().min(1, "Secret key name is required")
})
.array()
.default([])
});
type TForm = z.infer<typeof formSchema>;
export const SecretDetectionIgnoreKeysSection = () => {
const { currentWorkspace } = useWorkspace();
const { membership } = useProjectPermission();
const { mutateAsync: updateProject } = useUpdateProject();
const {
control,
formState: { isSubmitting, isDirty },
handleSubmit,
reset
} = useForm<TForm>({
resolver: zodResolver(formSchema),
defaultValues: {
ignoreKeys: []
}
});
const ignoreKeysFormFields = useFieldArray({
control,
name: "ignoreKeys"
});
useEffect(() => {
const existingIgnoreKeys = currentWorkspace?.secretDetectionIgnoreKeys || [];
reset({
ignoreKeys:
existingIgnoreKeys.length > 0 ? existingIgnoreKeys.map((key) => ({ key })) : [{ key: "" }] // Show one empty field by default
});
}, [currentWorkspace?.secretDetectionIgnoreKeys, reset]);
const handleIgnoreKeysSubmit = async ({ ignoreKeys }: TForm) => {
try {
await updateProject({
projectID: currentWorkspace.id,
secretDetectionIgnoreKeys: ignoreKeys.map((item) => item.key)
});
createNotification({
text: "Successfully updated secret detection ignore keys",
type: "success"
});
} catch {
createNotification({
text: "Failed updating secret detection ignore keys",
type: "error"
});
}
};
const isAdmin = membership.roles.includes(ProjectMembershipRole.Admin);
if (!currentWorkspace) return null;
return (
<div className="mb-6 rounded-lg border border-mineshaft-600 bg-mineshaft-900 p-4">
<div className="flex w-full items-center justify-between">
<p className="text-xl font-semibold">Secret Detection Ignore Keys</p>
</div>
<p className="mb-4 mt-2 max-w-2xl text-sm text-gray-400">
Define secret keys that should be ignored when scanning parameter folders for misplaced
secrets. These keys will not trigger policy violation alerts even if they contain sensitive
data.
</p>
<form onSubmit={handleSubmit(handleIgnoreKeysSubmit)} autoComplete="off">
<div className="mb-4">
<p className="mb-3 text-sm font-medium text-gray-300">Ignored Secret Keys</p>
<div className="flex flex-col space-y-2">
{ignoreKeysFormFields.fields.map(({ id: ignoreKeyFieldId }, i) => (
<div key={ignoreKeyFieldId} className="flex items-end space-x-2">
<div className="flex-grow">
{i === 0 && <span className="text-xs text-mineshaft-400">Secret Key Name</span>}
<Controller
control={control}
name={`ignoreKeys.${i}.key`}
render={({ field, fieldState: { error } }) => (
<FormControl
isError={Boolean(error?.message)}
errorText={error?.message}
className="mb-0"
>
<Input {...field} placeholder="PUBLIC_API_KEY" isDisabled={!isAdmin} />
</FormControl>
)}
/>
</div>
<IconButton
ariaLabel="delete ignore key"
className="bottom-0.5 h-9"
variant="outline_bg"
onClick={() => ignoreKeysFormFields.remove(i)}
isDisabled={!isAdmin}
>
<FontAwesomeIcon icon={faTrash} />
</IconButton>
</div>
))}
<div className="mt-2 flex justify-end">
<Button
leftIcon={<FontAwesomeIcon icon={faPlus} />}
size="xs"
variant="outline_bg"
onClick={() => ignoreKeysFormFields.append({ key: "" })}
isDisabled={!isAdmin}
>
Add Ignore Key
</Button>
</div>
</div>
</div>
<Button
colorSchema="secondary"
type="submit"
isLoading={isSubmitting}
disabled={!isAdmin || !isDirty}
>
Save
</Button>
</form>
</div>
);
};