awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-14 10:29:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

improvements: address feedback and setup queue worker profiles

Browse Source
This commit is contained in:
Scott Wilson
2025-06-02 14:40:06 -07:00
parent 3ae2ec1f51
commit d924580599
31 changed files with 356 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-router.ts
+6 -6
View File
@@ -189,7 +189,7 @@ export const registerSecretScanningV2Router = async (server: FastifyZodProvider)
server.route({
method: "GET",
url: "/configs/:projectId",
url: "/configs",
config: {
rateLimit: readLimit
},
@@ -197,7 +197,7 @@ export const registerSecretScanningV2Router = async (server: FastifyZodProvider)
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: "Get the Secret Scanning Config for the specified project.",
params: z.object({
querystring: z.object({
projectId: z
.string()
.trim()
@@ -213,7 +213,7 @@ export const registerSecretScanningV2Router = async (server: FastifyZodProvider)
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const {
params: { projectId },
query: { projectId },
permission
} = req;
@@ -233,7 +233,7 @@ export const registerSecretScanningV2Router = async (server: FastifyZodProvider)
server.route({
method: "PATCH",
url: "/configs/:projectId",
url: "/configs",
config: {
rateLimit: writeLimit
},
@@ -241,7 +241,7 @@ export const registerSecretScanningV2Router = async (server: FastifyZodProvider)
hide: false,
tags: [ApiDocsTags.SecretScanning],
description: "Update the specified Secret Scanning Configuration.",
params: z.object({
querystring: z.object({
projectId: z.string().trim().min(1, "Finding ID required").describe(SecretScanningConfigs.UPDATE.projectId)
}),
body: z.object({
@@ -254,7 +254,7 @@ export const registerSecretScanningV2Router = async (server: FastifyZodProvider)
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const {
params: { projectId },
query: { projectId },
body,
permission
} = req;
backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-factory.ts
+7
View File
@@ -1,5 +1,6 @@
import { join } from "path";
import { ProbotOctokit } from "probot";
import RE2 from "re2";
import { scanContentAndGetFindings } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
@@ -28,6 +29,8 @@ import { listGitHubRadarRepositories, TGitHubRadarConnection } from "@app/servic
import { TGitHubDataSourceWithConnection, TQueueGitHubResourceDiffScan } from "./github-secret-scanning-types";
const GitHubRepositoryRegex = new RE2(/^[a-zA-Z0-9._-]+\/[a-zA-Z0-9._-]+$/);
export const GitHubSecretScanningFactory = () => {
const initialize: TSecretScanningFactoryInitialize<TGitHubRadarConnection> = async (
{ connection, secretScanningV2DAL },
@@ -104,6 +107,10 @@ export const GitHubSecretScanningFactory = () => {
const repoPath = join(tempFolder, "repo.git");
if (!GitHubRepositoryRegex.test(resourceName)) {
throw new Error("Invalid GitHub repository name");
}
await cloneRepository({
cloneUrl: `https://x-access-token:${token}@github.com/${resourceName}.git`,
repoPath
backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-service.ts
+2 -1
View File
@@ -13,7 +13,8 @@ export const githubSecretScanningService = (
) => {
const handleInstallationDeletedEvent = async (installationId: number) => {
const dataSource = await secretScanningV2DAL.dataSources.findOne({
externalId: String(installationId)
externalId: String(installationId),
type: SecretScanningDataSource.GitHub
});
if (!dataSource) {
backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts
+4 -1
View File
@@ -1,5 +1,6 @@
import { AxiosError } from "axios";
import { exec } from "child_process";
import RE2 from "re2";
import { readFindingsFile } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
@@ -83,6 +84,8 @@ export const replaceNonChangesWithNewlines = (patch: string) => {
.join("\n");
};
const HunkHeaderRegex = new RE2(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/);
export const convertPatchLineToFileLineNumber = (patch: string, patchLineNumber: number) => {
const lines = patch.split("\n");
let currentPatchLine = 0;
@@ -92,7 +95,7 @@ export const convertPatchLineToFileLineNumber = (patch: string, patchLineNumber:
currentPatchLine += 1;
// Hunk header: @@ -a,b +c,d @@
const hunkHeaderMatch = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/);
const hunkHeaderMatch = line.match(HunkHeaderRegex);
if (hunkHeaderMatch) {
const startLine = parseInt(hunkHeaderMatch[1], 10);
currentNewLine = startLine;
backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
+4 -7
View File
@@ -28,7 +28,6 @@ import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
import { TSecretScanningV2DALFactory } from "./secret-scanning-v2-dal";
import {
SecretScanningDataSource,
SecretScanningFindingStatus,
SecretScanningResource,
SecretScanningScanStatus,
SecretScanningScanType
@@ -217,12 +216,11 @@ export const secretScanningV2QueueServiceFactory = async ({
dataSourceType: dataSource.type,
resourceName: resource.name,
resourceType: resource.type,
scanId,
status: SecretScanningFindingStatus.Unresolved
scanId
})),
["projectId", "fingerprint"],
tx,
["resourceName", "dataSourceName", "status"]
["resourceName", "dataSourceName"]
);
}
@@ -441,12 +439,11 @@ export const secretScanningV2QueueServiceFactory = async ({
dataSourceType: dataSource.type,
resourceName: resource.name,
resourceType: resource.type,
scanId,
status: SecretScanningFindingStatus.Unresolved
scanId
})),
["projectId", "fingerprint"],
tx,
["resourceName", "dataSourceName", "status"]
["resourceName", "dataSourceName"]
);
}
backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-service.ts
+24 -1
View File
@@ -1,4 +1,5 @@
import { ForbiddenError } from "@casl/ability";
import { join } from "path";
import { ActionProjectType } from "@app/db/schemas";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
@@ -9,6 +10,12 @@ import {
ProjectPermissionSecretScanningFindingActions,
ProjectPermissionSub
} from "@app/ee/services/permission/project-permission";
import {
createTempFolder,
deleteTempFolder,
scanContentAndGetFindings,
writeTextToFile
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import { githubSecretScanningService } from "@app/ee/services/secret-scanning-v2/github/github-secret-scanning-service";
import { SecretScanningFindingStatus } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { SECRET_SCANNING_FACTORY_MAP } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-factory";
@@ -810,10 +817,26 @@ export const secretScanningV2ServiceFactory = ({
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSecretScanningConfigActions.Read,
ProjectPermissionSecretScanningConfigActions.Update,
ProjectPermissionSub.SecretScanningConfigs
);
if (content) {
const tempFolder = await createTempFolder();
try {
const configPath = join(tempFolder, "infisical-scan.toml");
await writeTextToFile(configPath, content);
await scanContentAndGetFindings("", configPath);
} catch (e) {
throw new BadRequestError({
message: "Unable to parse configuration: Check syntax and formatting."
});
} finally {
await deleteTempFolder(tempFolder);
}
}
const [config] = await secretScanningV2DAL.configs.upsert(
[
{
backend/src/lib/config/env.ts
+3
View File
@@ -1,5 +1,7 @@
import { z } from "zod";
import { QueueWorkerProfile } from "@app/lib/types";
import { removeTrailingSlash } from "../fn";
import { CustomLogger } from "../logger/logger";
import { zpStr } from "../zod";
@@ -57,6 +59,7 @@ const envSchema = z
ENCRYPTION_KEY: zpStr(z.string().optional()),
ROOT_ENCRYPTION_KEY: zpStr(z.string().optional()),
QUEUE_WORKERS_ENABLED: zodStrBool.default("true"),
QUEUE_WORKER_PROFILE: z.nativeEnum(QueueWorkerProfile).default(QueueWorkerProfile.All),
HTTPS_ENABLED: zodStrBool,
ROTATION_DEVELOPMENT_MODE: zodStrBool.default("false").optional(),
// smtp options
backend/src/lib/types/index.ts
+6
View File
@@ -78,3 +78,9 @@ export type OrgServiceActor = {
authMethod: ActorAuthMethod;
orgId: string;
};
export enum QueueWorkerProfile {
All = "all",
Standard = "standard",
SecretScanning = "secret-scanning"
}
backend/src/queue/queue-service.ts
+58 -2
View File
@@ -19,6 +19,7 @@ import {
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import { getConfig } from "@app/lib/config/env";
import { logger } from "@app/lib/logger";
import { QueueWorkerProfile } from "@app/lib/types";
import {
TFailedIntegrationSyncEmailsPayload,
TIntegrationSyncPayload,
@@ -269,6 +270,37 @@ export type TQueueJobTypes = {
};
};
const SECRET_SCANNING_JOBS = [
QueueJobs.SecretScanningV2FullScan,
QueueJobs.SecretScanningV2DiffScan,
QueueJobs.SecretScanningV2SendNotification,
QueueJobs.SecretScan
];
const NON_STANDARD_JOBS = [...SECRET_SCANNING_JOBS];
const SECRET_SCANNING_QUEUES = [
QueueName.SecretScanningV2,
QueueName.SecretFullRepoScan,
QueueName.SecretPushEventScan
];
const NON_STANDARD_QUEUES = [...SECRET_SCANNING_QUEUES];
const isQueueEnabled = (name: QueueName) => {
const appCfg = getConfig();
switch (appCfg.QUEUE_WORKER_PROFILE) {
case QueueWorkerProfile.Standard:
return !NON_STANDARD_QUEUES.includes(name);
case QueueWorkerProfile.SecretScanning:
return SECRET_SCANNING_QUEUES.includes(name);
case QueueWorkerProfile.All:
default:
// allow all
return true;
}
};
export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;
export const queueServiceFactory = (
redisUrl: string,
@@ -325,7 +357,7 @@ export const queueServiceFactory = (
});
const appCfg = getConfig();
if (appCfg.QUEUE_WORKERS_ENABLED) {
if (appCfg.QUEUE_WORKERS_ENABLED && isQueueEnabled(name)) {
workerContainer[name] = new Worker<TQueueJobTypes[T]["payload"], void, TQueueJobTypes[T]["name"]>(name, jobFn, {
...queueSettings,
connection
@@ -344,6 +376,30 @@ export const queueServiceFactory = (
throw new Error(`${jobName} queue is already initialized`);
}
const appCfg = getConfig();
if (!appCfg.QUEUE_WORKERS_ENABLED) return;
switch (appCfg.QUEUE_WORKER_PROFILE) {
case QueueWorkerProfile.Standard:
if (NON_STANDARD_JOBS.includes(jobName)) {
// only process standard jobs
return;
}
break;
case QueueWorkerProfile.SecretScanning:
if (!SECRET_SCANNING_JOBS.includes(jobName)) {
// only process secret scanning jobs
return;
}
break;
case QueueWorkerProfile.All:
default:
// allow all
}
await pgBoss.createQueue(jobName);
queueContainerPg[jobName] = true;
@@ -363,7 +419,7 @@ export const queueServiceFactory = (
listener: WorkerListener<TQueueJobTypes[T]["payload"], void, TQueueJobTypes[T]["name"]>[U]
) => {
const appCfg = getConfig();
if (!appCfg.QUEUE_WORKERS_ENABLED) {
if (!appCfg.QUEUE_WORKERS_ENABLED || !isQueueEnabled(name)) {
return;
}