mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-10-19 14:33:52 +00:00
feat: changed backend-pg to backend
This commit is contained in:
Akhil Mohan
@@ -0,0 +1 @@
|
||||
export * from "./secret-scanning-queue";
|
||||
@@ -0,0 +1,151 @@
|
||||
import { exec } from "child_process";
|
||||
import { mkdir, readFile, rm, writeFile } from "fs";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
|
||||
import { SecretMatch } from "./secret-scanning-queue-types";
|
||||
|
||||
export function createTempFolder(): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const tempDir = tmpdir();
|
||||
const tempFolderName = Math.random().toString(36).substring(2);
|
||||
const tempFolderPath = join(tempDir, tempFolderName);
|
||||
|
||||
mkdir(tempFolderPath, (err: any) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve(tempFolderPath);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function writeTextToFile(filePath: string, content: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
writeFile(filePath, content, (err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function cloneRepo(
|
||||
installationAcccessToken: string,
|
||||
repositoryFullName: string,
|
||||
repoPath: string
|
||||
): Promise<void> {
|
||||
const cloneUrl = `https://x-access-token:${installationAcccessToken}@github.com/${repositoryFullName}.git`;
|
||||
const command = `git clone ${cloneUrl} ${repoPath} --bare`;
|
||||
return new Promise((resolve, reject) => {
|
||||
exec(command, (error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function runInfisicalScanOnRepo(repoPath: string, outputPath: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const command = `cd ${repoPath} && infisical scan --exit-code=77 -r "${outputPath}"`;
|
||||
exec(command, (error) => {
|
||||
if (error && error.code !== 77) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function runInfisicalScan(inputPath: string, outputPath: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const command = `cat "${inputPath}" | infisical scan --exit-code=77 --pipe -r "${outputPath}"`;
|
||||
exec(command, (error) => {
|
||||
if (error && error.code !== 77) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function readFindingsFile(filePath: string): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
readFile(filePath, "utf8", (err, data) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve(data);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function deleteTempFolder(folderPath: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
rm(folderPath, { recursive: true }, (err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function convertKeysToLowercase<T>(obj: T): T {
|
||||
const convertedObj = {} as T;
|
||||
|
||||
for (const key in obj) {
|
||||
if (Object.prototype.hasOwnProperty.call(obj, key)) {
|
||||
const lowercaseKey = key.charAt(0).toLowerCase() + key.slice(1);
|
||||
convertedObj[lowercaseKey as keyof T] = obj[key];
|
||||
}
|
||||
}
|
||||
|
||||
return convertedObj;
|
||||
}
|
||||
|
||||
export async function scanFullRepoContentAndGetFindings(
|
||||
octokit: any,
|
||||
installationId: string,
|
||||
repositoryFullName: string
|
||||
): Promise<SecretMatch[]> {
|
||||
const tempFolder = await createTempFolder();
|
||||
const findingsPath = join(tempFolder, "findings.json");
|
||||
const repoPath = join(tempFolder, "repo.git");
|
||||
try {
|
||||
const {
|
||||
data: { token }
|
||||
} = await octokit.apps.createInstallationAccessToken({ installation_id: installationId });
|
||||
await cloneRepo(token, repositoryFullName, repoPath);
|
||||
await runInfisicalScanOnRepo(repoPath, findingsPath);
|
||||
const findingsData = await readFindingsFile(findingsPath);
|
||||
return JSON.parse(findingsData);
|
||||
} finally {
|
||||
await deleteTempFolder(tempFolder);
|
||||
}
|
||||
}
|
||||
|
||||
export async function scanContentAndGetFindings(textContent: string): Promise<SecretMatch[]> {
|
||||
const tempFolder = await createTempFolder();
|
||||
const filePath = join(tempFolder, "content.txt");
|
||||
const findingsPath = join(tempFolder, "findings.json");
|
||||
|
||||
try {
|
||||
await writeTextToFile(filePath, textContent);
|
||||
await runInfisicalScan(filePath, findingsPath);
|
||||
const findingsData = await readFindingsFile(findingsPath);
|
||||
return JSON.parse(findingsData);
|
||||
} finally {
|
||||
await deleteTempFolder(tempFolder);
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
import { Commit } from "@octokit/webhooks-types";
|
||||
|
||||
export type SecretMatch = {
|
||||
Description: string;
|
||||
StartLine: number;
|
||||
EndLine: number;
|
||||
StartColumn: number;
|
||||
EndColumn: number;
|
||||
Match: string;
|
||||
Secret: string;
|
||||
File: string;
|
||||
SymlinkFile: string;
|
||||
Commit: string;
|
||||
Entropy: number;
|
||||
Author: string;
|
||||
Email: string;
|
||||
Date: string;
|
||||
Message: string;
|
||||
Tags: string[];
|
||||
RuleID: string;
|
||||
Fingerprint: string;
|
||||
FingerPrintWithoutCommitId: string;
|
||||
};
|
||||
|
||||
export type TScanPushEventPayload = {
|
||||
organizationId: string;
|
||||
commits: Commit[];
|
||||
pusher: {
|
||||
name: string;
|
||||
email: string | null;
|
||||
};
|
||||
repository: {
|
||||
id: number;
|
||||
fullName: string;
|
||||
};
|
||||
installationId: string;
|
||||
};
|
||||
|
||||
export type TScanFullRepoEventPayload = {
|
||||
organizationId: string;
|
||||
installationId: string;
|
||||
repository: {
|
||||
id: number;
|
||||
fullName: string;
|
||||
};
|
||||
};
|
||||
+254
@@ -0,0 +1,254 @@
|
||||
import { ProbotOctokit } from "probot";
|
||||
|
||||
import { OrgMembershipRole } from "@app/db/schemas";
|
||||
import { getConfig } from "@app/lib/config/env";
|
||||
import { logger } from "@app/lib/logger";
|
||||
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
|
||||
import { TOrgDALFactory } from "@app/services/org/org-dal";
|
||||
import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
|
||||
import { TTelemetryServiceFactory } from "@app/services/telemetry/telemetry-service";
|
||||
import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
|
||||
|
||||
import { TSecretScanningDALFactory } from "../secret-scanning-dal";
|
||||
import {
|
||||
scanContentAndGetFindings,
|
||||
scanFullRepoContentAndGetFindings
|
||||
} from "./secret-scanning-fns";
|
||||
import {
|
||||
SecretMatch,
|
||||
TScanFullRepoEventPayload,
|
||||
TScanPushEventPayload
|
||||
} from "./secret-scanning-queue-types";
|
||||
|
||||
type TSecretScanningQueueFactoryDep = {
|
||||
queueService: TQueueServiceFactory;
|
||||
secretScanningDAL: TSecretScanningDALFactory;
|
||||
smtpService: Pick<TSmtpService, "sendMail">;
|
||||
orgMembershipDAL: Pick<TOrgDALFactory, "findMembership">;
|
||||
telemetryService: Pick<TTelemetryServiceFactory, "sendPostHogEvents">;
|
||||
};
|
||||
|
||||
export type TSecretScanningQueueFactory = ReturnType<typeof secretScanningQueueFactory>;
|
||||
|
||||
export const secretScanningQueueFactory = ({
|
||||
queueService,
|
||||
secretScanningDAL,
|
||||
smtpService,
|
||||
telemetryService,
|
||||
orgMembershipDAL: orgMemberDAL
|
||||
}: TSecretScanningQueueFactoryDep) => {
|
||||
const startFullRepoScan = async (payload: TScanFullRepoEventPayload) => {
|
||||
await queueService.queue(QueueName.SecretFullRepoScan, QueueJobs.SecretScan, payload, {
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: "exponential",
|
||||
delay: 5000
|
||||
},
|
||||
removeOnComplete: true,
|
||||
removeOnFail: {
|
||||
count: 20 // keep the most recent 20 jobs
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const startPushEventScan = async (payload: TScanPushEventPayload) => {
|
||||
await queueService.queue(QueueName.SecretPushEventScan, QueueJobs.SecretScan, payload, {
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: "exponential",
|
||||
delay: 5000
|
||||
},
|
||||
removeOnComplete: true,
|
||||
removeOnFail: {
|
||||
count: 20 // keep the most recent 20 jobs
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const getOrgAdminEmails = async (organizationId: string) => {
|
||||
// get emails of admins
|
||||
const adminsOfWork = await orgMemberDAL.findMembership({
|
||||
orgId: organizationId,
|
||||
role: OrgMembershipRole.Admin
|
||||
});
|
||||
return adminsOfWork.map((userObject) => userObject.email);
|
||||
};
|
||||
|
||||
queueService.start(QueueName.SecretPushEventScan, async (job) => {
|
||||
const appCfg = getConfig();
|
||||
const { organizationId, commits, pusher, repository, installationId } = job.data;
|
||||
const [owner, repo] = repository.fullName.split("/");
|
||||
const octokit = new ProbotOctokit({
|
||||
auth: {
|
||||
appId: appCfg.SECRET_SCANNING_GIT_APP_ID,
|
||||
privateKey: appCfg.SECRET_SCANNING_PRIVATE_KEY,
|
||||
installationId
|
||||
}
|
||||
});
|
||||
const allFindingsByFingerprint: { [key: string]: SecretMatch } = {};
|
||||
|
||||
for (const commit of commits) {
|
||||
for (const filepath of [...commit.added, ...commit.modified]) {
|
||||
// eslint-disable-next-line
|
||||
const fileContentsResponse = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path: filepath
|
||||
});
|
||||
|
||||
const { data }: any = fileContentsResponse;
|
||||
const fileContent = Buffer.from(data.content, "base64").toString();
|
||||
|
||||
// eslint-disable-next-line
|
||||
const findings = await scanContentAndGetFindings(`\n${fileContent}`); // extra line to count lines correctly
|
||||
|
||||
for (const finding of findings) {
|
||||
const fingerPrintWithCommitId = `${commit.id}:${filepath}:${finding.RuleID}:${finding.StartLine}`;
|
||||
const fingerPrintWithoutCommitId = `${filepath}:${finding.RuleID}:${finding.StartLine}`;
|
||||
finding.Fingerprint = fingerPrintWithCommitId;
|
||||
finding.FingerPrintWithoutCommitId = fingerPrintWithoutCommitId;
|
||||
finding.Commit = commit.id;
|
||||
finding.File = filepath;
|
||||
finding.Author = commit.author.name;
|
||||
finding.Email = commit?.author?.email ? commit?.author?.email : "";
|
||||
|
||||
allFindingsByFingerprint[fingerPrintWithCommitId] = finding;
|
||||
}
|
||||
}
|
||||
}
|
||||
await secretScanningDAL.transaction(async (tx) => {
|
||||
if (!Object.keys(allFindingsByFingerprint).length) return;
|
||||
secretScanningDAL.upsert(
|
||||
Object.keys(allFindingsByFingerprint).map((key) => ({
|
||||
installationId,
|
||||
email: allFindingsByFingerprint[key].Email,
|
||||
author: allFindingsByFingerprint[key].Author,
|
||||
date: allFindingsByFingerprint[key].Date,
|
||||
file: allFindingsByFingerprint[key].File,
|
||||
tags: allFindingsByFingerprint[key].Tags,
|
||||
commit: allFindingsByFingerprint[key].Commit,
|
||||
ruleID: allFindingsByFingerprint[key].RuleID,
|
||||
endLine: String(allFindingsByFingerprint[key].EndLine),
|
||||
entropy: String(allFindingsByFingerprint[key].Entropy),
|
||||
message: allFindingsByFingerprint[key].Message,
|
||||
endColumn: String(allFindingsByFingerprint[key].EndColumn),
|
||||
startLine: String(allFindingsByFingerprint[key].StartLine),
|
||||
startColumn: String(allFindingsByFingerprint[key].StartColumn),
|
||||
fingerPrintWithoutCommitId: allFindingsByFingerprint[key].FingerPrintWithoutCommitId,
|
||||
description: allFindingsByFingerprint[key].Description,
|
||||
symlinkFile: allFindingsByFingerprint[key].SymlinkFile,
|
||||
orgId: organizationId,
|
||||
pusherEmail: pusher.email,
|
||||
pusherName: pusher.name,
|
||||
repositoryFullName: repository.fullName,
|
||||
repositoryId: String(repository.id),
|
||||
fingerprint: allFindingsByFingerprint[key].Fingerprint
|
||||
})),
|
||||
tx
|
||||
);
|
||||
});
|
||||
|
||||
const adminEmails = await getOrgAdminEmails(organizationId);
|
||||
if (pusher?.email) {
|
||||
adminEmails.push(pusher.email);
|
||||
}
|
||||
if (Object.keys(allFindingsByFingerprint).length) {
|
||||
await smtpService.sendMail({
|
||||
template: SmtpTemplates.SecretLeakIncident,
|
||||
subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`,
|
||||
recipients: adminEmails,
|
||||
substitutions: {
|
||||
numberOfSecrets: Object.keys(allFindingsByFingerprint).length,
|
||||
pusher_email: pusher.email,
|
||||
pusher_name: pusher.name
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
telemetryService.sendPostHogEvents({
|
||||
event: PostHogEventTypes.SecretScannerPush,
|
||||
distinctId: repository.fullName,
|
||||
properties: {
|
||||
numberOfRisks: Object.keys(allFindingsByFingerprint).length
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
queueService.start(QueueName.SecretFullRepoScan, async (job) => {
|
||||
const appCfg = getConfig();
|
||||
const { organizationId, repository, installationId } = job.data;
|
||||
const octokit = new ProbotOctokit({
|
||||
auth: {
|
||||
appId: appCfg.SECRET_SCANNING_GIT_APP_ID,
|
||||
privateKey: appCfg.SECRET_SCANNING_PRIVATE_KEY,
|
||||
installationId
|
||||
}
|
||||
});
|
||||
|
||||
const findings = await scanFullRepoContentAndGetFindings(
|
||||
octokit,
|
||||
installationId,
|
||||
repository.fullName
|
||||
);
|
||||
await secretScanningDAL.transaction(async (tx) => {
|
||||
if (!findings.length) return;
|
||||
// eslint-disable-next-line
|
||||
await secretScanningDAL.upsert(
|
||||
findings.map((finding) => ({
|
||||
installationId,
|
||||
email: finding.Email,
|
||||
author: finding.Author,
|
||||
date: finding.Date,
|
||||
file: finding.File,
|
||||
tags: finding.Tags,
|
||||
commit: finding.Commit,
|
||||
ruleID: finding.RuleID,
|
||||
endLine: String(finding.EndLine),
|
||||
entropy: String(finding.Entropy),
|
||||
message: finding.Message,
|
||||
endColumn: String(finding.EndColumn),
|
||||
startLine: String(finding.StartLine),
|
||||
startColumn: String(finding.StartColumn),
|
||||
fingerPrintWithoutCommitId: finding.FingerPrintWithoutCommitId,
|
||||
description: finding.Description,
|
||||
symlinkFile: finding.SymlinkFile,
|
||||
orgId: organizationId,
|
||||
repositoryFullName: repository.fullName,
|
||||
repositoryId: String(repository.id),
|
||||
fingerprint: finding.Fingerprint
|
||||
})),
|
||||
tx
|
||||
);
|
||||
});
|
||||
|
||||
const adminEmails = await getOrgAdminEmails(organizationId);
|
||||
if (findings.length) {
|
||||
await smtpService.sendMail({
|
||||
template: SmtpTemplates.SecretLeakIncident,
|
||||
subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`,
|
||||
recipients: adminEmails,
|
||||
substitutions: {
|
||||
numberOfSecrets: findings.length
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
telemetryService.sendPostHogEvents({
|
||||
event: PostHogEventTypes.SecretScannerFull,
|
||||
distinctId: repository.fullName,
|
||||
properties: {
|
||||
numberOfRisks: findings.length
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
queueService.listen(QueueName.SecretPushEventScan, "failed", (job, err) => {
|
||||
logger.error("Failed to secret scan on push", job?.data, err);
|
||||
});
|
||||
|
||||
queueService.listen(QueueName.SecretFullRepoScan, "failed", (job, err) => {
|
||||
logger.error("Failed to do full repo secret scan", job?.data, err);
|
||||
});
|
||||
|
||||
return { startFullRepoScan, startPushEventScan };
|
||||
};
|
||||
Reference in New Issue
Block a user