mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-09-22 13:39:35 +00:00
feat(infisical-pg): completed secret scanner
This commit is contained in:
@@ -1,5 +1,9 @@
|
||||
FROM node:20-alpine
|
||||
|
||||
RUN apk add --no-cache bash curl && curl -1sLf \
|
||||
'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.alpine.sh' | bash \
|
||||
&& apk add infisical=0.8.1 && apk add --no-cache git
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json package.json
|
||||
|
||||
2098
backend-pg/package-lock.json
generated
2098
backend-pg/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -31,6 +31,7 @@
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"devDependencies": {
|
||||
"@octokit/webhooks-types": "^7.3.1",
|
||||
"@types/bcrypt": "^5.0.2",
|
||||
"@types/jmespath": "^0.15.2",
|
||||
"@types/jsonwebtoken": "^9.0.5",
|
||||
@@ -108,6 +109,8 @@
|
||||
"pg": "^8.11.3",
|
||||
"picomatch": "^3.0.1",
|
||||
"pino": "^8.16.2",
|
||||
"probot": "^12.3.3",
|
||||
"smee-client": "^2.0.0",
|
||||
"tweetnacl": "^1.0.3",
|
||||
"tweetnacl-util": "^0.15.1",
|
||||
"zod": "^3.22.4",
|
||||
|
||||
@@ -85,9 +85,7 @@ const main = async () => {
|
||||
.whereRaw("table_schema = current_schema()")
|
||||
.select<{ tableName: string }[]>("table_name as tableName")
|
||||
.orderBy("table_name")
|
||||
).filter(
|
||||
(el) => el.tableName !== "infisical_migrations_lock" && el.tableName !== "infisical_migrations"
|
||||
);
|
||||
).filter((el) => el.tableName.includes("migration"));
|
||||
|
||||
console.log("Select a table to generate schema");
|
||||
console.table(tables);
|
||||
|
||||
2
backend-pg/src/@types/fastify.d.ts
vendored
2
backend-pg/src/@types/fastify.d.ts
vendored
@@ -8,6 +8,7 @@ import { TSamlConfigServiceFactory } from "@app/ee/services/saml-config/saml-con
|
||||
import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
|
||||
import { TSecretApprovalRequestServiceFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-service";
|
||||
import { TSecretRotationServiceFactory } from "@app/ee/services/secret-rotation/secret-rotation-service";
|
||||
import { TSecretScanningServiceFactory } from "@app/ee/services/secret-scanning/secret-scanning-service";
|
||||
import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
|
||||
import { TApiKeyServiceFactory } from "@app/services/api-key/api-key-service";
|
||||
import { TAuthLoginFactory } from "@app/services/auth/auth-login-service";
|
||||
@@ -105,6 +106,7 @@ declare module "fastify" {
|
||||
snapshot: TSecretSnapshotServiceFactory;
|
||||
saml: TSamlConfigServiceFactory;
|
||||
auditLog: TAuditLogServiceFactory;
|
||||
secretScanning: TSecretScanningServiceFactory;
|
||||
};
|
||||
|
||||
// this is exclusive use for middlewares in which we need to inject data
|
||||
|
||||
20
backend-pg/src/@types/knex.d.ts
vendored
20
backend-pg/src/@types/knex.d.ts
vendored
@@ -16,6 +16,12 @@ import {
|
||||
TBackupPrivateKey,
|
||||
TBackupPrivateKeyInsert,
|
||||
TBackupPrivateKeyUpdate,
|
||||
TGitAppInstallSessions,
|
||||
TGitAppInstallSessionsInsert,
|
||||
TGitAppInstallSessionsUpdate,
|
||||
TGitAppOrg,
|
||||
TGitAppOrgInsert,
|
||||
TGitAppOrgUpdate,
|
||||
TIdentities,
|
||||
TIdentitiesInsert,
|
||||
TIdentitiesUpdate,
|
||||
@@ -113,6 +119,9 @@ import {
|
||||
TSecretRotationsInsert,
|
||||
TSecretRotationsUpdate,
|
||||
TSecrets,
|
||||
TSecretScanningGitRisks,
|
||||
TSecretScanningGitRisksInsert,
|
||||
TSecretScanningGitRisksUpdate,
|
||||
TSecretsInsert,
|
||||
TSecretSnapshotFolders,
|
||||
TSecretSnapshotFoldersInsert,
|
||||
@@ -373,6 +382,17 @@ declare module "knex/types/tables" {
|
||||
>;
|
||||
[TableName.OrgBot]: Knex.CompositeTableType<TOrgBots, TOrgBotsInsert, TOrgBotsUpdate>;
|
||||
[TableName.AuditLog]: Knex.CompositeTableType<TAuditLogs, TAuditLogsInsert, TAuditLogsUpdate>;
|
||||
[TableName.GitAppInstallSession]: Knex.CompositeTableType<
|
||||
TGitAppInstallSessions,
|
||||
TGitAppInstallSessionsInsert,
|
||||
TGitAppInstallSessionsUpdate
|
||||
>;
|
||||
[TableName.GitAppOrg]: Knex.CompositeTableType<TGitAppOrg, TGitAppOrgInsert, TGitAppOrgUpdate>;
|
||||
[TableName.SecretScanningGitRisk]: Knex.CompositeTableType<
|
||||
TSecretScanningGitRisks,
|
||||
TSecretScanningGitRisksInsert,
|
||||
TSecretScanningGitRisksUpdate
|
||||
>;
|
||||
// Junction tables
|
||||
[TableName.JnSecretTag]: Knex.CompositeTableType<
|
||||
TSecretTagJunction,
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import { Knex } from "knex";
|
||||
|
||||
import { TableName } from "../schemas";
|
||||
import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
|
||||
|
||||
export async function up(knex: Knex): Promise<void> {
|
||||
if (!(await knex.schema.hasTable(TableName.GitAppInstallSession))) {
|
||||
await knex.schema.createTable(TableName.GitAppInstallSession, (t) => {
|
||||
t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
|
||||
t.string("sessionId").notNullable().unique();
|
||||
t.uuid("userId");
|
||||
// one to one relationship
|
||||
t.uuid("orgId").notNullable().unique();
|
||||
t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
|
||||
t.timestamps(true, true, true);
|
||||
});
|
||||
}
|
||||
createOnUpdateTrigger(knex, TableName.GitAppInstallSession);
|
||||
|
||||
if (!(await knex.schema.hasTable(TableName.GitAppOrg))) {
|
||||
await knex.schema.createTable(TableName.GitAppOrg, (t) => {
|
||||
t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
|
||||
t.string("installationId").notNullable().unique();
|
||||
t.uuid("userId").notNullable();
|
||||
// one to one relationship
|
||||
t.uuid("orgId").notNullable().unique();
|
||||
t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
|
||||
t.timestamps(true, true, true);
|
||||
});
|
||||
}
|
||||
createOnUpdateTrigger(knex, TableName.GitAppOrg);
|
||||
|
||||
if (!(await knex.schema.hasTable(TableName.SecretScanningGitRisk))) {
|
||||
await knex.schema.createTable(TableName.SecretScanningGitRisk, (t) => {
|
||||
t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
|
||||
t.string("description");
|
||||
t.string("startLine");
|
||||
t.string("endLine");
|
||||
t.string("startColumn");
|
||||
t.string("endColumn");
|
||||
t.string("file");
|
||||
t.string("symlinkFile");
|
||||
t.string("commit");
|
||||
t.string("entropy");
|
||||
t.string("author");
|
||||
t.string("email");
|
||||
t.string("date");
|
||||
t.text("message");
|
||||
t.specificType("tags", "text[]");
|
||||
t.string("ruleID");
|
||||
t.string("fingerprint").unique();
|
||||
t.string("fingerPrintWithoutCommitId");
|
||||
t.boolean("isFalsePositive").defaultTo(false);
|
||||
t.boolean("isResolved").defaultTo(false);
|
||||
t.string("riskOwner");
|
||||
t.string("installationId").notNullable();
|
||||
t.string("repositoryId");
|
||||
t.string("repositoryLink");
|
||||
t.string("repositoryFullName");
|
||||
t.string("pusherName");
|
||||
t.string("pusherEmail");
|
||||
t.string("status");
|
||||
// one to one relationship
|
||||
t.uuid("orgId").notNullable();
|
||||
t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
|
||||
t.timestamps(true, true, true);
|
||||
});
|
||||
}
|
||||
createOnUpdateTrigger(knex, TableName.SecretScanningGitRisk);
|
||||
}
|
||||
|
||||
export async function down(knex: Knex): Promise<void> {
|
||||
await knex.schema.dropTableIfExists(TableName.SecretScanningGitRisk);
|
||||
await knex.schema.dropTableIfExists(TableName.GitAppOrg);
|
||||
await knex.schema.dropTableIfExists(TableName.GitAppInstallSession);
|
||||
await dropOnUpdateTrigger(knex, TableName.SecretScanningGitRisk);
|
||||
await dropOnUpdateTrigger(knex, TableName.GitAppOrg);
|
||||
await dropOnUpdateTrigger(knex, TableName.GitAppInstallSession);
|
||||
}
|
||||
21
backend-pg/src/db/schemas/git-app-install-sessions.ts
Normal file
21
backend-pg/src/db/schemas/git-app-install-sessions.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
// Code generated by automation script, DO NOT EDIT.
|
||||
// Automated by pulling database and generating zod schema
|
||||
// To update. Just run npm run generate:schema
|
||||
// Written by akhilmhdh.
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
import { TImmutableDBKeys } from "./models";
|
||||
|
||||
export const GitAppInstallSessionsSchema = z.object({
|
||||
id: z.string().uuid(),
|
||||
sessionId: z.string(),
|
||||
userId: z.string().uuid().nullable().optional(),
|
||||
orgId: z.string().uuid(),
|
||||
createdAt: z.date(),
|
||||
updatedAt: z.date(),
|
||||
});
|
||||
|
||||
export type TGitAppInstallSessions = z.infer<typeof GitAppInstallSessionsSchema>;
|
||||
export type TGitAppInstallSessionsInsert = Omit<TGitAppInstallSessions, TImmutableDBKeys>;
|
||||
export type TGitAppInstallSessionsUpdate = Partial<Omit<TGitAppInstallSessions, TImmutableDBKeys>>;
|
||||
21
backend-pg/src/db/schemas/git-app-org.ts
Normal file
21
backend-pg/src/db/schemas/git-app-org.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
// Code generated by automation script, DO NOT EDIT.
|
||||
// Automated by pulling database and generating zod schema
|
||||
// To update. Just run npm run generate:schema
|
||||
// Written by akhilmhdh.
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
import { TImmutableDBKeys } from "./models";
|
||||
|
||||
export const GitAppOrgSchema = z.object({
|
||||
id: z.string().uuid(),
|
||||
installationId: z.string(),
|
||||
userId: z.string().uuid(),
|
||||
orgId: z.string().uuid(),
|
||||
createdAt: z.date(),
|
||||
updatedAt: z.date(),
|
||||
});
|
||||
|
||||
export type TGitAppOrg = z.infer<typeof GitAppOrgSchema>;
|
||||
export type TGitAppOrgInsert = Omit<TGitAppOrg, TImmutableDBKeys>;
|
||||
export type TGitAppOrgUpdate = Partial<Omit<TGitAppOrg, TImmutableDBKeys>>;
|
||||
@@ -3,6 +3,8 @@ export * from "./audit-logs";
|
||||
export * from "./auth-token-sessions";
|
||||
export * from "./auth-tokens";
|
||||
export * from "./backup-private-key";
|
||||
export * from "./git-app-install-sessions";
|
||||
export * from "./git-app-org";
|
||||
export * from "./identities";
|
||||
export * from "./identity-access-tokens";
|
||||
export * from "./identity-org-memberships";
|
||||
@@ -36,6 +38,7 @@ export * from "./secret-folders";
|
||||
export * from "./secret-imports";
|
||||
export * from "./secret-rotation-outputs";
|
||||
export * from "./secret-rotations";
|
||||
export * from "./secret-scanning-git-risks";
|
||||
export * from "./secret-snapshot-folders";
|
||||
export * from "./secret-snapshot-secrets";
|
||||
export * from "./secret-snapshots";
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
// Code generated by automation script, DO NOT EDIT.
|
||||
// Automated by pulling database and generating zod schema
|
||||
// To update. Just run npm run generate:schema
|
||||
// Written by akhilmhdh.
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
import { TImmutableDBKeys } from "./models";
|
||||
|
||||
export const KnexMigrationsLockSchema = z.object({
|
||||
index: z.number(),
|
||||
is_locked: z.number().nullable().optional(),
|
||||
});
|
||||
|
||||
export type TKnexMigrationsLock = z.infer<typeof KnexMigrationsLockSchema>;
|
||||
export type TKnexMigrationsLockInsert = Omit<TKnexMigrationsLock, TImmutableDBKeys>;
|
||||
export type TKnexMigrationsLockUpdate = Partial<Omit<TKnexMigrationsLock, TImmutableDBKeys>>;
|
||||
@@ -1,19 +0,0 @@
|
||||
// Code generated by automation script, DO NOT EDIT.
|
||||
// Automated by pulling database and generating zod schema
|
||||
// To update. Just run npm run generate:schema
|
||||
// Written by akhilmhdh.
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
import { TImmutableDBKeys } from "./models";
|
||||
|
||||
export const KnexMigrationsSchema = z.object({
|
||||
id: z.number(),
|
||||
name: z.string().nullable().optional(),
|
||||
batch: z.number().nullable().optional(),
|
||||
migration_time: z.date().nullable().optional(),
|
||||
});
|
||||
|
||||
export type TKnexMigrations = z.infer<typeof KnexMigrationsSchema>;
|
||||
export type TKnexMigrationsInsert = Omit<TKnexMigrations, TImmutableDBKeys>;
|
||||
export type TKnexMigrationsUpdate = Partial<Omit<TKnexMigrations, TImmutableDBKeys>>;
|
||||
@@ -50,6 +50,9 @@ export enum TableName {
|
||||
SecretRotationOutput = "secret_rotation_outputs",
|
||||
SamlConfig = "saml_configs",
|
||||
AuditLog = "audit_logs",
|
||||
GitAppInstallSession = "git_app_install_sessions",
|
||||
GitAppOrg = "git_app_org",
|
||||
SecretScanningGitRisk = "secret_scanning_git_risks",
|
||||
// junction tables
|
||||
JnSecretTag = "secret_tag_junction",
|
||||
JnSecretVersionTag = "secret_version_tag_junction"
|
||||
|
||||
46
backend-pg/src/db/schemas/secret-scanning-git-risks.ts
Normal file
46
backend-pg/src/db/schemas/secret-scanning-git-risks.ts
Normal file
@@ -0,0 +1,46 @@
|
||||
// Code generated by automation script, DO NOT EDIT.
|
||||
// Automated by pulling database and generating zod schema
|
||||
// To update. Just run npm run generate:schema
|
||||
// Written by akhilmhdh.
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
import { TImmutableDBKeys } from "./models";
|
||||
|
||||
export const SecretScanningGitRisksSchema = z.object({
|
||||
id: z.string().uuid(),
|
||||
description: z.string().nullable().optional(),
|
||||
startLine: z.string().nullable().optional(),
|
||||
endLine: z.string().nullable().optional(),
|
||||
startColumn: z.string().nullable().optional(),
|
||||
endColumn: z.string().nullable().optional(),
|
||||
file: z.string().nullable().optional(),
|
||||
symlinkFile: z.string().nullable().optional(),
|
||||
commit: z.string().nullable().optional(),
|
||||
entropy: z.string().nullable().optional(),
|
||||
author: z.string().nullable().optional(),
|
||||
email: z.string().nullable().optional(),
|
||||
date: z.string().nullable().optional(),
|
||||
message: z.string().nullable().optional(),
|
||||
tags: z.string().array().nullable().optional(),
|
||||
ruleID: z.string().nullable().optional(),
|
||||
fingerprint: z.string().nullable().optional(),
|
||||
fingerPrintWithoutCommitId: z.string().nullable().optional(),
|
||||
isFalsePositive: z.boolean().default(false).nullable().optional(),
|
||||
isResolved: z.boolean().default(false).nullable().optional(),
|
||||
riskOwner: z.string().nullable().optional(),
|
||||
installationId: z.string(),
|
||||
repositoryId: z.string().nullable().optional(),
|
||||
repositoryLink: z.string().nullable().optional(),
|
||||
repositoryFullName: z.string().nullable().optional(),
|
||||
pusherName: z.string().nullable().optional(),
|
||||
pusherEmail: z.string().nullable().optional(),
|
||||
status: z.string().nullable().optional(),
|
||||
orgId: z.string().uuid(),
|
||||
createdAt: z.date(),
|
||||
updatedAt: z.date(),
|
||||
});
|
||||
|
||||
export type TSecretScanningGitRisks = z.infer<typeof SecretScanningGitRisksSchema>;
|
||||
export type TSecretScanningGitRisksInsert = Omit<TSecretScanningGitRisks, TImmutableDBKeys>;
|
||||
export type TSecretScanningGitRisksUpdate = Partial<Omit<TSecretScanningGitRisks, TImmutableDBKeys>>;
|
||||
@@ -6,6 +6,7 @@ import { registerSecretApprovalPolicyRouter } from "./secret-approval-policy-rou
|
||||
import { registerSecretApprovalRequestRouter } from "./secret-approval-request-router";
|
||||
import { registerSecretRotationProviderRouter } from "./secret-rotation-provider-router";
|
||||
import { registerSecretRotationRouter } from "./secret-rotation-router";
|
||||
import { registerSecretScanningRouter } from "./secret-scanning-router";
|
||||
import { registerSnapshotRouter } from "./snapshot-router";
|
||||
|
||||
export const registerV1EERoutes = async (server: FastifyZodProvider) => {
|
||||
@@ -27,5 +28,6 @@ export const registerV1EERoutes = async (server: FastifyZodProvider) => {
|
||||
prefix: "/secret-rotation-providers"
|
||||
});
|
||||
await server.register(registerSamlRouter, { prefix: "/sso" });
|
||||
await server.register(registerSecretScanningRouter, { prefix: "/secret-scanning" });
|
||||
await server.register(registerSecretRotationRouter, { prefix: "/secret-rotations" });
|
||||
};
|
||||
|
||||
117
backend-pg/src/ee/routes/v1/secret-scanning-router.ts
Normal file
117
backend-pg/src/ee/routes/v1/secret-scanning-router.ts
Normal file
@@ -0,0 +1,117 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { GitAppOrgSchema, SecretScanningGitRisksSchema } from "@app/db/schemas";
|
||||
import { SecretScanningRiskStatus } from "@app/ee/services/secret-scanning/secret-scanning-types";
|
||||
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
|
||||
import { AuthMode } from "@app/services/auth/auth-type";
|
||||
|
||||
export const registerSecretScanningRouter = async (server: FastifyZodProvider) => {
|
||||
server.route({
|
||||
url: "/create-installation-session/organization",
|
||||
method: "POST",
|
||||
schema: {
|
||||
body: z.object({ organizationId: z.string().trim() }),
|
||||
response: {
|
||||
200: z.object({
|
||||
sessionId: z.string()
|
||||
})
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT]),
|
||||
handler: async (req) => {
|
||||
const session = await server.services.secretScanning.createInstallationSession({
|
||||
actor: req.permission.type,
|
||||
actorId: req.permission.id,
|
||||
orgId: req.body.organizationId
|
||||
});
|
||||
return session;
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
url: "/link-installation",
|
||||
method: "POST",
|
||||
schema: {
|
||||
body: z.object({
|
||||
installationId: z.string(),
|
||||
sessionId: z.string().trim()
|
||||
}),
|
||||
response: {
|
||||
200: GitAppOrgSchema
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT]),
|
||||
handler: async (req) => {
|
||||
const { installatedApp } = await server.services.secretScanning.linkInstallationToOrg({
|
||||
actor: req.permission.type,
|
||||
actorId: req.permission.id,
|
||||
...req.body
|
||||
});
|
||||
return installatedApp;
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
url: "/installation-status/organization/:organizationId",
|
||||
method: "GET",
|
||||
schema: {
|
||||
params: z.object({ organizationId: z.string().trim() }),
|
||||
response: {
|
||||
200: z.object({ appInstallationCompleted: z.boolean() })
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT]),
|
||||
handler: async (req) => {
|
||||
const appInstallationCompleted =
|
||||
await server.services.secretScanning.getOrgInstallationStatus({
|
||||
actor: req.permission.type,
|
||||
actorId: req.permission.id,
|
||||
orgId: req.params.organizationId
|
||||
});
|
||||
return { appInstallationCompleted };
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
url: "/organization/:organizationId/risks",
|
||||
method: "GET",
|
||||
schema: {
|
||||
params: z.object({ organizationId: z.string().trim() }),
|
||||
response: {
|
||||
200: z.object({ risks: SecretScanningGitRisksSchema.array() })
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT]),
|
||||
handler: async (req) => {
|
||||
const { risks } = await server.services.secretScanning.getRisksByOrg({
|
||||
actor: req.permission.type,
|
||||
actorId: req.permission.id,
|
||||
orgId: req.params.organizationId
|
||||
});
|
||||
return { risks };
|
||||
}
|
||||
});
|
||||
|
||||
server.route({
|
||||
url: "/organization/:organizationId/risks/:riskId/status",
|
||||
method: "POST",
|
||||
schema: {
|
||||
params: z.object({ organizationId: z.string().trim(), riskId: z.string().trim() }),
|
||||
body: z.object({ status: z.nativeEnum(SecretScanningRiskStatus) }),
|
||||
response: {
|
||||
200: SecretScanningGitRisksSchema
|
||||
}
|
||||
},
|
||||
onRequest: verifyAuth([AuthMode.JWT]),
|
||||
handler: async (req) => {
|
||||
const { risk } = await server.services.secretScanning.updateRiskStatus({
|
||||
actor: req.permission.type,
|
||||
actorId: req.permission.id,
|
||||
orgId: req.params.organizationId,
|
||||
riskId: req.params.riskId,
|
||||
...req.body
|
||||
});
|
||||
return risk;
|
||||
}
|
||||
});
|
||||
};
|
||||
27
backend-pg/src/ee/services/secret-scanning/git-app-dal.ts
Normal file
27
backend-pg/src/ee/services/secret-scanning/git-app-dal.ts
Normal file
@@ -0,0 +1,27 @@
|
||||
import { Knex } from "knex";
|
||||
|
||||
import { TDbClient } from "@app/db";
|
||||
import { TableName,TGitAppOrgInsert } from "@app/db/schemas";
|
||||
import { DatabaseError } from "@app/lib/errors";
|
||||
import { ormify } from "@app/lib/knex";
|
||||
|
||||
export type TGitAppDalFactory = ReturnType<typeof gitAppDalFactory>;
|
||||
|
||||
export const gitAppDalFactory = (db: TDbClient) => {
|
||||
const gitAppOrm = ormify(db, TableName.GitAppOrg);
|
||||
|
||||
const upsert = async (data: TGitAppOrgInsert, tx?: Knex) => {
|
||||
try {
|
||||
const [doc] = await (tx || db)(TableName.GitAppOrg)
|
||||
.insert(data)
|
||||
.onConflict("orgId")
|
||||
.merge()
|
||||
.returning("*");
|
||||
return doc;
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "UpsertGitAppOrm" });
|
||||
}
|
||||
};
|
||||
|
||||
return { ...gitAppOrm, upsert };
|
||||
};
|
||||
@@ -0,0 +1,27 @@
|
||||
import { Knex } from "knex";
|
||||
|
||||
import { TDbClient } from "@app/db";
|
||||
import { TableName,TGitAppInstallSessionsInsert } from "@app/db/schemas";
|
||||
import { DatabaseError } from "@app/lib/errors";
|
||||
import { ormify } from "@app/lib/knex";
|
||||
|
||||
export type TGitAppInstallSessionDalFactory = ReturnType<typeof gitAppInstallSessionDalFactory>;
|
||||
|
||||
export const gitAppInstallSessionDalFactory = (db: TDbClient) => {
|
||||
const gitAppInstallSessionOrm = ormify(db, TableName.GitAppInstallSession);
|
||||
|
||||
const upsert = async (data: TGitAppInstallSessionsInsert, tx?: Knex) => {
|
||||
try {
|
||||
const [doc] = await (tx || db)(TableName.GitAppInstallSession)
|
||||
.insert(data)
|
||||
.onConflict("orgId")
|
||||
.merge()
|
||||
.returning("*");
|
||||
return doc;
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "UpsertGitAppOrm" });
|
||||
}
|
||||
};
|
||||
|
||||
return { ...gitAppInstallSessionOrm, upsert };
|
||||
};
|
||||
@@ -0,0 +1,26 @@
|
||||
import { Knex } from "knex";
|
||||
|
||||
import { TDbClient } from "@app/db";
|
||||
import { TableName,TSecretScanningGitRisksInsert } from "@app/db/schemas";
|
||||
import { DatabaseError } from "@app/lib/errors";
|
||||
import { ormify } from "@app/lib/knex";
|
||||
|
||||
export type TSecretScanningDalFactory = ReturnType<typeof secretScanningDalFactory>;
|
||||
|
||||
export const secretScanningDalFactory = (db: TDbClient) => {
|
||||
const gitRiskOrm = ormify(db, TableName.SecretScanningGitRisk);
|
||||
|
||||
const upsert = async (data: TSecretScanningGitRisksInsert[], tx?: Knex) => {
|
||||
try {
|
||||
const docs = await (tx || db)(TableName.SecretScanningGitRisk)
|
||||
.insert(data)
|
||||
.onConflict("fingerprint")
|
||||
.merge();
|
||||
return docs;
|
||||
} catch (error) {
|
||||
throw new DatabaseError({ error, name: "GitRiskUpsert" });
|
||||
}
|
||||
};
|
||||
|
||||
return { ...gitRiskOrm, upsert };
|
||||
};
|
||||
@@ -0,0 +1 @@
|
||||
export * from "./secret-scanning-queue";
|
||||
@@ -0,0 +1,151 @@
|
||||
import { exec } from "child_process";
|
||||
import { mkdir, readFile, rm, writeFile } from "fs";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
|
||||
import { SecretMatch } from "./secret-scanning-queue-types";
|
||||
|
||||
export function createTempFolder(): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const tempDir = tmpdir();
|
||||
const tempFolderName = Math.random().toString(36).substring(2);
|
||||
const tempFolderPath = join(tempDir, tempFolderName);
|
||||
|
||||
mkdir(tempFolderPath, (err: any) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve(tempFolderPath);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function writeTextToFile(filePath: string, content: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
writeFile(filePath, content, (err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function cloneRepo(
|
||||
installationAcccessToken: string,
|
||||
repositoryFullName: string,
|
||||
repoPath: string
|
||||
): Promise<void> {
|
||||
const cloneUrl = `https://x-access-token:${installationAcccessToken}@github.com/${repositoryFullName}.git`;
|
||||
const command = `git clone ${cloneUrl} ${repoPath} --bare`;
|
||||
return new Promise((resolve, reject) => {
|
||||
exec(command, (error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function runInfisicalScanOnRepo(repoPath: string, outputPath: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const command = `cd ${repoPath} && infisical scan --exit-code=77 -r "${outputPath}"`;
|
||||
exec(command, (error) => {
|
||||
if (error && error.code !== 77) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function runInfisicalScan(inputPath: string, outputPath: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const command = `cat "${inputPath}" | infisical scan --exit-code=77 --pipe -r "${outputPath}"`;
|
||||
exec(command, (error) => {
|
||||
if (error && error.code !== 77) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function readFindingsFile(filePath: string): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
readFile(filePath, "utf8", (err, data) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve(data);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function deleteTempFolder(folderPath: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
rm(folderPath, { recursive: true }, (err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function convertKeysToLowercase<T>(obj: T): T {
|
||||
const convertedObj = {} as T;
|
||||
|
||||
for (const key in obj) {
|
||||
if (Object.prototype.hasOwnProperty.call(obj, key)) {
|
||||
const lowercaseKey = key.charAt(0).toLowerCase() + key.slice(1);
|
||||
convertedObj[lowercaseKey as keyof T] = obj[key];
|
||||
}
|
||||
}
|
||||
|
||||
return convertedObj;
|
||||
}
|
||||
|
||||
export async function scanFullRepoContentAndGetFindings(
|
||||
octokit: any,
|
||||
installationId: string,
|
||||
repositoryFullName: string
|
||||
): Promise<SecretMatch[]> {
|
||||
const tempFolder = await createTempFolder();
|
||||
const findingsPath = join(tempFolder, "findings.json");
|
||||
const repoPath = join(tempFolder, "repo.git");
|
||||
try {
|
||||
const {
|
||||
data: { token }
|
||||
} = await octokit.apps.createInstallationAccessToken({ installation_id: installationId });
|
||||
await cloneRepo(token, repositoryFullName, repoPath);
|
||||
await runInfisicalScanOnRepo(repoPath, findingsPath);
|
||||
const findingsData = await readFindingsFile(findingsPath);
|
||||
return JSON.parse(findingsData);
|
||||
} finally {
|
||||
await deleteTempFolder(tempFolder);
|
||||
}
|
||||
}
|
||||
|
||||
export async function scanContentAndGetFindings(textContent: string): Promise<SecretMatch[]> {
|
||||
const tempFolder = await createTempFolder();
|
||||
const filePath = join(tempFolder, "content.txt");
|
||||
const findingsPath = join(tempFolder, "findings.json");
|
||||
|
||||
try {
|
||||
await writeTextToFile(filePath, textContent);
|
||||
await runInfisicalScan(filePath, findingsPath);
|
||||
const findingsData = await readFindingsFile(findingsPath);
|
||||
return JSON.parse(findingsData);
|
||||
} finally {
|
||||
await deleteTempFolder(tempFolder);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import { Commit } from "@octokit/webhooks-types";
|
||||
|
||||
export type SecretMatch = {
|
||||
Description: string;
|
||||
StartLine: number;
|
||||
EndLine: number;
|
||||
StartColumn: number;
|
||||
EndColumn: number;
|
||||
Match: string;
|
||||
Secret: string;
|
||||
File: string;
|
||||
SymlinkFile: string;
|
||||
Commit: string;
|
||||
Entropy: number;
|
||||
Author: string;
|
||||
Email: string;
|
||||
Date: string;
|
||||
Message: string;
|
||||
Tags: string[];
|
||||
RuleID: string;
|
||||
Fingerprint: string;
|
||||
FingerPrintWithoutCommitId: string;
|
||||
};
|
||||
|
||||
export type TScanPushEventPayload = {
|
||||
organizationId: string;
|
||||
commits: Commit[];
|
||||
pusher: {
|
||||
name: string;
|
||||
email: string | null;
|
||||
};
|
||||
repository: {
|
||||
id: number;
|
||||
fullName: string;
|
||||
};
|
||||
installationId: string;
|
||||
};
|
||||
|
||||
export type TScanFullRepoEventPayload = {
|
||||
organizationId: string;
|
||||
installationId: string;
|
||||
repository: {
|
||||
id: number;
|
||||
fullName: string;
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,241 @@
|
||||
import { ProbotOctokit } from "probot";
|
||||
|
||||
import { OrgMembershipRole } from "@app/db/schemas";
|
||||
import { getConfig } from "@app/lib/config/env";
|
||||
import { logger } from "@app/lib/logger";
|
||||
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
|
||||
import { TOrgDalFactory } from "@app/services/org/org-dal";
|
||||
import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
|
||||
import { TUserDalFactory } from "@app/services/user/user-dal";
|
||||
|
||||
import { TSecretScanningDalFactory } from "../secret-scanning-dal";
|
||||
import {
|
||||
scanContentAndGetFindings,
|
||||
scanFullRepoContentAndGetFindings} from "./secret-scanning-fns";
|
||||
import {
|
||||
SecretMatch,
|
||||
TScanFullRepoEventPayload,
|
||||
TScanPushEventPayload
|
||||
} from "./secret-scanning-queue-types";
|
||||
|
||||
type TSecretScanningQueueFactoryDep = {
|
||||
queueService: TQueueServiceFactory;
|
||||
secretScanningDal: TSecretScanningDalFactory;
|
||||
smtpService: Pick<TSmtpService, "sendMail">;
|
||||
orgMembershipDal: Pick<TOrgDalFactory, "findMembership">;
|
||||
userDal: Pick<TUserDalFactory, "find">;
|
||||
};
|
||||
|
||||
export type TSecretScanningQueueFactory = ReturnType<typeof secretScanningQueueFactory>;
|
||||
|
||||
export const secretScanningQueueFactory = ({
|
||||
queueService,
|
||||
secretScanningDal,
|
||||
smtpService,
|
||||
orgMembershipDal: orgMemberDal,
|
||||
userDal
|
||||
}: TSecretScanningQueueFactoryDep) => {
|
||||
const startFullRepoScan = async (payload: TScanFullRepoEventPayload) => {
|
||||
await queueService.queue(QueueName.SecretFullRepoScan, QueueJobs.SecretScan, payload, {
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: "exponential",
|
||||
delay: 5000
|
||||
},
|
||||
removeOnComplete: true,
|
||||
removeOnFail: {
|
||||
count: 20 // keep the most recent 20 jobs
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const startPushEventScan = async (payload: TScanPushEventPayload) => {
|
||||
await queueService.queue(QueueName.SecretPushEventScan, QueueJobs.SecretScan, payload, {
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: "exponential",
|
||||
delay: 5000
|
||||
},
|
||||
removeOnComplete: true,
|
||||
removeOnFail: {
|
||||
count: 20 // keep the most recent 20 jobs
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const getOrgAdminEmails = async (organizationId: string) => {
|
||||
// get emails of admins
|
||||
const adminsOfWork = await orgMemberDal.findMembership({
|
||||
orgId: organizationId,
|
||||
role: OrgMembershipRole.Admin
|
||||
});
|
||||
const userEmails = await userDal.find({
|
||||
$in: {
|
||||
id: adminsOfWork.map(({ userId }) => userId).filter(Boolean) as string[]
|
||||
}
|
||||
});
|
||||
return userEmails.map((userObject) => userObject.email);
|
||||
};
|
||||
|
||||
queueService.start(QueueName.SecretPushEventScan, async (job) => {
|
||||
const appCfg = getConfig();
|
||||
const { organizationId, commits, pusher, repository, installationId } = job.data;
|
||||
const [owner, repo] = repository.fullName.split("/");
|
||||
const octokit = new ProbotOctokit({
|
||||
auth: {
|
||||
appId: appCfg.SECRET_SCANNING_GIT_APP_ID,
|
||||
privateKey: appCfg.SECRET_SCANNING_PRIVATE_KEY,
|
||||
installationId
|
||||
}
|
||||
});
|
||||
const allFindingsByFingerprint: { [key: string]: SecretMatch } = {};
|
||||
|
||||
for (const commit of commits) {
|
||||
for (const filepath of [...commit.added, ...commit.modified]) {
|
||||
// eslint-disable-next-line
|
||||
const fileContentsResponse = await octokit.repos.getContent({
|
||||
owner,
|
||||
repo,
|
||||
path: filepath
|
||||
});
|
||||
|
||||
const { data }: any = fileContentsResponse;
|
||||
const fileContent = Buffer.from(data.content, "base64").toString();
|
||||
|
||||
// eslint-disable-next-line
|
||||
const findings = await scanContentAndGetFindings(`\n${fileContent}`); // extra line to count lines correctly
|
||||
|
||||
for (const finding of findings) {
|
||||
const fingerPrintWithCommitId = `${commit.id}:${filepath}:${finding.RuleID}:${finding.StartLine}`;
|
||||
const fingerPrintWithoutCommitId = `${filepath}:${finding.RuleID}:${finding.StartLine}`;
|
||||
finding.Fingerprint = fingerPrintWithCommitId;
|
||||
finding.FingerPrintWithoutCommitId = fingerPrintWithoutCommitId;
|
||||
finding.Commit = commit.id;
|
||||
finding.File = filepath;
|
||||
finding.Author = commit.author.name;
|
||||
finding.Email = commit?.author?.email ? commit?.author?.email : "";
|
||||
|
||||
allFindingsByFingerprint[fingerPrintWithCommitId] = finding;
|
||||
}
|
||||
}
|
||||
}
|
||||
await secretScanningDal.transaction(async (tx) => {
|
||||
if (!Object.keys(allFindingsByFingerprint).length) return;
|
||||
secretScanningDal.upsert(
|
||||
Object.keys(allFindingsByFingerprint).map((key) => ({
|
||||
installationId,
|
||||
email: allFindingsByFingerprint[key].Email,
|
||||
author: allFindingsByFingerprint[key].Author,
|
||||
date: allFindingsByFingerprint[key].Date,
|
||||
file: allFindingsByFingerprint[key].File,
|
||||
tags: allFindingsByFingerprint[key].Tags,
|
||||
commit: allFindingsByFingerprint[key].Commit,
|
||||
ruleID: allFindingsByFingerprint[key].RuleID,
|
||||
endLine: String(allFindingsByFingerprint[key].EndLine),
|
||||
entropy: String(allFindingsByFingerprint[key].Entropy),
|
||||
message: allFindingsByFingerprint[key].Message,
|
||||
endColumn: String(allFindingsByFingerprint[key].EndColumn),
|
||||
startLine: String(allFindingsByFingerprint[key].StartLine),
|
||||
startColumn: String(allFindingsByFingerprint[key].StartColumn),
|
||||
fingerPrintWithoutCommitId: allFindingsByFingerprint[key].FingerPrintWithoutCommitId,
|
||||
description: allFindingsByFingerprint[key].Description,
|
||||
symlinkFile: allFindingsByFingerprint[key].SymlinkFile,
|
||||
orgId: organizationId,
|
||||
pusherEmail: pusher.email,
|
||||
pusherName: pusher.name,
|
||||
repositoryFullName: repository.fullName,
|
||||
repositoryId: String(repository.id),
|
||||
fingerprint: allFindingsByFingerprint[key].Fingerprint
|
||||
})),
|
||||
tx
|
||||
);
|
||||
});
|
||||
|
||||
const adminEmails = await getOrgAdminEmails(organizationId);
|
||||
if (pusher?.email) {
|
||||
adminEmails.push(pusher.email);
|
||||
}
|
||||
if (Object.keys(allFindingsByFingerprint).length) {
|
||||
await smtpService.sendMail({
|
||||
template: SmtpTemplates.SecretLeakIncident,
|
||||
subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`,
|
||||
recipients: adminEmails,
|
||||
substitutions: {
|
||||
numberOfSecrets: Object.keys(allFindingsByFingerprint).length,
|
||||
pusher_email: pusher.email,
|
||||
pusher_name: pusher.name
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
queueService.start(QueueName.SecretFullRepoScan, async (job) => {
|
||||
const appCfg = getConfig();
|
||||
const { organizationId, repository, installationId } = job.data;
|
||||
const octokit = new ProbotOctokit({
|
||||
auth: {
|
||||
appId: appCfg.SECRET_SCANNING_GIT_APP_ID,
|
||||
privateKey: appCfg.SECRET_SCANNING_PRIVATE_KEY,
|
||||
installationId
|
||||
}
|
||||
});
|
||||
|
||||
const findings = await scanFullRepoContentAndGetFindings(
|
||||
octokit,
|
||||
installationId,
|
||||
repository.fullName
|
||||
);
|
||||
await secretScanningDal.transaction(async (tx) => {
|
||||
if (!findings.length) return;
|
||||
// eslint-disable-next-line
|
||||
await secretScanningDal.upsert(
|
||||
findings.map((finding) => ({
|
||||
installationId,
|
||||
email: finding.Email,
|
||||
author: finding.Author,
|
||||
date: finding.Date,
|
||||
file: finding.File,
|
||||
tags: finding.Tags,
|
||||
commit: finding.Commit,
|
||||
ruleID: finding.RuleID,
|
||||
endLine: String(finding.EndLine),
|
||||
entropy: String(finding.Entropy),
|
||||
message: finding.Message,
|
||||
endColumn: String(finding.EndColumn),
|
||||
startLine: String(finding.StartLine),
|
||||
startColumn: String(finding.StartColumn),
|
||||
fingerPrintWithoutCommitId: finding.FingerPrintWithoutCommitId,
|
||||
description: finding.Description,
|
||||
symlinkFile: finding.SymlinkFile,
|
||||
orgId: organizationId,
|
||||
repositoryFullName: repository.fullName,
|
||||
repositoryId: String(repository.id),
|
||||
fingerprint: finding.Fingerprint
|
||||
})),
|
||||
tx
|
||||
);
|
||||
});
|
||||
|
||||
const adminEmails = await getOrgAdminEmails(organizationId);
|
||||
if (findings.length) {
|
||||
await smtpService.sendMail({
|
||||
template: SmtpTemplates.SecretLeakIncident,
|
||||
subjectLine: `Incident alert: leaked secrets found in Github repository ${repository.fullName}`,
|
||||
recipients: adminEmails,
|
||||
substitutions: {
|
||||
numberOfSecrets: findings.length
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
queueService.listen(QueueName.SecretPushEventScan, "failed", (job, err) => {
|
||||
logger.error("Failed to secret scan on push", job?.data, err);
|
||||
});
|
||||
|
||||
queueService.listen(QueueName.SecretFullRepoScan, "failed", (job, err) => {
|
||||
logger.error("Failed to do full repo secret scan", job?.data, err);
|
||||
});
|
||||
|
||||
return { startFullRepoScan, startPushEventScan };
|
||||
};
|
||||
@@ -0,0 +1,191 @@
|
||||
import crypto from "node:crypto";
|
||||
|
||||
import { ForbiddenError } from "@casl/ability";
|
||||
import { PushEvent } from "@octokit/webhooks-types";
|
||||
import { ProbotOctokit } from "probot";
|
||||
|
||||
import {
|
||||
OrgPermissionActions,
|
||||
OrgPermissionSubjects
|
||||
} from "@app/ee/services/permission/org-permission";
|
||||
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
|
||||
import { getConfig } from "@app/lib/config/env";
|
||||
import { UnauthorizedError } from "@app/lib/errors";
|
||||
|
||||
import { TGitAppDalFactory } from "./git-app-dal";
|
||||
import { TGitAppInstallSessionDalFactory } from "./git-app-install-session-dal";
|
||||
import { TSecretScanningDalFactory } from "./secret-scanning-dal";
|
||||
import { TSecretScanningQueueFactory } from "./secret-scanning-queue";
|
||||
import {
|
||||
SecretScanningRiskStatus,
|
||||
TGetOrgInstallStatusDTO,
|
||||
TGetOrgRisksDTO,
|
||||
TInstallAppSessionDTO,
|
||||
TLinkInstallSessionDTO,
|
||||
TUpdateRiskStatusDTO
|
||||
} from "./secret-scanning-types";
|
||||
|
||||
type TSecretScanningServiceFactoryDep = {
|
||||
permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
|
||||
secretScanningDal: TSecretScanningDalFactory;
|
||||
gitAppInstallSessionDal: TGitAppInstallSessionDalFactory;
|
||||
gitAppOrgDal: TGitAppDalFactory;
|
||||
secretScanningQueue: TSecretScanningQueueFactory;
|
||||
};
|
||||
|
||||
export type TSecretScanningServiceFactory = ReturnType<typeof secretScanningServiceFactory>;
|
||||
|
||||
export const secretScanningServiceFactory = ({
|
||||
secretScanningDal,
|
||||
gitAppOrgDal,
|
||||
gitAppInstallSessionDal,
|
||||
permissionService,
|
||||
secretScanningQueue
|
||||
}: TSecretScanningServiceFactoryDep) => {
|
||||
const createInstallationSession = async ({ actor, orgId, actorId }: TInstallAppSessionDTO) => {
|
||||
const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId);
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
OrgPermissionActions.Create,
|
||||
OrgPermissionSubjects.SecretScanning
|
||||
);
|
||||
|
||||
const sessionId = crypto.randomBytes(16).toString("hex");
|
||||
await gitAppInstallSessionDal.upsert({ orgId, sessionId, userId: actorId });
|
||||
return { sessionId };
|
||||
};
|
||||
|
||||
const linkInstallationToOrg = async ({
|
||||
sessionId,
|
||||
actorId,
|
||||
installationId,
|
||||
actor
|
||||
}: TLinkInstallSessionDTO) => {
|
||||
const session = await gitAppInstallSessionDal.findOne({ sessionId });
|
||||
if (!session) throw new UnauthorizedError({ message: "Session not found" });
|
||||
|
||||
const { permission } = await permissionService.getOrgPermission(actor, actorId, session.orgId);
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
OrgPermissionActions.Create,
|
||||
OrgPermissionSubjects.SecretScanning
|
||||
);
|
||||
const installatedApp = await gitAppOrgDal.transaction(async (tx) => {
|
||||
await gitAppInstallSessionDal.deleteById(session.id, tx);
|
||||
return gitAppOrgDal.upsert({ orgId: session.orgId, installationId, userId: actorId }, tx);
|
||||
});
|
||||
|
||||
const appCfg = getConfig();
|
||||
const octokit = new ProbotOctokit({
|
||||
auth: {
|
||||
appId: appCfg.SECRET_SCANNING_GIT_APP_ID,
|
||||
privateKey: appCfg.SECRET_SCANNING_PRIVATE_KEY,
|
||||
installationId: installationId.toString()
|
||||
}
|
||||
});
|
||||
|
||||
const {
|
||||
data: { repositories }
|
||||
} = await octokit.apps.listReposAccessibleToInstallation();
|
||||
await Promise.all(
|
||||
repositories.map(({ id, full_name }) =>
|
||||
secretScanningQueue.startFullRepoScan({
|
||||
organizationId: session.orgId,
|
||||
installationId,
|
||||
repository: { id, fullName: full_name }
|
||||
})
|
||||
)
|
||||
);
|
||||
return { installatedApp };
|
||||
};
|
||||
|
||||
const getOrgInstallationStatus = async ({ actorId, orgId, actor }: TGetOrgInstallStatusDTO) => {
|
||||
const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId);
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
OrgPermissionActions.Read,
|
||||
OrgPermissionSubjects.SecretScanning
|
||||
);
|
||||
|
||||
const appInstallation = await gitAppOrgDal.findOne({ orgId });
|
||||
return Boolean(appInstallation);
|
||||
};
|
||||
|
||||
const getRisksByOrg = async ({ actor, orgId, actorId }: TGetOrgRisksDTO) => {
|
||||
const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId);
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
OrgPermissionActions.Read,
|
||||
OrgPermissionSubjects.SecretScanning
|
||||
);
|
||||
const risks = await secretScanningDal.find({ orgId }, { sort: [["createdAt", "desc"]] });
|
||||
return { risks };
|
||||
};
|
||||
|
||||
const updateRiskStatus = async ({
|
||||
actorId,
|
||||
orgId,
|
||||
actor,
|
||||
riskId,
|
||||
status
|
||||
}: TUpdateRiskStatusDTO) => {
|
||||
const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId);
|
||||
ForbiddenError.from(permission).throwUnlessCan(
|
||||
OrgPermissionActions.Edit,
|
||||
OrgPermissionSubjects.SecretScanning
|
||||
);
|
||||
|
||||
const isRiskResolved = Boolean(
|
||||
[
|
||||
SecretScanningRiskStatus.FalsePositive,
|
||||
SecretScanningRiskStatus.Revoked,
|
||||
SecretScanningRiskStatus.NotRevoked
|
||||
].includes(status)
|
||||
);
|
||||
|
||||
const risk = await secretScanningDal.updateById(riskId, {
|
||||
status,
|
||||
isResolved: isRiskResolved
|
||||
});
|
||||
return { risk };
|
||||
};
|
||||
|
||||
const handleRepoPushEvent = async (payload: PushEvent) => {
|
||||
const { commits, repository, installation, pusher } = payload;
|
||||
if (!commits || !repository || !installation || !pusher) {
|
||||
return;
|
||||
}
|
||||
|
||||
const installationLink = await gitAppOrgDal.findOne({
|
||||
installationId: String(installation.id)
|
||||
});
|
||||
if (!installationLink) return;
|
||||
|
||||
await secretScanningQueue.startPushEventScan({
|
||||
commits,
|
||||
pusher: { name: pusher.name, email: pusher.email },
|
||||
repository: { fullName: repository.full_name, id: repository.id },
|
||||
organizationId: installationLink.orgId,
|
||||
installationId: String(installation?.id)
|
||||
});
|
||||
};
|
||||
|
||||
const handleRepoDeleteEvent = async (installationId: string, repositoryIds: string[]) => {
|
||||
await secretScanningDal.transaction(async (tx) => {
|
||||
if (repositoryIds.length) {
|
||||
await Promise.all(
|
||||
Object.keys(repositoryIds).map((key) =>
|
||||
secretScanningDal.delete({ repositoryId: key }, tx)
|
||||
)
|
||||
);
|
||||
}
|
||||
await gitAppOrgDal.delete({ installationId }, tx);
|
||||
});
|
||||
};
|
||||
|
||||
return {
|
||||
createInstallationSession,
|
||||
linkInstallationToOrg,
|
||||
getOrgInstallationStatus,
|
||||
getRisksByOrg,
|
||||
updateRiskStatus,
|
||||
handleRepoPushEvent,
|
||||
handleRepoDeleteEvent
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,24 @@
|
||||
import { TOrgPermission } from "@app/lib/types";
|
||||
|
||||
export enum SecretScanningRiskStatus {
|
||||
FalsePositive = "RESOLVED_FALSE_POSITIVE",
|
||||
Revoked = "RESOLVED_REVOKED",
|
||||
NotRevoked = "RESOLVED_NOT_REVOKED",
|
||||
Unresolved = "UNRESOLVED"
|
||||
}
|
||||
|
||||
export type TInstallAppSessionDTO = TOrgPermission;
|
||||
|
||||
export type TLinkInstallSessionDTO = {
|
||||
installationId: string;
|
||||
sessionId: string;
|
||||
} & Omit<TOrgPermission, "orgId">;
|
||||
|
||||
export type TGetOrgInstallStatusDTO = TOrgPermission;
|
||||
|
||||
export type TGetOrgRisksDTO = TOrgPermission;
|
||||
|
||||
export type TUpdateRiskStatusDTO = {
|
||||
riskId: string;
|
||||
status: SecretScanningRiskStatus;
|
||||
} & TOrgPermission;
|
||||
@@ -76,9 +76,21 @@ const envSchema = z
|
||||
// google
|
||||
CLIENT_ID_GITLAB: zpStr(z.string().optional()),
|
||||
CLIENT_SECRET_GITLAB: zpStr(z.string().optional()),
|
||||
URL_GITLAB_URL: zpStr(z.string().optional().default(GITLAB_URL))
|
||||
URL_GITLAB_URL: zpStr(z.string().optional().default(GITLAB_URL)),
|
||||
// SECRET-SCANNING
|
||||
SECRET_SCANNING_WEBHOOK_PROXY: zpStr(z.string().optional()),
|
||||
SECRET_SCANNING_WEBHOOK_SECRET: zpStr(z.string().optional()),
|
||||
SECRET_SCANNING_GIT_APP_ID: zpStr(z.string().optional()),
|
||||
SECRET_SCANNING_PRIVATE_KEY: zpStr(z.string().optional())
|
||||
})
|
||||
.transform((data) => ({ ...data, isSmtpConfigured: Boolean(data.SMTP_HOST) }));
|
||||
.transform((data) => ({
|
||||
...data,
|
||||
isSmtpConfigured: Boolean(data.SMTP_HOST),
|
||||
isSecretScanningConfigured:
|
||||
Boolean(data.SECRET_SCANNING_GIT_APP_ID) &&
|
||||
Boolean(data.SECRET_SCANNING_PRIVATE_KEY) &&
|
||||
Boolean(data.SECRET_SCANNING_WEBHOOK_SECRET)
|
||||
}));
|
||||
|
||||
let envCfg: Readonly<z.infer<typeof envSchema>>;
|
||||
|
||||
|
||||
@@ -2,19 +2,26 @@ import { Job, JobsOptions, Queue, Worker, WorkerListener } from "bullmq";
|
||||
import Redis from "ioredis";
|
||||
|
||||
import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types";
|
||||
import {
|
||||
TScanFullRepoEventPayload,
|
||||
TScanPushEventPayload
|
||||
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
|
||||
|
||||
export enum QueueName {
|
||||
SecretRotation = "secret-rotation",
|
||||
AuditLog = "audit-log",
|
||||
IntegrationSync = "sync-integrations",
|
||||
SecretWebhook = "secret-webhook"
|
||||
SecretWebhook = "secret-webhook",
|
||||
SecretFullRepoScan = "secret-full-repo-scan",
|
||||
SecretPushEventScan = "secret-push-event-scan"
|
||||
}
|
||||
|
||||
export enum QueueJobs {
|
||||
SecretRotation = "secret-rotation-job",
|
||||
AuditLog = "audit-log-job",
|
||||
SecWebhook = "secret-webhook-trigger",
|
||||
IntegrationSync = "secret-integration-pull"
|
||||
IntegrationSync = "secret-integration-pull",
|
||||
SecretScan = "secret-scan"
|
||||
}
|
||||
|
||||
export type TQueueJobTypes = {
|
||||
@@ -34,6 +41,11 @@ export type TQueueJobTypes = {
|
||||
name: QueueJobs.IntegrationSync;
|
||||
payload: { projectId: string; environment: string; secretPath: string };
|
||||
};
|
||||
[QueueName.SecretFullRepoScan]: {
|
||||
name: QueueJobs.SecretScan;
|
||||
payload: TScanFullRepoEventPayload;
|
||||
};
|
||||
[QueueName.SecretPushEventScan]: { name: QueueJobs.SecretScan; payload: TScanPushEventPayload };
|
||||
};
|
||||
|
||||
export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;
|
||||
|
||||
@@ -53,12 +53,16 @@ export const main = async ({ db, smtp, logger, queue }: TMain) => {
|
||||
|
||||
await server.register(fastifySwagger);
|
||||
await server.register(fastifyFormBody);
|
||||
// allow empty body on post request
|
||||
// server.addContentTypeParser("application/json", { bodyLimit: 0 }, (_request, _payload, done) =>
|
||||
// done(null, null)
|
||||
// );
|
||||
|
||||
// Rate limiters and security headers
|
||||
await server.register<FastifyRateLimitOptions>(ratelimiter, globalRateLimiterCfg);
|
||||
await server.register(helmet, { contentSecurityPolicy: false });
|
||||
|
||||
await server.register(registerRoutes, { prefix: "/api", smtp, queue, db });
|
||||
await server.register(registerRoutes, { smtp, queue, db });
|
||||
await server.ready();
|
||||
server.swagger();
|
||||
return server;
|
||||
|
||||
65
backend-pg/src/server/plugins/secret-scanner.ts
Normal file
65
backend-pg/src/server/plugins/secret-scanner.ts
Normal file
@@ -0,0 +1,65 @@
|
||||
import { Probot } from "probot";
|
||||
import SmeeClient from "smee-client";
|
||||
|
||||
import { getConfig } from "@app/lib/config/env";
|
||||
import { logger } from "@app/lib/logger";
|
||||
|
||||
export const registerSecretScannerGhApp = async (server: FastifyZodProvider) => {
|
||||
const probotApp = (app: Probot) => {
|
||||
app.on("installation.deleted", async (context) => {
|
||||
const { payload } = context;
|
||||
const { installation, repositories } = payload;
|
||||
await server.services.secretScanning.handleRepoDeleteEvent(
|
||||
String(installation.id),
|
||||
(repositories || [])?.map(({ id }) => String(id))
|
||||
);
|
||||
});
|
||||
|
||||
app.on("installation", async (context) => {
|
||||
const { payload } = context;
|
||||
logger.info("Installed secret scanner to:", { repositories: payload.repositories });
|
||||
});
|
||||
|
||||
app.on("push", async (context) => {
|
||||
const { payload } = context;
|
||||
await server.services.secretScanning.handleRepoPushEvent(payload);
|
||||
});
|
||||
};
|
||||
|
||||
const appCfg = getConfig();
|
||||
if (appCfg.isSecretScanningConfigured) {
|
||||
const probot = new Probot({
|
||||
appId: appCfg.SECRET_SCANNING_GIT_APP_ID as string,
|
||||
privateKey: appCfg.SECRET_SCANNING_PRIVATE_KEY as string,
|
||||
secret: appCfg.SECRET_SCANNING_WEBHOOK_SECRET as string
|
||||
});
|
||||
|
||||
if (appCfg.NODE_ENV === "development") {
|
||||
const smee = new SmeeClient({
|
||||
source: appCfg.SECRET_SCANNING_WEBHOOK_PROXY as string,
|
||||
target: "http://backend:4000/ss-webhook",
|
||||
logger: console
|
||||
});
|
||||
smee.start();
|
||||
}
|
||||
|
||||
await probot.load(probotApp);
|
||||
|
||||
server.route({
|
||||
method: "POST",
|
||||
url: "/",
|
||||
handler: async (req, res) => {
|
||||
const eventName = req.headers["x-github-event"] as any;
|
||||
const signatureSHA256 = req.headers["x-hub-signature-256"] as string;
|
||||
const id = req.headers["x-github-delivery"] as string;
|
||||
await probot.webhooks.verifyAndReceive({
|
||||
id,
|
||||
name: eventName,
|
||||
payload: req.body as string,
|
||||
signature: signatureSHA256
|
||||
});
|
||||
res.send("ok");
|
||||
}
|
||||
});
|
||||
}
|
||||
};
|
||||
@@ -19,6 +19,11 @@ import { secretApprovalRequestServiceFactory } from "@app/ee/services/secret-app
|
||||
import { secretRotationDalFactory } from "@app/ee/services/secret-rotation/secret-rotation-dal";
|
||||
import { secretRotationQueueFactory } from "@app/ee/services/secret-rotation/secret-rotation-queue";
|
||||
import { secretRotationServiceFactory } from "@app/ee/services/secret-rotation/secret-rotation-service";
|
||||
import { gitAppDalFactory } from "@app/ee/services/secret-scanning/git-app-dal";
|
||||
import { gitAppInstallSessionDalFactory } from "@app/ee/services/secret-scanning/git-app-install-session-dal";
|
||||
import { secretScanningDalFactory } from "@app/ee/services/secret-scanning/secret-scanning-dal";
|
||||
import { secretScanningQueueFactory } from "@app/ee/services/secret-scanning/secret-scanning-queue";
|
||||
import { secretScanningServiceFactory } from "@app/ee/services/secret-scanning/secret-scanning-service";
|
||||
import { secretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
|
||||
import { snapshotDalFactory } from "@app/ee/services/secret-snapshot/snapshot-dal";
|
||||
import { snapshotFolderDalFactory } from "@app/ee/services/secret-snapshot/snapshot-folder-dal";
|
||||
@@ -90,6 +95,7 @@ import { webhookServiceFactory } from "@app/services/webhook/webhook-service";
|
||||
import { injectAuditLogInfo } from "../plugins/audit-log";
|
||||
import { injectIdentity } from "../plugins/auth/inject-identity";
|
||||
import { injectPermission } from "../plugins/auth/inject-permission";
|
||||
import { registerSecretScannerGhApp } from "../plugins/secret-scanner";
|
||||
import { registerV1Routes } from "./v1";
|
||||
import { registerV2Routes } from "./v2";
|
||||
import { registerV3Routes } from "./v3";
|
||||
@@ -102,6 +108,8 @@ export const registerRoutes = async (
|
||||
queue: queueService
|
||||
}: { db: Knex; smtp: TSmtpService; queue: TQueueServiceFactory }
|
||||
) => {
|
||||
server.register(registerSecretScannerGhApp, { prefix: "/ss-webhook" });
|
||||
|
||||
// db layers
|
||||
const userDal = userDalFactory(db);
|
||||
const authDal = authDalFactory(db);
|
||||
@@ -157,6 +165,10 @@ export const registerRoutes = async (
|
||||
const snapshotSecretDal = snapshotSecretDalFactory(db);
|
||||
const snapshotFolderDal = snapshotFolderDalFactory(db);
|
||||
|
||||
const gitAppInstallSessionDal = gitAppInstallSessionDalFactory(db);
|
||||
const gitAppOrgDal = gitAppDalFactory(db);
|
||||
const secretScanningDal = secretScanningDalFactory(db);
|
||||
|
||||
const permissionService = permissionServiceFactory({ permissionDal, orgRoleDal, projectRoleDal });
|
||||
const auditLogQueue = auditLogQueueServiceFactory({ auditLogDal, queueService });
|
||||
const auditLogService = auditLogServiceFactory({ auditLogDal, permissionService, auditLogQueue });
|
||||
@@ -210,6 +222,20 @@ export const registerRoutes = async (
|
||||
});
|
||||
const apiKeyService = apiKeyServiceFactory({ apiKeyDal });
|
||||
|
||||
const secretScanningQueue = secretScanningQueueFactory({
|
||||
userDal,
|
||||
smtpService,
|
||||
secretScanningDal,
|
||||
queueService,
|
||||
orgMembershipDal: orgDal
|
||||
});
|
||||
const secretScanningService = secretScanningServiceFactory({
|
||||
permissionService,
|
||||
gitAppOrgDal,
|
||||
gitAppInstallSessionDal,
|
||||
secretScanningDal,
|
||||
secretScanningQueue
|
||||
});
|
||||
const projectService = projectServiceFactory({
|
||||
permissionService,
|
||||
projectDal,
|
||||
@@ -391,7 +417,8 @@ export const registerRoutes = async (
|
||||
secretRotation: secretRotationService,
|
||||
snapshot: snapshotService,
|
||||
saml: samlService,
|
||||
auditLog: auditLogService
|
||||
auditLog: auditLogService,
|
||||
secretScanning: secretScanningService
|
||||
});
|
||||
|
||||
server.decorate<FastifyZodProvider["store"]>("store", {
|
||||
@@ -418,12 +445,12 @@ export const registerRoutes = async (
|
||||
}
|
||||
},
|
||||
handler: () => {
|
||||
const appCfg = getConfig();
|
||||
const cfg = getConfig();
|
||||
|
||||
return {
|
||||
date: new Date(),
|
||||
message: "Ok" as const,
|
||||
emailConfigured: appCfg.isSmtpConfigured,
|
||||
emailConfigured: cfg.isSmtpConfigured,
|
||||
inviteOnlySignup: false,
|
||||
redisConfigured: false,
|
||||
secretScanningConfigured: false
|
||||
@@ -437,8 +464,8 @@ export const registerRoutes = async (
|
||||
await v1Server.register(registerV1EERoutes);
|
||||
await v1Server.register(registerV1Routes);
|
||||
},
|
||||
{ prefix: "/v1" }
|
||||
{ prefix: "/api/v1" }
|
||||
);
|
||||
await server.register(registerV2Routes, { prefix: "/v2" });
|
||||
await server.register(registerV3Routes, { prefix: "/v3" });
|
||||
await server.register(registerV2Routes, { prefix: "/api/v2" });
|
||||
await server.register(registerV3Routes, { prefix: "/api/v3" });
|
||||
};
|
||||
|
||||
@@ -2,20 +2,23 @@ import SecurityClient from "@app/components/utilities/SecurityClient";
|
||||
|
||||
/**
|
||||
* Will create a new integration session and return it for the given org
|
||||
* @returns
|
||||
* @returns
|
||||
*/
|
||||
const createNewIntegrationSession = (organizationId: string) =>
|
||||
SecurityClient.fetchCall(`/api/v1/secret-scanning/create-installation-session/organization/${organizationId}`, {
|
||||
SecurityClient.fetchCall("/api/v1/secret-scanning/create-installation-session/organization", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
},
|
||||
body: JSON.stringify({
|
||||
organizationId
|
||||
})
|
||||
}).then(async (res) => {
|
||||
if (res && res.status === 200) {
|
||||
return res.json();
|
||||
}
|
||||
console.log("Failed to create integration session");
|
||||
console.log("response", res)
|
||||
console.log("response", res);
|
||||
return undefined;
|
||||
});
|
||||
|
||||
|
||||
@@ -2,20 +2,23 @@ import SecurityClient from "@app/components/utilities/SecurityClient";
|
||||
|
||||
/**
|
||||
* Will create a new integration session and return it for the given org
|
||||
* @returns
|
||||
* @returns
|
||||
*/
|
||||
const getInstallationStatus = (organizationId: string) =>
|
||||
SecurityClient.fetchCall(`/api/v1/secret-scanning/installation-status/organization/${organizationId}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"Content-Type": "application/json"
|
||||
SecurityClient.fetchCall(
|
||||
`/api/v1/secret-scanning/installation-status/organization/${organizationId}`,
|
||||
{
|
||||
method: "GET",
|
||||
headers: {
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
}
|
||||
}).then(async (res) => {
|
||||
).then(async (res) => {
|
||||
if (res && res.status === 200) {
|
||||
return (await res.json()).appInstallationComplete;
|
||||
return (await res.json()).appInstallationCompleted;
|
||||
}
|
||||
console.log("Failed to check installation status");
|
||||
console.log("response", res)
|
||||
console.log("response", res);
|
||||
return undefined;
|
||||
});
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ export type IGitRisks = {
|
||||
email: string;
|
||||
};
|
||||
createdAt: string;
|
||||
organization: string;
|
||||
orgId: string;
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -22,11 +22,11 @@ const SecretScanning = withPermission(
|
||||
const linkInstallation = async () => {
|
||||
if (
|
||||
typeof queryParams.state === "string" &&
|
||||
typeof queryParams.installationid === "string"
|
||||
typeof queryParams.installation_id === "string"
|
||||
) {
|
||||
try {
|
||||
const isLinked = await linkGitAppInstallationWithOrganization(
|
||||
queryParams.installationid as string,
|
||||
queryParams.installation_id as string,
|
||||
queryParams.state as string
|
||||
);
|
||||
if (isLinked) {
|
||||
@@ -47,12 +47,12 @@ const SecretScanning = withPermission(
|
||||
|
||||
fetchInstallationStatus();
|
||||
linkInstallation();
|
||||
}, [queryParams.state, queryParams.installationid]);
|
||||
}, [queryParams.state, queryParams.installation_id]);
|
||||
|
||||
const generateNewIntegrationSession = async () => {
|
||||
const session = await createNewIntegrationSession(String(localStorage.getItem("orgData.id")));
|
||||
router.push(
|
||||
`https://github.com/apps/infisical-radar/installations/new?state=${session.sessionId}`
|
||||
`https://github.com/apps/infisical-test/installations/new?state=${session.sessionId}`
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user