feature: secret scanning architecture and github data source (wip)

backend/src/lib/api-docs/constants.ts

@@ -3,6 +3,12 @@ import {
   SECRET_ROTATION_CONNECTION_MAP,
   SECRET_ROTATION_NAME_MAP
 } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-maps";
+import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
+import {
+  AUTO_SYNC_DESCRIPTION_HELPER,
+  SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP,
+  SECRET_SCANNING_DATA_SOURCE_NAME_MAP
+} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-maps";
 import { AppConnection } from "@app/services/app-connection/app-connection-enums";
 import { APP_CONNECTION_NAME_MAP } from "@app/services/app-connection/app-connection-maps";
 import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
@@ -55,7 +61,8 @@ export enum ApiDocsTags {
   SshHostGroups = "SSH Host Groups",
   KmsKeys = "KMS Keys",
   KmsEncryption = "KMS Encryption",
-  KmsSigning = "KMS Signing"
+  KmsSigning = "KMS Signing",
+  SecretScanning = "Secret Scanning"
 }
 
 export const GROUPS = {
@@ -2351,3 +2358,66 @@ export const SecretRotations = {
     }
   }
 };
+
+export const SecretScanningDataSources = {
+  LIST: (type?: SecretScanningDataSource) => ({
+    projectId: `The ID of the project to list ${type ? SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type] : "Scanning"} Data Sources from.`
+  }),
+  GET_BY_ID: (type: SecretScanningDataSource) => ({
+    dataSourceId: `The ID of the ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source to retrieve.`
+  }),
+  GET_BY_NAME: (type: SecretScanningDataSource) => ({
+    sourceName: `The name of the ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source to retrieve.`,
+    projectId: `The ID of the project the ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source is located in.`
+  }),
+  CREATE: (type: SecretScanningDataSource) => {
+    const sourceType = SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type];
+    const autoScanDescription = AUTO_SYNC_DESCRIPTION_HELPER[type];
+    return {
+      name: `The name of the ${sourceType} Data Source to create. Must be slug-friendly.`,
+      description: `An optional description for the ${sourceType} Data Source.`,
+      projectId: `The ID of the project to create the ${sourceType} Data Source in.`,
+      connectionId: `The ID of the ${
+        APP_CONNECTION_NAME_MAP[SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[type]]
+      } Connection to use for this Data Source.`,
+      isAutoScanEnabled: `Whether scans should be automatically performed when a ${autoScanDescription.verb} occurs to ${autoScanDescription.noun} associated with this Data Source.`,
+      config: `The configuration parameters to use for this Data Source.`
+    };
+  },
+  UPDATE: (type: SecretScanningDataSource) => {
+    const typeName = SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type];
+    const autoScanDescription = AUTO_SYNC_DESCRIPTION_HELPER[type];
+
+    return {
+      dataSourceId: `The ID of the ${typeName} Data Source to be updated.`,
+      name: `The updated name of the ${typeName} Data Source. Must be slug-friendly.`,
+      description: `The updated description of the ${typeName} Data Source.`,
+      isAutoScanEnabled: `Whether scans should be automatically performed when a ${autoScanDescription.verb} occurs to ${autoScanDescription.noun} associated with this Data Source.`,
+      config: `The updated configuration parameters to use for this Data Source.`
+    };
+  },
+  DELETE: (type: SecretScanningDataSource) => ({
+    dataSourceId: `The ID of the ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source to be deleted.`
+  }),
+  SCAN: (type: SecretScanningDataSource) => ({
+    dataSourceId: `The ID of the ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source to trigger a scan for.`,
+    resourceId: `The ID of the individual Data Source resource to trigger a scan for.`
+  }),
+  LIST_RESOURCES: (type: SecretScanningDataSource) => ({
+    dataSourceId: `The ID of the ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source to list resources from.`
+  }),
+  LIST_SCANS: (type: SecretScanningDataSource) => ({
+    dataSourceId: `The ID of the ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]} Data Source to list scans for.`
+  })
+};
+
+export const SecretScanningFindings = {
+  LIST: {
+    projectId: `The ID of the project to list Secret Scanning Findings from.`
+  },
+  UPDATE: {
+    findingId: "The ID of the Secret Scanning Finding to update the resolve status for.",
+    status: "The updated status of the specified Secret Scanning Finding.",
+    remarks: "Remarks pertaining to the resolve status of this finding."
+  }
+};

backend/src/lib/config/env.ts

@@ -215,6 +215,14 @@ const envSchema = z
     INF_APP_CONNECTION_GITHUB_APP_SLUG: zpStr(z.string().optional()),
     INF_APP_CONNECTION_GITHUB_APP_ID: zpStr(z.string().optional()),
 
+    // github radar app
+    INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_ID: zpStr(z.string().optional()),
+    INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_SECRET: zpStr(z.string().optional()),
+    INF_APP_CONNECTION_GITHUB_RADAR_APP_PRIVATE_KEY: zpStr(z.string().optional()),
+    INF_APP_CONNECTION_GITHUB_RADAR_APP_SLUG: zpStr(z.string().optional()),
+    INF_APP_CONNECTION_GITHUB_RADAR_APP_ID: zpStr(z.string().optional()),
+    INF_APP_CONNECTION_GITHUB_RADAR_APP_WEBHOOK_SECRET: zpStr(z.string().optional()),
+
     // gcp app
     INF_APP_CONNECTION_GCP_SERVICE_ACCOUNT_CREDENTIAL: zpStr(z.string().optional()),
 

backend/src/lib/knex/index.ts

@@ -179,13 +179,18 @@ export const ormify = <DbOps extends object, Tname extends keyof Tables>(db: Kne
       throw new DatabaseError({ error, name: "batchInsert" });
     }
   },
-  upsert: async (data: readonly Tables[Tname]["insert"][], onConflictField: keyof Tables[Tname]["base"], tx?: Knex) => {
+  upsert: async (
+    data: readonly Tables[Tname]["insert"][],
+    onConflictField: keyof Tables[Tname]["base"] | Array<keyof Tables[Tname]["base"]>,
+    tx?: Knex,
+    mergeColumns?: (keyof Knex.ResolveTableType<Knex.TableType<Tname>, "update">)[] | undefined
+  ) => {
     try {
       if (!data.length) return [];
       const res = await (tx || db)(tableName)
         .insert(data as never)
         .onConflict(onConflictField as never)
-        .merge()
+        .merge(mergeColumns)
         .returning("*");
       return res;
     } catch (error) {