diff --git a/backend/src/@types/fastify.d.ts b/backend/src/@types/fastify.d.ts index 02159fa9c..f3298625e 100644 --- a/backend/src/@types/fastify.d.ts +++ b/backend/src/@types/fastify.d.ts @@ -80,6 +80,7 @@ import { TSecretFolderServiceFactory } from "@app/services/secret-folder/secret- import { TSecretImportServiceFactory } from "@app/services/secret-import/secret-import-service"; import { TSecretReplicationServiceFactory } from "@app/services/secret-replication/secret-replication-service"; import { TSecretSharingServiceFactory } from "@app/services/secret-sharing/secret-sharing-service"; +import { TSecretSyncServiceFactory } from "@app/services/secret-sync/secret-sync-service"; import { TSecretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service"; import { TServiceTokenServiceFactory } from "@app/services/service-token/service-token-service"; import { TSlackServiceFactory } from "@app/services/slack/slack-service"; @@ -210,6 +211,7 @@ declare module "fastify" { projectTemplate: TProjectTemplateServiceFactory; totp: TTotpServiceFactory; appConnection: TAppConnectionServiceFactory; + secretSync: TSecretSyncServiceFactory; }; // this is exclusive use for middlewares in which we need to inject data // everywhere else access using service layer diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index aaa2014b8..2dad77392 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -372,6 +372,7 @@ import { TExternalGroupOrgRoleMappingsInsert, TExternalGroupOrgRoleMappingsUpdate } from "@app/db/schemas/external-group-org-role-mappings"; +import { TSecretSyncs, TSecretSyncsInsert, TSecretSyncsUpdate } from "@app/db/schemas/secret-syncs"; import { TSecretV2TagJunction, TSecretV2TagJunctionInsert, @@ -900,5 +901,6 @@ declare module "knex/types/tables" { TAppConnectionsInsert, TAppConnectionsUpdate >; + [TableName.SecretSync]: KnexOriginal.CompositeTableType; } } diff --git a/backend/src/db/migrations/20250122055102_secret-sync.ts b/backend/src/db/migrations/20250122055102_secret-sync.ts new file mode 100644 index 000000000..5f37950e3 --- /dev/null +++ b/backend/src/db/migrations/20250122055102_secret-sync.ts @@ -0,0 +1,50 @@ +import { Knex } from "knex"; + +import { TableName } from "@app/db/schemas"; +import { createOnUpdateTrigger, dropOnUpdateTrigger } from "@app/db/utils"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasTable(TableName.SecretSync))) { + await knex.schema.createTable(TableName.SecretSync, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.string("name", 32).notNullable(); + t.string("description"); + t.string("destination").notNullable(); + t.boolean("isAutoSyncEnabled").notNullable().defaultTo(true); + t.integer("version").defaultTo(1).notNullable(); + t.jsonb("destinationConfig").notNullable(); + t.jsonb("syncOptions").notNullable(); + // we're including projectId in addition to folder ID because we allow folderId to be null (if the folder + // is deleted), to preserve sync configuration + t.string("projectId").notNullable(); + t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE"); + t.uuid("folderId"); + t.foreign("folderId").references("id").inTable(TableName.SecretFolder).onDelete("SET NULL"); + t.uuid("connectionId").notNullable(); + t.foreign("connectionId").references("id").inTable(TableName.AppConnection); + t.timestamps(true, true, true); + // sync secrets to destination + t.string("syncStatus"); + t.string("lastSyncJobId"); + t.string("lastSyncMessage"); + t.datetime("lastSyncedAt"); + // import secrets from destination + t.string("importStatus"); + t.string("lastImportJobId"); + t.string("lastImportMessage"); + t.datetime("lastImportedAt"); + // remove secrets from destination + t.string("removeStatus"); + t.string("lastRemoveJobId"); + t.string("lastRemoveMessage"); + t.datetime("lastRemovedAt"); + }); + + await createOnUpdateTrigger(knex, TableName.SecretSync); + } +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists(TableName.SecretSync); + await dropOnUpdateTrigger(knex, TableName.SecretSync); +} diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index c991b913d..3ead85530 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -131,7 +131,8 @@ export enum TableName { WorkflowIntegrations = "workflow_integrations", SlackIntegrations = "slack_integrations", ProjectSlackConfigs = "project_slack_configs", - AppConnection = "app_connections" + AppConnection = "app_connections", + SecretSync = "secret_syncs" } export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt"; diff --git a/backend/src/db/schemas/secret-syncs.ts b/backend/src/db/schemas/secret-syncs.ts new file mode 100644 index 000000000..0e0728e87 --- /dev/null +++ b/backend/src/db/schemas/secret-syncs.ts @@ -0,0 +1,40 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const SecretSyncsSchema = z.object({ + id: z.string().uuid(), + name: z.string(), + description: z.string().nullable().optional(), + destination: z.string(), + isAutoSyncEnabled: z.boolean().default(true), + version: z.number().default(1), + destinationConfig: z.unknown(), + syncOptions: z.unknown(), + projectId: z.string(), + folderId: z.string().uuid().nullable().optional(), + connectionId: z.string().uuid(), + createdAt: z.date(), + updatedAt: z.date(), + syncStatus: z.string().nullable().optional(), + lastSyncJobId: z.string().nullable().optional(), + lastSyncMessage: z.string().nullable().optional(), + lastSyncedAt: z.date().nullable().optional(), + importStatus: z.string().nullable().optional(), + lastImportJobId: z.string().nullable().optional(), + lastImportMessage: z.string().nullable().optional(), + lastImportedAt: z.date().nullable().optional(), + removeStatus: z.string().nullable().optional(), + lastRemoveJobId: z.string().nullable().optional(), + lastRemoveMessage: z.string().nullable().optional(), + lastRemovedAt: z.date().nullable().optional() +}); + +export type TSecretSyncs = z.infer; +export type TSecretSyncsInsert = Omit, TImmutableDBKeys>; +export type TSecretSyncsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/ee/routes/v1/org-role-router.ts b/backend/src/ee/routes/v1/org-role-router.ts index 3a0ad47da..c8ee03a99 100644 --- a/backend/src/ee/routes/v1/org-role-router.ts +++ b/backend/src/ee/routes/v1/org-role-router.ts @@ -24,6 +24,7 @@ export const registerOrgRoleRouter = async (server: FastifyZodProvider) => { ), name: z.string().trim(), description: z.string().trim().nullish(), + // TODO(scott): once UI refactored permissions: OrgPermissionSchema.array() permissions: z.any().array() }), response: { @@ -96,6 +97,7 @@ export const registerOrgRoleRouter = async (server: FastifyZodProvider) => { .optional(), name: z.string().trim().optional(), description: z.string().trim().nullish(), + // TODO(scott): once UI refactored permissions: OrgPermissionSchema.array().optional() permissions: z.any().array().optional() }), response: { diff --git a/backend/src/ee/services/audit-log/audit-log-service.ts b/backend/src/ee/services/audit-log/audit-log-service.ts index e51c08fbe..ff7dede5f 100644 --- a/backend/src/ee/services/audit-log/audit-log-service.ts +++ b/backend/src/ee/services/audit-log/audit-log-service.ts @@ -81,7 +81,8 @@ export const auditLogServiceFactory = ({ } // add all cases in which project id or org id cannot be added if (data.event.type !== EventType.LOGIN_IDENTITY_UNIVERSAL_AUTH) { - if (!data.projectId && !data.orgId) throw new BadRequestError({ message: "Must either project id or org id" }); + if (!data.projectId && !data.orgId) + throw new BadRequestError({ message: "Must specify either project id or org id" }); } return auditLogQueue.pushToLog(data); diff --git a/backend/src/ee/services/audit-log/audit-log-types.ts b/backend/src/ee/services/audit-log/audit-log-types.ts index 2a5657cf0..9c19cd3cc 100644 --- a/backend/src/ee/services/audit-log/audit-log-types.ts +++ b/backend/src/ee/services/audit-log/audit-log-types.ts @@ -13,6 +13,13 @@ import { CertKeyAlgorithm } from "@app/services/certificate/certificate-types"; import { CaStatus } from "@app/services/certificate-authority/certificate-authority-types"; import { TIdentityTrustedIp } from "@app/services/identity/identity-types"; import { PkiItemType } from "@app/services/pki-collection/pki-collection-types"; +import { SecretSync, SecretSyncImportBehavior } from "@app/services/secret-sync/secret-sync-enums"; +import { + TCreateSecretSyncDTO, + TDeleteSecretSyncDTO, + TSecretSyncRaw, + TUpdateSecretSyncDTO +} from "@app/services/secret-sync/secret-sync-types"; export type TListProjectAuditLogDTO = { filter: { @@ -226,13 +233,22 @@ export enum EventType { DELETE_PROJECT_TEMPLATE = "delete-project-template", APPLY_PROJECT_TEMPLATE = "apply-project-template", GET_APP_CONNECTIONS = "get-app-connections", + GET_AVAILABLE_APP_CONNECTIONS_DETAILS = "get-available-app-connections-details", GET_APP_CONNECTION = "get-app-connection", CREATE_APP_CONNECTION = "create-app-connection", UPDATE_APP_CONNECTION = "update-app-connection", DELETE_APP_CONNECTION = "delete-app-connection", CREATE_SHARED_SECRET = "create-shared-secret", DELETE_SHARED_SECRET = "delete-shared-secret", - READ_SHARED_SECRET = "read-shared-secret" + READ_SHARED_SECRET = "read-shared-secret", + GET_SECRET_SYNCS = "get-secret-syncs", + GET_SECRET_SYNC = "get-secret-sync", + CREATE_SECRET_SYNC = "create-secret-sync", + UPDATE_SECRET_SYNC = "update-secret-sync", + DELETE_SECRET_SYNC = "delete-secret-sync", + SECRET_SYNC_SYNC_SECRETS = "secret-sync-sync-secrets", + SECRET_SYNC_IMPORT_SECRETS = "secret-sync-import-secrets", + SECRET_SYNC_REMOVE_SECRETS = "secret-sync-remove-secrets" } interface UserActorMetadata { @@ -1893,6 +1909,15 @@ interface GetAppConnectionsEvent { }; } +interface GetAvailableAppConnectionsDetailsEvent { + type: EventType.GET_AVAILABLE_APP_CONNECTIONS_DETAILS; + metadata: { + app?: AppConnection; + count: number; + connectionIds: string[]; + }; +} + interface GetAppConnectionEvent { type: EventType.GET_APP_CONNECTION; metadata: { @@ -1946,6 +1971,78 @@ interface ReadSharedSecretEvent { }; } +interface GetSecretSyncsEvent { + type: EventType.GET_SECRET_SYNCS; + metadata: { + destination?: SecretSync; + count: number; + syncIds: string[]; + }; +} + +interface GetSecretSyncEvent { + type: EventType.GET_SECRET_SYNC; + metadata: { + destination: SecretSync; + syncId: string; + }; +} + +interface CreateSecretSyncEvent { + type: EventType.CREATE_SECRET_SYNC; + metadata: Omit & { syncId: string }; +} + +interface UpdateSecretSyncEvent { + type: EventType.UPDATE_SECRET_SYNC; + metadata: TUpdateSecretSyncDTO; +} + +interface DeleteSecretSyncEvent { + type: EventType.DELETE_SECRET_SYNC; + metadata: TDeleteSecretSyncDTO; +} + +interface SecretSyncSyncSecretsEvent { + type: EventType.SECRET_SYNC_SYNC_SECRETS; + metadata: Pick< + TSecretSyncRaw, + "syncOptions" | "destinationConfig" | "destination" | "syncStatus" | "connectionId" | "folderId" + > & { + syncId: string; + syncMessage: string | null; + jobId: string; + jobRanAt: Date; + }; +} + +interface SecretSyncImportSecretsEvent { + type: EventType.SECRET_SYNC_IMPORT_SECRETS; + metadata: Pick< + TSecretSyncRaw, + "syncOptions" | "destinationConfig" | "destination" | "importStatus" | "connectionId" | "folderId" + > & { + syncId: string; + importMessage: string | null; + jobId: string; + jobRanAt: Date; + importBehavior: SecretSyncImportBehavior; + }; +} + +interface SecretSyncRemoveSecretsEvent { + type: EventType.SECRET_SYNC_REMOVE_SECRETS; + metadata: Pick< + TSecretSyncRaw, + "syncOptions" | "destinationConfig" | "destination" | "removeStatus" | "connectionId" | "folderId" + > & { + syncId: string; + removeMessage: string | null; + jobId: string; + jobRanAt: Date; + }; +} + export type Event = | GetSecretsEvent | GetSecretEvent @@ -2119,10 +2216,19 @@ export type Event = | DeleteProjectTemplateEvent | ApplyProjectTemplateEvent | GetAppConnectionsEvent + | GetAvailableAppConnectionsDetailsEvent | GetAppConnectionEvent | CreateAppConnectionEvent | UpdateAppConnectionEvent | DeleteAppConnectionEvent | CreateSharedSecretEvent | DeleteSharedSecretEvent - | ReadSharedSecretEvent; + | ReadSharedSecretEvent + | GetSecretSyncsEvent + | GetSecretSyncEvent + | CreateSecretSyncEvent + | UpdateSecretSyncEvent + | DeleteSecretSyncEvent + | SecretSyncSyncSecretsEvent + | SecretSyncImportSecretsEvent + | SecretSyncRemoveSecretsEvent; diff --git a/backend/src/ee/services/license/license-fns.ts b/backend/src/ee/services/license/license-fns.ts index 014fcccfd..c54daa3a6 100644 --- a/backend/src/ee/services/license/license-fns.ts +++ b/backend/src/ee/services/license/license-fns.ts @@ -50,8 +50,7 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({ }, pkiEst: false, enforceMfa: false, - projectTemplates: false, - appConnections: false + projectTemplates: false }); export const setupLicenseRequestWithStore = (baseURL: string, refreshUrl: string, licenseKey: string) => { diff --git a/backend/src/ee/services/license/license-types.ts b/backend/src/ee/services/license/license-types.ts index 678e15fd4..0242377b0 100644 --- a/backend/src/ee/services/license/license-types.ts +++ b/backend/src/ee/services/license/license-types.ts @@ -68,7 +68,6 @@ export type TFeatureSet = { pkiEst: boolean; enforceMfa: boolean; projectTemplates: false; - appConnections: false; // TODO: remove once live }; export type TOrgPlansTableDTO = { diff --git a/backend/src/ee/services/permission/org-permission.ts b/backend/src/ee/services/permission/org-permission.ts index 487d2155c..c72008057 100644 --- a/backend/src/ee/services/permission/org-permission.ts +++ b/backend/src/ee/services/permission/org-permission.ts @@ -1,4 +1,12 @@ -import { AbilityBuilder, createMongoAbility, MongoAbility } from "@casl/ability"; +import { AbilityBuilder, createMongoAbility, ForcedSubject, MongoAbility } from "@casl/ability"; +import { z } from "zod"; + +import { + CASL_ACTION_SCHEMA_ENUM, + CASL_ACTION_SCHEMA_NATIVE_ENUM +} from "@app/ee/services/permission/permission-schemas"; +import { PermissionConditionSchema } from "@app/ee/services/permission/permission-types"; +import { PermissionConditionOperators } from "@app/lib/casl"; export enum OrgPermissionActions { Read = "read", @@ -7,6 +15,14 @@ export enum OrgPermissionActions { Delete = "delete" } +export enum OrgPermissionAppConnectionActions { + Read = "read", + Create = "create", + Edit = "edit", + Delete = "delete", + Connect = "connect" +} + export enum OrgPermissionAdminConsoleAction { AccessAllProjects = "access-all-projects" } @@ -31,6 +47,10 @@ export enum OrgPermissionSubjects { AppConnections = "app-connections" } +export type AppConnectionSubjectFields = { + connectionId: string; +}; + export type OrgPermissionSet = | [OrgPermissionActions.Create, OrgPermissionSubjects.Workspace] | [OrgPermissionActions, OrgPermissionSubjects.Role] @@ -47,9 +67,109 @@ export type OrgPermissionSet = | [OrgPermissionActions, OrgPermissionSubjects.Kms] | [OrgPermissionActions, OrgPermissionSubjects.AuditLogs] | [OrgPermissionActions, OrgPermissionSubjects.ProjectTemplates] - | [OrgPermissionActions, OrgPermissionSubjects.AppConnections] + | [ + OrgPermissionAppConnectionActions, + ( + | OrgPermissionSubjects.AppConnections + | (ForcedSubject & AppConnectionSubjectFields) + ) + ] | [OrgPermissionAdminConsoleAction, OrgPermissionSubjects.AdminConsole]; +const AppConnectionConditionSchema = z + .object({ + connectionId: z.union([ + z.string(), + z + .object({ + [PermissionConditionOperators.$EQ]: PermissionConditionSchema[PermissionConditionOperators.$EQ], + [PermissionConditionOperators.$NEQ]: PermissionConditionSchema[PermissionConditionOperators.$NEQ], + [PermissionConditionOperators.$IN]: PermissionConditionSchema[PermissionConditionOperators.$IN] + }) + .partial() + ]) + }) + .partial(); + +export const OrgPermissionSchema = z.discriminatedUnion("subject", [ + z.object({ + subject: z.literal(OrgPermissionSubjects.Workspace).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_ENUM([OrgPermissionActions.Create]).describe("Describe what action an entity can take.") + }), + z.object({ + subject: z.literal(OrgPermissionSubjects.Role).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.") + }), + z.object({ + subject: z.literal(OrgPermissionSubjects.Member).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.") + }), + z.object({ + subject: z.literal(OrgPermissionSubjects.Settings).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.") + }), + z.object({ + subject: z.literal(OrgPermissionSubjects.IncidentAccount).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.") + }), + z.object({ + subject: z.literal(OrgPermissionSubjects.Sso).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.") + }), + z.object({ + subject: z.literal(OrgPermissionSubjects.Scim).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.") + }), + z.object({ + subject: z.literal(OrgPermissionSubjects.Ldap).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.") + }), + z.object({ + subject: z.literal(OrgPermissionSubjects.Groups).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.") + }), + z.object({ + subject: z.literal(OrgPermissionSubjects.SecretScanning).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.") + }), + z.object({ + subject: z.literal(OrgPermissionSubjects.Billing).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.") + }), + z.object({ + subject: z.literal(OrgPermissionSubjects.Identity).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.") + }), + z.object({ + subject: z.literal(OrgPermissionSubjects.Kms).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.") + }), + z.object({ + subject: z.literal(OrgPermissionSubjects.AuditLogs).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.") + }), + z.object({ + subject: z.literal(OrgPermissionSubjects.ProjectTemplates).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.") + }), + z.object({ + subject: z.literal(OrgPermissionSubjects.AppConnections).describe("The entity this permission pertains to."), + inverted: z.boolean().optional().describe("Whether rule allows or forbids."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionAppConnectionActions).describe( + "Describe what action an entity can take." + ), + conditions: AppConnectionConditionSchema.describe( + "When specified, only matching conditions will be allowed to access given resource." + ).optional() + }), + z.object({ + subject: z.literal(OrgPermissionSubjects.AdminConsole).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionAdminConsoleAction).describe( + "Describe what action an entity can take." + ) + }) +]); + const buildAdminPermission = () => { const { can, rules } = new AbilityBuilder>(createMongoAbility); // ws permissions @@ -125,10 +245,11 @@ const buildAdminPermission = () => { can(OrgPermissionActions.Edit, OrgPermissionSubjects.ProjectTemplates); can(OrgPermissionActions.Delete, OrgPermissionSubjects.ProjectTemplates); - can(OrgPermissionActions.Read, OrgPermissionSubjects.AppConnections); - can(OrgPermissionActions.Create, OrgPermissionSubjects.AppConnections); - can(OrgPermissionActions.Edit, OrgPermissionSubjects.AppConnections); - can(OrgPermissionActions.Delete, OrgPermissionSubjects.AppConnections); + can(OrgPermissionAppConnectionActions.Read, OrgPermissionSubjects.AppConnections); + can(OrgPermissionAppConnectionActions.Create, OrgPermissionSubjects.AppConnections); + can(OrgPermissionAppConnectionActions.Edit, OrgPermissionSubjects.AppConnections); + can(OrgPermissionAppConnectionActions.Delete, OrgPermissionSubjects.AppConnections); + can(OrgPermissionAppConnectionActions.Connect, OrgPermissionSubjects.AppConnections); can(OrgPermissionAdminConsoleAction.AccessAllProjects, OrgPermissionSubjects.AdminConsole); @@ -160,7 +281,7 @@ const buildMemberPermission = () => { can(OrgPermissionActions.Read, OrgPermissionSubjects.AuditLogs); - can(OrgPermissionActions.Read, OrgPermissionSubjects.AppConnections); + can(OrgPermissionAppConnectionActions.Connect, OrgPermissionSubjects.AppConnections); return rules; }; diff --git a/backend/src/ee/services/permission/permission-schemas.ts b/backend/src/ee/services/permission/permission-schemas.ts new file mode 100644 index 000000000..fb462e4aa --- /dev/null +++ b/backend/src/ee/services/permission/permission-schemas.ts @@ -0,0 +1,9 @@ +import { z } from "zod"; + +export const CASL_ACTION_SCHEMA_NATIVE_ENUM = (actions: ACTION) => + z + .union([z.nativeEnum(actions), z.nativeEnum(actions).array().min(1)]) + .transform((el) => (typeof el === "string" ? [el] : el)); + +export const CASL_ACTION_SCHEMA_ENUM = (actions: ACTION) => + z.union([z.enum(actions), z.enum(actions).array().min(1)]).transform((el) => (typeof el === "string" ? [el] : el)); diff --git a/backend/src/ee/services/permission/project-permission.ts b/backend/src/ee/services/permission/project-permission.ts index 47142054e..3dc7daddc 100644 --- a/backend/src/ee/services/permission/project-permission.ts +++ b/backend/src/ee/services/permission/project-permission.ts @@ -1,6 +1,10 @@ import { AbilityBuilder, createMongoAbility, ForcedSubject, MongoAbility } from "@casl/ability"; import { z } from "zod"; +import { + CASL_ACTION_SCHEMA_ENUM, + CASL_ACTION_SCHEMA_NATIVE_ENUM +} from "@app/ee/services/permission/permission-schemas"; import { conditionsMatcher, PermissionConditionOperators } from "@app/lib/casl"; import { UnpackedPermissionSchema } from "@app/server/routes/santizedSchemas/permission"; @@ -30,6 +34,16 @@ export enum ProjectPermissionDynamicSecretActions { Lease = "lease" } +export enum ProjectPermissionSecretSyncActions { + Read = "read", + Create = "create", + Edit = "edit", + Delete = "delete", + SyncSecrets = "sync-secrets", + ImportSecrets = "import-secrets", + RemoveSecrets = "remove-secrets" +} + export enum ProjectPermissionSub { Role = "role", Member = "member", @@ -60,7 +74,8 @@ export enum ProjectPermissionSub { PkiAlerts = "pki-alerts", PkiCollections = "pki-collections", Kms = "kms", - Cmek = "cmek" + Cmek = "cmek", + SecretSyncs = "secret-syncs" } export type SecretSubjectFields = { @@ -140,6 +155,7 @@ export type ProjectPermissionSet = | [ProjectPermissionActions, ProjectPermissionSub.SshCertificateTemplates] | [ProjectPermissionActions, ProjectPermissionSub.PkiAlerts] | [ProjectPermissionActions, ProjectPermissionSub.PkiCollections] + | [ProjectPermissionSecretSyncActions, ProjectPermissionSub.SecretSyncs] | [ProjectPermissionCmekActions, ProjectPermissionSub.Cmek] | [ProjectPermissionActions.Delete, ProjectPermissionSub.Project] | [ProjectPermissionActions.Edit, ProjectPermissionSub.Project] @@ -147,14 +163,6 @@ export type ProjectPermissionSet = | [ProjectPermissionActions.Create, ProjectPermissionSub.SecretRollback] | [ProjectPermissionActions.Edit, ProjectPermissionSub.Kms]; -const CASL_ACTION_SCHEMA_NATIVE_ENUM = (actions: ACTION) => - z - .union([z.nativeEnum(actions), z.nativeEnum(actions).array().min(1)]) - .transform((el) => (typeof el === "string" ? [el] : el)); - -const CASL_ACTION_SCHEMA_ENUM = (actions: ACTION) => - z.union([z.enum(actions), z.enum(actions).array().min(1)]).transform((el) => (typeof el === "string" ? [el] : el)); - // akhilmhdh: don't modify this for v2 // if you want to update create a new schema const SecretConditionV1Schema = z @@ -392,10 +400,15 @@ const GeneralPermissionSchema = [ }), z.object({ subject: z.literal(ProjectPermissionSub.Cmek).describe("The entity this permission pertains to."), - inverted: z.boolean().optional().describe("Whether rule allows or forbids."), action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionCmekActions).describe( "Describe what action an entity can take." ) + }), + z.object({ + subject: z.literal(ProjectPermissionSub.SecretSyncs).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionSecretSyncActions).describe( + "Describe what action an entity can take." + ) }) ]; @@ -549,6 +562,18 @@ const buildAdminPermissionRules = () => { ], ProjectPermissionSub.Cmek ); + can( + [ + ProjectPermissionSecretSyncActions.Create, + ProjectPermissionSecretSyncActions.Edit, + ProjectPermissionSecretSyncActions.Delete, + ProjectPermissionSecretSyncActions.Read, + ProjectPermissionSecretSyncActions.SyncSecrets, + ProjectPermissionSecretSyncActions.ImportSecrets, + ProjectPermissionSecretSyncActions.RemoveSecrets + ], + ProjectPermissionSub.SecretSyncs + ); return rules; }; @@ -713,6 +738,19 @@ const buildMemberPermissionRules = () => { ProjectPermissionSub.Cmek ); + can( + [ + ProjectPermissionSecretSyncActions.Create, + ProjectPermissionSecretSyncActions.Edit, + ProjectPermissionSecretSyncActions.Delete, + ProjectPermissionSecretSyncActions.Read, + ProjectPermissionSecretSyncActions.SyncSecrets, + ProjectPermissionSecretSyncActions.ImportSecrets, + ProjectPermissionSecretSyncActions.RemoveSecrets + ], + ProjectPermissionSub.SecretSyncs + ); + return rules; }; @@ -746,6 +784,7 @@ const buildViewerPermissionRules = () => { can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificateAuthorities); can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificates); can(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificateTemplates); + can(ProjectPermissionSecretSyncActions.Read, ProjectPermissionSub.SecretSyncs); return rules; }; diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index 723a22817..dbfdfd063 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -23,6 +23,8 @@ export const KeyStorePrefixes = { `sync-integration-mutex-${projectId}-${environmentSlug}-${secretPath}` as const, SyncSecretIntegrationLastRunTimestamp: (projectId: string, environmentSlug: string, secretPath: string) => `sync-integration-last-run-${projectId}-${environmentSlug}-${secretPath}` as const, + SecretSyncLock: (syncId: string) => `secret-sync-mutex-${syncId}` as const, + SecretSyncLastRunTimestamp: (syncId: string) => `secret-sync-last-run-${syncId}` as const, IdentityAccessTokenStatusUpdate: (identityAccessTokenId: string) => `identity-access-token-status:${identityAccessTokenId}`, ServiceTokenStatusUpdate: (serviceTokenId: string) => `service-token-status:${serviceTokenId}` @@ -30,6 +32,7 @@ export const KeyStorePrefixes = { export const KeyStoreTtls = { SetSyncSecretIntegrationLastRunTimestampInSeconds: 60, + SetSecretSyncLastRunTimestampInSeconds: 60, AccessTokenStatusUpdateInSeconds: 120 }; diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index e6fa7344a..800788179 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -1,5 +1,7 @@ import { AppConnection } from "@app/services/app-connection/app-connection-enums"; import { APP_CONNECTION_NAME_MAP } from "@app/services/app-connection/app-connection-maps"; +import { SecretSync } from "@app/services/secret-sync/secret-sync-enums"; +import { SECRET_SYNC_CONNECTION_MAP, SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps"; export const GROUPS = { CREATE: { @@ -1643,6 +1645,83 @@ export const AppConnections = { }; }, DELETE: (app: AppConnection) => ({ - connectionId: `The ID of the ${APP_CONNECTION_NAME_MAP[app]} connection to be deleted.` + connectionId: `The ID of the ${APP_CONNECTION_NAME_MAP[app]} Connection to be deleted.` }) }; + +export const SecretSyncs = { + LIST: (destination?: SecretSync) => ({ + projectId: `The ID of the project to list ${destination ? SECRET_SYNC_NAME_MAP[destination] : "Secret"} Syncs from.` + }), + GET_BY_ID: (destination: SecretSync) => ({ + syncId: `The ID of the ${SECRET_SYNC_NAME_MAP[destination]} Sync to retrieve.` + }), + GET_BY_NAME: (destination: SecretSync) => ({ + syncName: `The name of the ${SECRET_SYNC_NAME_MAP[destination]} Sync to retrieve.`, + projectId: `The ID of the project the ${SECRET_SYNC_NAME_MAP[destination]} Sync is associated with.` + }), + CREATE: (destination: SecretSync) => { + const destinationName = SECRET_SYNC_NAME_MAP[destination]; + return { + name: `The name of the ${destinationName} Sync to create. Must be slug-friendly.`, + description: `An optional description for the ${destinationName} Sync.`, + projectId: "The ID of the project to create the sync in.", + environment: `The slug of the project environment to sync secrets from.`, + secretPath: `The folder path to sync secrets from.`, + connectionId: `The ID of the ${ + APP_CONNECTION_NAME_MAP[SECRET_SYNC_CONNECTION_MAP[destination]] + } Connection to use for syncing.`, + isAutoSyncEnabled: `Whether secrets should be automatically synced when changes occur at the source location or not.`, + syncOptions: "Optional parameters to modify how secrets are synced." + }; + }, + UPDATE: (destination: SecretSync) => { + const destinationName = SECRET_SYNC_NAME_MAP[destination]; + return { + syncId: `The ID of the ${destinationName} Sync to be updated.`, + connectionId: `The updated ID of the ${ + APP_CONNECTION_NAME_MAP[SECRET_SYNC_CONNECTION_MAP[destination]] + } Connection to use for syncing.`, + name: `The updated name of the ${destinationName} Sync. Must be slug-friendly.`, + environment: `The updated slug of the project environment to sync secrets from.`, + secretPath: `The updated folder path to sync secrets from.`, + description: `The updated description of the ${destinationName} Sync.`, + isAutoSyncEnabled: `Whether secrets should be automatically synced when changes occur at the source location or not.`, + syncOptions: "Optional parameters to modify how secrets are synced." + }; + }, + DELETE: (destination: SecretSync) => ({ + syncId: `The ID of the ${SECRET_SYNC_NAME_MAP[destination]} Sync to be deleted.`, + removeSecrets: `Whether previously synced secrets should be removed prior to deletion.` + }), + SYNC_SECRETS: (destination: SecretSync) => ({ + syncId: `The ID of the ${SECRET_SYNC_NAME_MAP[destination]} Sync to trigger a sync for.` + }), + IMPORT_SECRETS: (destination: SecretSync) => ({ + syncId: `The ID of the ${SECRET_SYNC_NAME_MAP[destination]} Sync to trigger importing secrets for.`, + importBehavior: `Specify whether Infisical should prioritize secret values from Infisical or ${SECRET_SYNC_NAME_MAP[destination]}.` + }), + REMOVE_SECRETS: (destination: SecretSync) => ({ + syncId: `The ID of the ${SECRET_SYNC_NAME_MAP[destination]} Sync to trigger removing secrets for.` + }), + SYNC_OPTIONS: (destination: SecretSync) => { + const destinationName = SECRET_SYNC_NAME_MAP[destination]; + return { + INITIAL_SYNC_BEHAVIOR: `Specify how Infisical should resolve the initial sync to the ${destinationName} destination.`, + PREPEND_PREFIX: `Optionally prepend a prefix to your secrets' keys when syncing to ${destinationName}.`, + APPEND_SUFFIX: `Optionally append a suffix to your secrets' keys when syncing to ${destinationName}.` + }; + }, + DESTINATION_CONFIG: { + AWS_PARAMETER_STORE: { + REGION: "The AWS region to sync secrets to.", + PATH: "The Parameter Store path to sync secrets to." + }, + GITHUB: { + ORG: "The name of the GitHub organization.", + OWNER: "The name of the GitHub account owner of the repository.", + REPO: "The name of the GitHub repository.", + ENV: "The name of the GitHub environment." + } + } +}; diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index 330193052..f9aec5881 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -15,6 +15,12 @@ import { TIntegrationSyncPayload, TSyncSecretsDTO } from "@app/services/secret/secret-types"; +import { + TQueueSecretSyncImportSecretsByIdDTO, + TQueueSecretSyncRemoveSecretsByIdDTO, + TQueueSecretSyncSyncSecretsByIdDTO, + TQueueSendSecretSyncActionFailedNotificationsDTO +} from "@app/services/secret-sync/secret-sync-types"; export enum QueueName { SecretRotation = "secret-rotation", @@ -36,7 +42,8 @@ export enum QueueName { SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication ProjectV3Migration = "project-v3-migration", AccessTokenStatusUpdate = "access-token-status-update", - ImportSecretsFromExternalSource = "import-secrets-from-external-source" + ImportSecretsFromExternalSource = "import-secrets-from-external-source", + AppConnectionSecretSync = "app-connection-secret-sync" } export enum QueueJobs { @@ -61,7 +68,11 @@ export enum QueueJobs { ProjectV3Migration = "project-v3-migration", IdentityAccessTokenStatusUpdate = "identity-access-token-status-update", ServiceTokenStatusUpdate = "service-token-status-update", - ImportSecretsFromExternalSource = "import-secrets-from-external-source" + ImportSecretsFromExternalSource = "import-secrets-from-external-source", + SecretSyncSyncSecrets = "secret-sync-sync-secrets", + SecretSyncImportSecrets = "secret-sync-import-secrets", + SecretSyncRemoveSecrets = "secret-sync-remove-secrets", + SecretSyncSendActionFailedNotifications = "secret-sync-send-action-failed-notifications" } export type TQueueJobTypes = { @@ -184,6 +195,23 @@ export type TQueueJobTypes = { }; }; }; + [QueueName.AppConnectionSecretSync]: + | { + name: QueueJobs.SecretSyncSyncSecrets; + payload: TQueueSecretSyncSyncSecretsByIdDTO; + } + | { + name: QueueJobs.SecretSyncImportSecrets; + payload: TQueueSecretSyncImportSecretsByIdDTO; + } + | { + name: QueueJobs.SecretSyncRemoveSecrets; + payload: TQueueSecretSyncRemoveSecretsByIdDTO; + } + | { + name: QueueJobs.SecretSyncSendActionFailedNotifications; + payload: TQueueSendSecretSyncActionFailedNotificationsDTO; + }; }; export type TQueueServiceFactory = ReturnType; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 3e5575daa..442f11080 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -196,6 +196,9 @@ import { secretImportDALFactory } from "@app/services/secret-import/secret-impor import { secretImportServiceFactory } from "@app/services/secret-import/secret-import-service"; import { secretSharingDALFactory } from "@app/services/secret-sharing/secret-sharing-dal"; import { secretSharingServiceFactory } from "@app/services/secret-sharing/secret-sharing-service"; +import { secretSyncDALFactory } from "@app/services/secret-sync/secret-sync-dal"; +import { secretSyncQueueFactory } from "@app/services/secret-sync/secret-sync-queue"; +import { secretSyncServiceFactory } from "@app/services/secret-sync/secret-sync-service"; import { secretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { secretTagServiceFactory } from "@app/services/secret-tag/secret-tag-service"; import { secretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal"; @@ -318,6 +321,7 @@ export const registerRoutes = async ( const trustedIpDAL = trustedIpDALFactory(db); const telemetryDAL = telemetryDALFactory(db); const appConnectionDAL = appConnectionDALFactory(db); + const secretSyncDAL = secretSyncDALFactory(db, folderDAL); // ee db layer ops const permissionDAL = permissionDALFactory(db); @@ -824,6 +828,29 @@ export const registerRoutes = async ( kmsService }); + const secretSyncQueue = secretSyncQueueFactory({ + queueService, + secretSyncDAL, + folderDAL, + secretImportDAL, + secretV2BridgeDAL, + kmsService, + keyStore, + auditLogService, + smtpService, + projectDAL, + projectMembershipDAL, + projectBotDAL, + secretDAL, + secretBlindIndexDAL, + secretVersionDAL, + secretTagDAL, + secretVersionTagDAL, + secretVersionV2BridgeDAL, + secretVersionTagV2BridgeDAL, + resourceMetadataDAL + }); + const secretQueueService = secretQueueFactory({ keyStore, queueService, @@ -858,7 +885,8 @@ export const registerRoutes = async ( projectKeyDAL, projectUserMembershipRoleDAL, orgService, - resourceMetadataDAL + resourceMetadataDAL, + secretSyncQueue }); const projectService = projectServiceFactory({ @@ -1369,8 +1397,17 @@ export const registerRoutes = async ( const appConnectionService = appConnectionServiceFactory({ appConnectionDAL, permissionService, - kmsService, - licenseService + kmsService + }); + + const secretSyncService = secretSyncServiceFactory({ + secretSyncDAL, + permissionService, + appConnectionService, + folderDAL, + secretSyncQueue, + projectBotService, + keyStore }); await superAdminService.initServerCfg(); @@ -1470,7 +1507,8 @@ export const registerRoutes = async ( externalGroupOrgRoleMapping: externalGroupOrgRoleMappingService, projectTemplate: projectTemplateService, totp: totpService, - appConnection: appConnectionService + appConnection: appConnectionService, + secretSync: secretSyncService }); const cronJobs: CronJob[] = []; diff --git a/backend/src/server/routes/v1/app-connection-routers/apps/app-connection-endpoints.ts b/backend/src/server/routes/v1/app-connection-routers/app-connection-endpoints.ts similarity index 80% rename from backend/src/server/routes/v1/app-connection-routers/apps/app-connection-endpoints.ts rename to backend/src/server/routes/v1/app-connection-routers/app-connection-endpoints.ts index ec3b633a1..41a87feb5 100644 --- a/backend/src/server/routes/v1/app-connection-routers/apps/app-connection-endpoints.ts +++ b/backend/src/server/routes/v1/app-connection-routers/app-connection-endpoints.ts @@ -15,7 +15,7 @@ export const registerAppConnectionEndpoints = ; updateSchema: z.ZodType<{ name?: string; credentials?: I["credentials"]; description?: string | null }>; - responseSchema: z.ZodTypeAny; + sanitizedResponseSchema: z.ZodTypeAny; }) => { const appName = APP_CONNECTION_NAME_MAP[app]; @@ -39,7 +39,7 @@ export const registerAppConnectionEndpoints = { + const appConnections = await server.services.appConnection.listAvailableAppConnectionsForUser( + app, + req.permission + ); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + orgId: req.permission.orgId, + event: { + type: EventType.GET_AVAILABLE_APP_CONNECTIONS_DETAILS, + metadata: { + app, + count: appConnections.length, + connectionIds: appConnections.map((connection) => connection.id) + } + } + }); + + return { appConnections }; + } + }); + server.route({ method: "GET", url: "/:connectionId", @@ -75,7 +113,7 @@ export const registerAppConnectionEndpoints = - registerAppConnectionEndpoints({ - app: AppConnection.GitHub, - server, - responseSchema: SanitizedGitHubConnectionSchema, - createSchema: CreateGitHubConnectionSchema, - updateSchema: UpdateGitHubConnectionSchema - }); diff --git a/backend/src/server/routes/v1/app-connection-routers/apps/index.ts b/backend/src/server/routes/v1/app-connection-routers/apps/index.ts deleted file mode 100644 index b56a65f50..000000000 --- a/backend/src/server/routes/v1/app-connection-routers/apps/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { registerAwsConnectionRouter } from "@app/server/routes/v1/app-connection-routers/apps/aws-connection-router"; -import { registerGitHubConnectionRouter } from "@app/server/routes/v1/app-connection-routers/apps/github-connection-router"; -import { AppConnection } from "@app/services/app-connection/app-connection-enums"; - -export const APP_CONNECTION_REGISTER_MAP: Record Promise> = { - [AppConnection.AWS]: registerAwsConnectionRouter, - [AppConnection.GitHub]: registerGitHubConnectionRouter -}; diff --git a/backend/src/server/routes/v1/app-connection-routers/apps/aws-connection-router.ts b/backend/src/server/routes/v1/app-connection-routers/aws-connection-router.ts similarity index 90% rename from backend/src/server/routes/v1/app-connection-routers/apps/aws-connection-router.ts rename to backend/src/server/routes/v1/app-connection-routers/aws-connection-router.ts index 189ca4fbd..87ed022a2 100644 --- a/backend/src/server/routes/v1/app-connection-routers/apps/aws-connection-router.ts +++ b/backend/src/server/routes/v1/app-connection-routers/aws-connection-router.ts @@ -11,7 +11,7 @@ export const registerAwsConnectionRouter = async (server: FastifyZodProvider) => registerAppConnectionEndpoints({ app: AppConnection.AWS, server, - responseSchema: SanitizedAwsConnectionSchema, + sanitizedResponseSchema: SanitizedAwsConnectionSchema, createSchema: CreateAwsConnectionSchema, updateSchema: UpdateAwsConnectionSchema }); diff --git a/backend/src/server/routes/v1/app-connection-routers/github-connection-router.ts b/backend/src/server/routes/v1/app-connection-routers/github-connection-router.ts new file mode 100644 index 000000000..9c33f3fad --- /dev/null +++ b/backend/src/server/routes/v1/app-connection-routers/github-connection-router.ts @@ -0,0 +1,117 @@ +import { z } from "zod"; + +import { readLimit } from "@app/server/config/rateLimiter"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { AppConnection } from "@app/services/app-connection/app-connection-enums"; +import { + CreateGitHubConnectionSchema, + SanitizedGitHubConnectionSchema, + UpdateGitHubConnectionSchema +} from "@app/services/app-connection/github"; +import { AuthMode } from "@app/services/auth/auth-type"; + +import { registerAppConnectionEndpoints } from "./app-connection-endpoints"; + +export const registerGitHubConnectionRouter = async (server: FastifyZodProvider) => { + registerAppConnectionEndpoints({ + app: AppConnection.GitHub, + server, + sanitizedResponseSchema: SanitizedGitHubConnectionSchema, + createSchema: CreateGitHubConnectionSchema, + updateSchema: UpdateGitHubConnectionSchema + }); + + // The below endpoints are not exposed and for Infisical App use + + server.route({ + method: "GET", + url: `/:connectionId/repositories`, + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + connectionId: z.string().uuid() + }), + response: { + 200: z.object({ + repositories: z + .object({ id: z.number(), name: z.string(), owner: z.object({ login: z.string(), id: z.number() }) }) + .array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { connectionId } = req.params; + + const repositories = await server.services.appConnection.github.listRepositories(connectionId, req.permission); + + return { repositories }; + } + }); + + server.route({ + method: "GET", + url: `/:connectionId/organizations`, + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + connectionId: z.string().uuid() + }), + response: { + 200: z.object({ + organizations: z.object({ id: z.number(), login: z.string() }).array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { connectionId } = req.params; + + const organizations = await server.services.appConnection.github.listOrganizations(connectionId, req.permission); + + return { organizations }; + } + }); + + server.route({ + method: "GET", + url: `/:connectionId/environments`, + config: { + rateLimit: readLimit + }, + schema: { + params: z.object({ + connectionId: z.string().uuid() + }), + querystring: z.object({ + repo: z.string().min(1, "Repository name is required"), + owner: z.string().min(1, "Repository owner name is required") + }), + response: { + 200: z.object({ + environments: z.object({ id: z.number(), name: z.string() }).array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { connectionId } = req.params; + const { repo, owner } = req.query; + + const environments = await server.services.appConnection.github.listEnvironments( + { + connectionId, + repo, + owner + }, + req.permission + ); + + return { environments }; + } + }); +}; diff --git a/backend/src/server/routes/v1/app-connection-routers/index.ts b/backend/src/server/routes/v1/app-connection-routers/index.ts index 720057449..2570cb3ad 100644 --- a/backend/src/server/routes/v1/app-connection-routers/index.ts +++ b/backend/src/server/routes/v1/app-connection-routers/index.ts @@ -1,2 +1,12 @@ +import { AppConnection } from "@app/services/app-connection/app-connection-enums"; + +import { registerAwsConnectionRouter } from "./aws-connection-router"; +import { registerGitHubConnectionRouter } from "./github-connection-router"; + export * from "./app-connection-router"; -export * from "./apps"; + +export const APP_CONNECTION_REGISTER_ROUTER_MAP: Record Promise> = + { + [AppConnection.AWS]: registerAwsConnectionRouter, + [AppConnection.GitHub]: registerGitHubConnectionRouter + }; diff --git a/backend/src/server/routes/v1/index.ts b/backend/src/server/routes/v1/index.ts index 7fae1d1f9..e3f6c7f2e 100644 --- a/backend/src/server/routes/v1/index.ts +++ b/backend/src/server/routes/v1/index.ts @@ -1,6 +1,10 @@ -import { APP_CONNECTION_REGISTER_MAP, registerAppConnectionRouter } from "@app/server/routes/v1/app-connection-routers"; +import { + APP_CONNECTION_REGISTER_ROUTER_MAP, + registerAppConnectionRouter +} from "@app/server/routes/v1/app-connection-routers"; import { registerCmekRouter } from "@app/server/routes/v1/cmek-router"; import { registerDashboardRouter } from "@app/server/routes/v1/dashboard-router"; +import { registerSecretSyncRouter, SECRET_SYNC_REGISTER_ROUTER_MAP } from "@app/server/routes/v1/secret-sync-routers"; import { registerAdminRouter } from "./admin-router"; import { registerAuthRoutes } from "./auth-router"; @@ -113,12 +117,28 @@ export const registerV1Routes = async (server: FastifyZodProvider) => { await server.register(registerExternalGroupOrgRoleMappingRouter, { prefix: "/external-group-mappings" }); await server.register( - async (appConnectionsRouter) => { - await appConnectionsRouter.register(registerAppConnectionRouter); - for await (const [app, router] of Object.entries(APP_CONNECTION_REGISTER_MAP)) { - await appConnectionsRouter.register(router, { prefix: `/${app}` }); + async (appConnectionRouter) => { + // register generic app connection endpoints + await appConnectionRouter.register(registerAppConnectionRouter); + + // register service specific endpoints (app-connections/aws, app-connections/github, etc.) + for await (const [app, router] of Object.entries(APP_CONNECTION_REGISTER_ROUTER_MAP)) { + await appConnectionRouter.register(router, { prefix: `/${app}` }); } }, { prefix: "/app-connections" } ); + + await server.register( + async (secretSyncRouter) => { + // register generic secret sync endpoints + await secretSyncRouter.register(registerSecretSyncRouter); + + // register service specific secret sync endpoints (secret-syncs/aws-parameter-store, secret-syncs/github, etc.) + for await (const [destination, router] of Object.entries(SECRET_SYNC_REGISTER_ROUTER_MAP)) { + await secretSyncRouter.register(router, { prefix: `/${destination}` }); + } + }, + { prefix: "/secret-syncs" } + ); }; diff --git a/backend/src/server/routes/v1/secret-sync-routers/aws-parameter-store-sync-router.ts b/backend/src/server/routes/v1/secret-sync-routers/aws-parameter-store-sync-router.ts new file mode 100644 index 000000000..8f02b9e6e --- /dev/null +++ b/backend/src/server/routes/v1/secret-sync-routers/aws-parameter-store-sync-router.ts @@ -0,0 +1,17 @@ +import { + AwsParameterStoreSyncSchema, + CreateAwsParameterStoreSyncSchema, + UpdateAwsParameterStoreSyncSchema +} from "@app/services/secret-sync/aws-parameter-store"; +import { SecretSync } from "@app/services/secret-sync/secret-sync-enums"; + +import { registerSyncSecretsEndpoints } from "./secret-sync-endpoints"; + +export const registerAwsParameterStoreSyncRouter = async (server: FastifyZodProvider) => + registerSyncSecretsEndpoints({ + destination: SecretSync.AWSParameterStore, + server, + responseSchema: AwsParameterStoreSyncSchema, + createSchema: CreateAwsParameterStoreSyncSchema, + updateSchema: UpdateAwsParameterStoreSyncSchema + }); diff --git a/backend/src/server/routes/v1/secret-sync-routers/github-sync-router.ts b/backend/src/server/routes/v1/secret-sync-routers/github-sync-router.ts new file mode 100644 index 000000000..a84d70354 --- /dev/null +++ b/backend/src/server/routes/v1/secret-sync-routers/github-sync-router.ts @@ -0,0 +1,13 @@ +import { CreateGitHubSyncSchema, GitHubSyncSchema, UpdateGitHubSyncSchema } from "@app/services/secret-sync/github"; +import { SecretSync } from "@app/services/secret-sync/secret-sync-enums"; + +import { registerSyncSecretsEndpoints } from "./secret-sync-endpoints"; + +export const registerGitHubSyncRouter = async (server: FastifyZodProvider) => + registerSyncSecretsEndpoints({ + destination: SecretSync.GitHub, + server, + responseSchema: GitHubSyncSchema, + createSchema: CreateGitHubSyncSchema, + updateSchema: UpdateGitHubSyncSchema + }); diff --git a/backend/src/server/routes/v1/secret-sync-routers/index.ts b/backend/src/server/routes/v1/secret-sync-routers/index.ts new file mode 100644 index 000000000..ecc21b776 --- /dev/null +++ b/backend/src/server/routes/v1/secret-sync-routers/index.ts @@ -0,0 +1,11 @@ +import { SecretSync } from "@app/services/secret-sync/secret-sync-enums"; + +import { registerAwsParameterStoreSyncRouter } from "./aws-parameter-store-sync-router"; +import { registerGitHubSyncRouter } from "./github-sync-router"; + +export * from "./secret-sync-router"; + +export const SECRET_SYNC_REGISTER_ROUTER_MAP: Record Promise> = { + [SecretSync.AWSParameterStore]: registerAwsParameterStoreSyncRouter, + [SecretSync.GitHub]: registerGitHubSyncRouter +}; diff --git a/backend/src/server/routes/v1/secret-sync-routers/secret-sync-endpoints.ts b/backend/src/server/routes/v1/secret-sync-routers/secret-sync-endpoints.ts new file mode 100644 index 000000000..31826f86f --- /dev/null +++ b/backend/src/server/routes/v1/secret-sync-routers/secret-sync-endpoints.ts @@ -0,0 +1,408 @@ +import { z } from "zod"; + +import { EventType } from "@app/ee/services/audit-log/audit-log-types"; +import { SecretSyncs } from "@app/lib/api-docs"; +import { startsWithVowel } from "@app/lib/fn"; +import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { AuthMode } from "@app/services/auth/auth-type"; +import { SecretSync, SecretSyncImportBehavior } from "@app/services/secret-sync/secret-sync-enums"; +import { SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps"; +import { TSecretSync, TSecretSyncInput } from "@app/services/secret-sync/secret-sync-types"; + +export const registerSyncSecretsEndpoints = ({ + server, + destination, + createSchema, + updateSchema, + responseSchema +}: { + destination: SecretSync; + server: FastifyZodProvider; + createSchema: z.ZodType<{ + name: string; + environment: string; + secretPath: string; + projectId: string; + connectionId: string; + destinationConfig: I["destinationConfig"]; + syncOptions: I["syncOptions"]; + description?: string | null; + isAutoSyncEnabled?: boolean; + }>; + updateSchema: z.ZodType<{ + connectionId?: string; + name?: string; + environment?: string; + secretPath?: string; + destinationConfig?: I["destinationConfig"]; + syncOptions?: I["syncOptions"]; + description?: string | null; + isAutoSyncEnabled?: boolean; + }>; + responseSchema: z.ZodTypeAny; +}) => { + const destinationName = SECRET_SYNC_NAME_MAP[destination]; + + server.route({ + method: "GET", + url: `/`, + config: { + rateLimit: readLimit + }, + schema: { + description: `List the ${destinationName} Syncs for the specified project.`, + querystring: z.object({ + projectId: z.string().trim().min(1, "Project ID required").describe(SecretSyncs.LIST(destination).projectId) + }), + response: { + 200: z.object({ secretSyncs: responseSchema.array() }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { + query: { projectId } + } = req; + + const secretSyncs = (await server.services.secretSync.listSecretSyncsByProjectId( + { projectId, destination }, + req.permission + )) as T[]; + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId, + event: { + type: EventType.GET_SECRET_SYNCS, + metadata: { + destination, + count: secretSyncs.length, + syncIds: secretSyncs.map((connection) => connection.id) + } + } + }); + + return { secretSyncs }; + } + }); + + server.route({ + method: "GET", + url: "/:syncId", + config: { + rateLimit: readLimit + }, + schema: { + description: `Get the specified ${destinationName} Sync by ID.`, + params: z.object({ + syncId: z.string().uuid().describe(SecretSyncs.GET_BY_ID(destination).syncId) + }), + response: { + 200: z.object({ secretSync: responseSchema }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { syncId } = req.params; + + const secretSync = (await server.services.secretSync.findSecretSyncById( + { syncId, destination }, + req.permission + )) as T; + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: secretSync.projectId, + event: { + type: EventType.GET_SECRET_SYNC, + metadata: { + syncId, + destination + } + } + }); + + return { secretSync }; + } + }); + + server.route({ + method: "GET", + url: `/sync-name/:syncName`, + config: { + rateLimit: readLimit + }, + schema: { + description: `Get the specified ${destinationName} Sync by name and project ID.`, + params: z.object({ + syncName: z.string().trim().min(1, "Sync name required").describe(SecretSyncs.GET_BY_NAME(destination).syncName) + }), + querystring: z.object({ + projectId: z + .string() + .trim() + .min(1, "Project ID required") + .describe(SecretSyncs.GET_BY_NAME(destination).projectId) + }), + response: { + 200: z.object({ secretSync: responseSchema }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { syncName } = req.params; + const { projectId } = req.query; + + const secretSync = (await server.services.secretSync.findSecretSyncByName( + { syncName, projectId, destination }, + req.permission + )) as T; + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId, + event: { + type: EventType.GET_SECRET_SYNC, + metadata: { + syncId: secretSync.id, + destination + } + } + }); + + return { secretSync }; + } + }); + + server.route({ + method: "POST", + url: "/", + config: { + rateLimit: writeLimit + }, + schema: { + description: `Create ${ + startsWithVowel(destinationName) ? "an" : "a" + } ${destinationName} Sync for the specified project environment.`, + body: createSchema, + response: { + 200: z.object({ secretSync: responseSchema }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const secretSync = (await server.services.secretSync.createSecretSync( + { ...req.body, destination }, + req.permission + )) as T; + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: secretSync.projectId, + event: { + type: EventType.CREATE_SECRET_SYNC, + metadata: { + syncId: secretSync.id, + destination, + ...req.body + } + } + }); + + return { secretSync }; + } + }); + + server.route({ + method: "PATCH", + url: "/:syncId", + config: { + rateLimit: writeLimit + }, + schema: { + description: `Update the specified ${destinationName} Sync.`, + params: z.object({ + syncId: z.string().uuid().describe(SecretSyncs.UPDATE(destination).syncId) + }), + body: updateSchema, + response: { + 200: z.object({ secretSync: responseSchema }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { syncId } = req.params; + + const secretSync = (await server.services.secretSync.updateSecretSync( + { ...req.body, syncId, destination }, + req.permission + )) as T; + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: secretSync.projectId, + event: { + type: EventType.UPDATE_SECRET_SYNC, + metadata: { + syncId, + destination, + ...req.body + } + } + }); + + return { secretSync }; + } + }); + + server.route({ + method: "DELETE", + url: `/:syncId`, + config: { + rateLimit: writeLimit + }, + schema: { + description: `Delete the specified ${destinationName} Sync.`, + params: z.object({ + syncId: z.string().uuid().describe(SecretSyncs.DELETE(destination).syncId) + }), + querystring: z.object({ + removeSecrets: z + .enum(["true", "false"]) + .default("false") + .transform((value) => value === "true") + .describe(SecretSyncs.DELETE(destination).removeSecrets) + }), + response: { + 200: z.object({ secretSync: responseSchema }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { syncId } = req.params; + const { removeSecrets } = req.query; + + const secretSync = (await server.services.secretSync.deleteSecretSync( + { destination, syncId, removeSecrets }, + req.permission + )) as T; + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + orgId: req.permission.orgId, + event: { + type: EventType.DELETE_SECRET_SYNC, + metadata: { + destination, + syncId, + removeSecrets + } + } + }); + + return { secretSync }; + } + }); + + server.route({ + method: "POST", + url: "/:syncId/sync-secrets", + config: { + rateLimit: writeLimit + }, + schema: { + description: `Trigger a sync for the specified ${destinationName} Sync.`, + params: z.object({ + syncId: z.string().uuid().describe(SecretSyncs.SYNC_SECRETS(destination).syncId) + }), + response: { + 200: z.object({ secretSync: responseSchema }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { syncId } = req.params; + + const secretSync = (await server.services.secretSync.triggerSecretSyncSyncSecretsById( + { + syncId, + destination, + auditLogInfo: req.auditLogInfo + }, + req.permission + )) as T; + + return { secretSync }; + } + }); + + server.route({ + method: "POST", + url: "/:syncId/import-secrets", + config: { + rateLimit: writeLimit + }, + schema: { + description: `Import secrets from the specified ${destinationName} Sync destination.`, + params: z.object({ + syncId: z.string().uuid().describe(SecretSyncs.IMPORT_SECRETS(destination).syncId) + }), + querystring: z.object({ + importBehavior: z + .nativeEnum(SecretSyncImportBehavior) + .describe(SecretSyncs.IMPORT_SECRETS(destination).importBehavior) + }), + response: { + 200: z.object({ secretSync: responseSchema }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { syncId } = req.params; + const { importBehavior } = req.query; + + const secretSync = (await server.services.secretSync.triggerSecretSyncImportSecretsById( + { + syncId, + destination, + importBehavior + }, + req.permission + )) as T; + + return { secretSync }; + } + }); + + server.route({ + method: "POST", + url: "/:syncId/remove-secrets", + config: { + rateLimit: writeLimit + }, + schema: { + description: `Remove previously synced secrets from the specified ${destinationName} Sync destination.`, + params: z.object({ + syncId: z.string().uuid().describe(SecretSyncs.REMOVE_SECRETS(destination).syncId) + }), + response: { + 200: z.object({ secretSync: responseSchema }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { syncId } = req.params; + + const secretSync = (await server.services.secretSync.triggerSecretSyncRemoveSecretsById( + { + syncId, + destination + }, + req.permission + )) as T; + + return { secretSync }; + } + }); +}; diff --git a/backend/src/server/routes/v1/secret-sync-routers/secret-sync-router.ts b/backend/src/server/routes/v1/secret-sync-routers/secret-sync-router.ts new file mode 100644 index 000000000..5736767dd --- /dev/null +++ b/backend/src/server/routes/v1/secret-sync-routers/secret-sync-router.ts @@ -0,0 +1,82 @@ +import { z } from "zod"; + +import { EventType } from "@app/ee/services/audit-log/audit-log-types"; +import { SecretSyncs } from "@app/lib/api-docs"; +import { readLimit } from "@app/server/config/rateLimiter"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { AuthMode } from "@app/services/auth/auth-type"; +import { + AwsParameterStoreSyncListItemSchema, + AwsParameterStoreSyncSchema +} from "@app/services/secret-sync/aws-parameter-store"; +import { GitHubSyncListItemSchema, GitHubSyncSchema } from "@app/services/secret-sync/github"; + +const SecretSyncSchema = z.discriminatedUnion("destination", [AwsParameterStoreSyncSchema, GitHubSyncSchema]); + +const SecretSyncOptionsSchema = z.discriminatedUnion("destination", [ + AwsParameterStoreSyncListItemSchema, + GitHubSyncListItemSchema +]); + +export const registerSecretSyncRouter = async (server: FastifyZodProvider) => { + server.route({ + method: "GET", + url: "/options", + config: { + rateLimit: readLimit + }, + schema: { + description: "List the available Secret Sync Options.", + response: { + 200: z.object({ + secretSyncOptions: SecretSyncOptionsSchema.array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: () => { + const secretSyncOptions = server.services.secretSync.listSecretSyncOptions(); + return { secretSyncOptions }; + } + }); + + server.route({ + method: "GET", + url: "/", + config: { + rateLimit: readLimit + }, + schema: { + description: "List all the Secret Syncs for the specified project.", + querystring: z.object({ + projectId: z.string().trim().min(1, "Project ID required").describe(SecretSyncs.LIST().projectId) + }), + response: { + 200: z.object({ secretSyncs: SecretSyncSchema.array() }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const { + query: { projectId }, + permission + } = req; + + const secretSyncs = await server.services.secretSync.listSecretSyncsByProjectId({ projectId }, permission); + + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId, + event: { + type: EventType.GET_SECRET_SYNCS, + metadata: { + syncIds: secretSyncs.map((sync) => sync.id), + count: secretSyncs.length + } + } + }); + + return { secretSyncs }; + } + }); +}; diff --git a/backend/src/services/app-connection/app-connection-enums.ts b/backend/src/services/app-connection/app-connection-enums.ts index d69b7dec1..e96886e9f 100644 --- a/backend/src/services/app-connection/app-connection-enums.ts +++ b/backend/src/services/app-connection/app-connection-enums.ts @@ -2,3 +2,50 @@ export enum AppConnection { GitHub = "github", AWS = "aws" } + +export enum AWSRegion { + // US + US_EAST_1 = "us-east-1", // N. Virginia + US_EAST_2 = "us-east-2", // Ohio + US_WEST_1 = "us-west-1", // N. California + US_WEST_2 = "us-west-2", // Oregon + + // GovCloud + US_GOV_EAST_1 = "us-gov-east-1", // US-East + US_GOV_WEST_1 = "us-gov-west-1", // US-West + + // Africa + AF_SOUTH_1 = "af-south-1", // Cape Town + + // Asia Pacific + AP_EAST_1 = "ap-east-1", // Hong Kong + AP_SOUTH_1 = "ap-south-1", // Mumbai + AP_SOUTH_2 = "ap-south-2", // Hyderabad + AP_NORTHEAST_1 = "ap-northeast-1", // Tokyo + AP_NORTHEAST_2 = "ap-northeast-2", // Seoul + AP_NORTHEAST_3 = "ap-northeast-3", // Osaka + AP_SOUTHEAST_1 = "ap-southeast-1", // Singapore + AP_SOUTHEAST_2 = "ap-southeast-2", // Sydney + AP_SOUTHEAST_3 = "ap-southeast-3", // Jakarta + AP_SOUTHEAST_4 = "ap-southeast-4", // Melbourne + + // Canada + CA_CENTRAL_1 = "ca-central-1", // Central + + // Europe + EU_CENTRAL_1 = "eu-central-1", // Frankfurt + EU_CENTRAL_2 = "eu-central-2", // Zurich + EU_WEST_1 = "eu-west-1", // Ireland + EU_WEST_2 = "eu-west-2", // London + EU_WEST_3 = "eu-west-3", // Paris + EU_SOUTH_1 = "eu-south-1", // Milan + EU_SOUTH_2 = "eu-south-2", // Spain + EU_NORTH_1 = "eu-north-1", // Stockholm + + // Middle East + ME_SOUTH_1 = "me-south-1", // Bahrain + ME_CENTRAL_1 = "me-central-1", // UAE + + // South America + SA_EAST_1 = "sa-east-1" // Sao Paulo +} diff --git a/backend/src/services/app-connection/app-connection-fns.ts b/backend/src/services/app-connection/app-connection-fns.ts index 787839cf7..3f52b1285 100644 --- a/backend/src/services/app-connection/app-connection-fns.ts +++ b/backend/src/services/app-connection/app-connection-fns.ts @@ -1,3 +1,4 @@ +import { TAppConnections } from "@app/db/schemas/app-connections"; import { AppConnection } from "@app/services/app-connection/app-connection-enums"; import { TAppConnectionServiceFactoryDep } from "@app/services/app-connection/app-connection-service"; import { TAppConnection, TAppConnectionConfig } from "@app/services/app-connection/app-connection-types"; @@ -64,9 +65,8 @@ export const validateAppConnectionCredentials = async ( ): Promise => { const { app } = appConnection; switch (app) { - case AppConnection.AWS: { + case AppConnection.AWS: return validateAwsConnectionCredentials(appConnection); - } case AppConnection.GitHub: return validateGitHubConnectionCredentials(appConnection); default: @@ -90,3 +90,17 @@ export const getAppConnectionMethodName = (method: TAppConnection["method"]) => throw new Error(`Unhandled App Connection Method: ${method}`); } }; + +export const decryptAppConnection = async ( + appConnection: TAppConnections, + kmsService: TAppConnectionServiceFactoryDep["kmsService"] +) => { + return { + ...appConnection, + credentials: await decryptAppConnectionCredentials({ + encryptedCredentials: appConnection.encryptedCredentials, + orgId: appConnection.orgId, + kmsService + }) + } as TAppConnection; +}; diff --git a/backend/src/services/app-connection/app-connection-service.ts b/backend/src/services/app-connection/app-connection-service.ts index 9b9f16626..91e7d9dc4 100644 --- a/backend/src/services/app-connection/app-connection-service.ts +++ b/backend/src/services/app-connection/app-connection-service.ts @@ -1,13 +1,12 @@ -import { ForbiddenError } from "@casl/ability"; +import { ForbiddenError, subject } from "@casl/ability"; -import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; -import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission"; +import { OrgPermissionAppConnectionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; -import { BadRequestError, NotFoundError } from "@app/lib/errors"; +import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; import { DiscriminativePick, OrgServiceActor } from "@app/lib/types"; import { AppConnection } from "@app/services/app-connection/app-connection-enums"; import { - decryptAppConnectionCredentials, + decryptAppConnection, encryptAppConnectionCredentials, getAppConnectionMethodName, listAppConnectionOptions, @@ -23,6 +22,7 @@ import { } from "@app/services/app-connection/app-connection-types"; import { ValidateAwsConnectionCredentialsSchema } from "@app/services/app-connection/aws"; import { ValidateGitHubConnectionCredentialsSchema } from "@app/services/app-connection/github"; +import { githubConnectionService } from "@app/services/app-connection/github/github-connection-service"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { TAppConnectionDALFactory } from "./app-connection-dal"; @@ -31,7 +31,6 @@ export type TAppConnectionServiceFactoryDep = { appConnectionDAL: TAppConnectionDALFactory; permissionService: Pick; kmsService: Pick; - licenseService: Pick; // TODO: remove once launched }; export type TAppConnectionServiceFactory = ReturnType; @@ -44,19 +43,9 @@ const VALIDATE_APP_CONNECTION_CREDENTIALS_MAP: Record { - // app connections are disabled for public until launch - const checkAppServicesAvailability = async (orgId: string) => { - const subscription = await licenseService.getPlan(orgId); - - if (!subscription.appConnections) throw new BadRequestError({ message: "App Connections are not available yet." }); - }; - const listAppConnectionsByOrg = async (actor: OrgServiceActor, app?: AppConnection) => { - await checkAppServicesAvailability(actor.orgId); - const { permission } = await permissionService.getOrgPermission( actor.type, actor.id, @@ -65,7 +54,10 @@ export const appConnectionServiceFactory = ({ actor.orgId ); - ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.AppConnections); + ForbiddenError.from(permission).throwUnlessCan( + OrgPermissionAppConnectionActions.Read, + OrgPermissionSubjects.AppConnections + ); const appConnections = await appConnectionDAL.find( app @@ -78,24 +70,11 @@ export const appConnectionServiceFactory = ({ return Promise.all( appConnections .sort((a, b) => a.name.toLowerCase().localeCompare(b.name.toLowerCase())) - .map(async ({ encryptedCredentials, ...connection }) => { - const credentials = await decryptAppConnectionCredentials({ - encryptedCredentials, - kmsService, - orgId: connection.orgId - }); - - return { - ...connection, - credentials - } as TAppConnection; - }) + .map((appConnection) => decryptAppConnection(appConnection, kmsService)) ); }; const findAppConnectionById = async (app: AppConnection, connectionId: string, actor: OrgServiceActor) => { - await checkAppServicesAvailability(actor.orgId); - const appConnection = await appConnectionDAL.findById(connectionId); if (!appConnection) throw new NotFoundError({ message: `Could not find App Connection with ID ${connectionId}` }); @@ -108,24 +87,18 @@ export const appConnectionServiceFactory = ({ appConnection.orgId ); - ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.AppConnections); + ForbiddenError.from(permission).throwUnlessCan( + OrgPermissionAppConnectionActions.Read, + OrgPermissionSubjects.AppConnections + ); if (appConnection.app !== app) throw new BadRequestError({ message: `App Connection with ID ${connectionId} is not for App "${app}"` }); - return { - ...appConnection, - credentials: await decryptAppConnectionCredentials({ - encryptedCredentials: appConnection.encryptedCredentials, - orgId: appConnection.orgId, - kmsService - }) - } as TAppConnection; + return decryptAppConnection(appConnection, kmsService); }; const findAppConnectionByName = async (app: AppConnection, connectionName: string, actor: OrgServiceActor) => { - await checkAppServicesAvailability(actor.orgId); - const appConnection = await appConnectionDAL.findOne({ name: connectionName, orgId: actor.orgId }); if (!appConnection) @@ -139,27 +112,21 @@ export const appConnectionServiceFactory = ({ appConnection.orgId ); - ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.AppConnections); + ForbiddenError.from(permission).throwUnlessCan( + OrgPermissionAppConnectionActions.Read, + OrgPermissionSubjects.AppConnections + ); if (appConnection.app !== app) throw new BadRequestError({ message: `App Connection with name ${connectionName} is not for App "${app}"` }); - return { - ...appConnection, - credentials: await decryptAppConnectionCredentials({ - encryptedCredentials: appConnection.encryptedCredentials, - orgId: appConnection.orgId, - kmsService - }) - } as TAppConnection; + return decryptAppConnection(appConnection, kmsService); }; const createAppConnection = async ( { method, app, credentials, ...params }: TCreateAppConnectionDTO, actor: OrgServiceActor ) => { - await checkAppServicesAvailability(actor.orgId); - const { permission } = await permissionService.getOrgPermission( actor.type, actor.id, @@ -168,7 +135,10 @@ export const appConnectionServiceFactory = ({ actor.orgId ); - ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.AppConnections); + ForbiddenError.from(permission).throwUnlessCan( + OrgPermissionAppConnectionActions.Create, + OrgPermissionSubjects.AppConnections + ); const appConnection = await appConnectionDAL.transaction(async (tx) => { const isConflictingName = Boolean( @@ -216,15 +186,13 @@ export const appConnectionServiceFactory = ({ }; }); - return appConnection; + return appConnection as TAppConnection; }; const updateAppConnection = async ( { connectionId, credentials, ...params }: TUpdateAppConnectionDTO, actor: OrgServiceActor ) => { - await checkAppServicesAvailability(actor.orgId); - const appConnection = await appConnectionDAL.findById(connectionId); if (!appConnection) throw new NotFoundError({ message: `Could not find App Connection with ID ${connectionId}` }); @@ -237,7 +205,10 @@ export const appConnectionServiceFactory = ({ appConnection.orgId ); - ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.AppConnections); + ForbiddenError.from(permission).throwUnlessCan( + OrgPermissionAppConnectionActions.Edit, + OrgPermissionSubjects.AppConnections + ); const updatedAppConnection = await appConnectionDAL.transaction(async (tx) => { if (params.name && appConnection.name !== params.name) { @@ -304,19 +275,10 @@ export const appConnectionServiceFactory = ({ return updatedConnection; }); - return { - ...updatedAppConnection, - credentials: await decryptAppConnectionCredentials({ - encryptedCredentials: updatedAppConnection.encryptedCredentials, - orgId: updatedAppConnection.orgId, - kmsService - }) - } as TAppConnection; + return decryptAppConnection(updatedAppConnection, kmsService); }; const deleteAppConnection = async (app: AppConnection, connectionId: string, actor: OrgServiceActor) => { - await checkAppServicesAvailability(actor.orgId); - const appConnection = await appConnectionDAL.findById(connectionId); if (!appConnection) throw new NotFoundError({ message: `Could not find App Connection with ID ${connectionId}` }); @@ -329,23 +291,85 @@ export const appConnectionServiceFactory = ({ appConnection.orgId ); - ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Delete, OrgPermissionSubjects.AppConnections); + ForbiddenError.from(permission).throwUnlessCan( + OrgPermissionAppConnectionActions.Delete, + OrgPermissionSubjects.AppConnections + ); if (appConnection.app !== app) throw new BadRequestError({ message: `App Connection with ID ${connectionId} is not for App "${app}"` }); - // TODO: specify delete error message if due to existing dependencies + // TODO (scott): add option to delete all dependencies - const deletedAppConnection = await appConnectionDAL.deleteById(connectionId); + try { + const deletedAppConnection = await appConnectionDAL.deleteById(connectionId); - return { - ...deletedAppConnection, - credentials: await decryptAppConnectionCredentials({ - encryptedCredentials: deletedAppConnection.encryptedCredentials, - orgId: deletedAppConnection.orgId, - kmsService - }) - } as TAppConnection; + return await decryptAppConnection(deletedAppConnection, kmsService); + } catch (err) { + if (err instanceof DatabaseError && (err.error as { code: string })?.code === "23503") { + throw new BadRequestError({ + message: + "Cannot delete App Connection with existing connections. Remove all existing connections and try again." + }); + } + + throw err; + } + }; + + const connectAppConnectionById = async ( + app: AppConnection, + connectionId: string, + actor: OrgServiceActor + ) => { + const appConnection = await appConnectionDAL.findById(connectionId); + + if (!appConnection) throw new NotFoundError({ message: `Could not find App Connection with ID ${connectionId}` }); + + const { permission: orgPermission } = await permissionService.getOrgPermission( + actor.type, + actor.id, + appConnection.orgId, + actor.authMethod, + actor.orgId + ); + + ForbiddenError.from(orgPermission).throwUnlessCan( + OrgPermissionAppConnectionActions.Connect, + subject(OrgPermissionSubjects.AppConnections, { connectionId: appConnection.id }) + ); + + if (appConnection.app !== app) + throw new BadRequestError({ + message: `${ + APP_CONNECTION_NAME_MAP[appConnection.app as AppConnection] + } Connection with ID ${connectionId} cannot be used to connect to ${APP_CONNECTION_NAME_MAP[app]}` + }); + + const connection = await decryptAppConnection(appConnection, kmsService); + + return connection as T; + }; + + const listAvailableAppConnectionsForUser = async (app: AppConnection, actor: OrgServiceActor) => { + const { permission: orgPermission } = await permissionService.getOrgPermission( + actor.type, + actor.id, + actor.orgId, + actor.authMethod, + actor.orgId + ); + + const appConnections = await appConnectionDAL.find({ app, orgId: actor.orgId }); + + const availableConnections = appConnections.filter((connection) => + orgPermission.can( + OrgPermissionAppConnectionActions.Connect, + subject(OrgPermissionSubjects.AppConnections, { connectionId: connection.id }) + ) + ); + + return availableConnections as Omit[]; }; return { @@ -355,6 +379,9 @@ export const appConnectionServiceFactory = ({ findAppConnectionByName, createAppConnection, updateAppConnection, - deleteAppConnection + deleteAppConnection, + connectAppConnectionById, + listAvailableAppConnectionsForUser, + github: githubConnectionService(connectAppConnectionById) }; }; diff --git a/backend/src/services/app-connection/aws/aws-connection-fns.ts b/backend/src/services/app-connection/aws/aws-connection-fns.ts index 36008bc58..d81421529 100644 --- a/backend/src/services/app-connection/aws/aws-connection-fns.ts +++ b/backend/src/services/app-connection/aws/aws-connection-fns.ts @@ -4,7 +4,7 @@ import { randomUUID } from "crypto"; import { getConfig } from "@app/lib/config/env"; import { BadRequestError, InternalServerError } from "@app/lib/errors"; -import { AppConnection } from "@app/services/app-connection/app-connection-enums"; +import { AppConnection, AWSRegion } from "@app/services/app-connection/app-connection-enums"; import { AwsConnectionMethod } from "./aws-connection-enums"; import { TAwsConnectionConfig } from "./aws-connection-types"; @@ -20,7 +20,7 @@ export const getAwsAppConnectionListItem = () => { }; }; -export const getAwsConnectionConfig = async (appConnection: TAwsConnectionConfig, region = "us-east-1") => { +export const getAwsConnectionConfig = async (appConnection: TAwsConnectionConfig, region = AWSRegion.US_EAST_1) => { const appCfg = getConfig(); let accessKeyId: string; diff --git a/backend/src/services/app-connection/aws/aws-connection-schemas.ts b/backend/src/services/app-connection/aws/aws-connection-schemas.ts index 914e92671..c06c6f0ed 100644 --- a/backend/src/services/app-connection/aws/aws-connection-schemas.ts +++ b/backend/src/services/app-connection/aws/aws-connection-schemas.ts @@ -38,11 +38,11 @@ export const AwsConnectionSchema = z.intersection( export const SanitizedAwsConnectionSchema = z.discriminatedUnion("method", [ BaseAwsConnectionSchema.extend({ method: z.literal(AwsConnectionMethod.AssumeRole), - credentials: AwsConnectionAssumeRoleCredentialsSchema.omit({ roleArn: true }) + credentials: AwsConnectionAssumeRoleCredentialsSchema.pick({}) }), BaseAwsConnectionSchema.extend({ method: z.literal(AwsConnectionMethod.AccessKey), - credentials: AwsConnectionAccessTokenCredentialsSchema.omit({ secretAccessKey: true }) + credentials: AwsConnectionAccessTokenCredentialsSchema.pick({ accessKeyId: true }) }) ]); @@ -75,7 +75,7 @@ export const UpdateAwsConnectionSchema = z export const AwsConnectionListItemSchema = z.object({ name: z.literal("AWS"), app: z.literal(AppConnection.AWS), - // the below is preferable but currently breaks mintlify + // the below is preferable but currently breaks with our zod to json schema parser // methods: z.tuple([z.literal(AwsConnectionMethod.AssumeRole), z.literal(AwsConnectionMethod.AccessKey)]), methods: z.nativeEnum(AwsConnectionMethod).array(), accessKeyId: z.string().optional() diff --git a/backend/src/services/app-connection/github/github-connection-fns.ts b/backend/src/services/app-connection/github/github-connection-fns.ts index 01fa7846f..391ba5f96 100644 --- a/backend/src/services/app-connection/github/github-connection-fns.ts +++ b/backend/src/services/app-connection/github/github-connection-fns.ts @@ -1,3 +1,5 @@ +import { createAppAuth } from "@octokit/auth-app"; +import { Octokit } from "@octokit/rest"; import { AxiosResponse } from "axios"; import { getConfig } from "@app/lib/config/env"; @@ -8,7 +10,7 @@ import { IntegrationUrls } from "@app/services/integration-auth/integration-list import { AppConnection } from "../app-connection-enums"; import { GitHubConnectionMethod } from "./github-connection-enums"; -import { TGitHubConnectionConfig } from "./github-connection-types"; +import { TGitHubConnection, TGitHubConnectionConfig } from "./github-connection-types"; export const getGitHubConnectionListItem = () => { const { INF_APP_CONNECTION_GITHUB_OAUTH_CLIENT_ID, INF_APP_CONNECTION_GITHUB_APP_SLUG } = getConfig(); @@ -22,10 +24,131 @@ export const getGitHubConnectionListItem = () => { }; }; +export const getGitHubClient = (appConnection: TGitHubConnection) => { + const appCfg = getConfig(); + + const { method, credentials } = appConnection; + + let client: Octokit; + + switch (method) { + case GitHubConnectionMethod.App: + if (!appCfg.INF_APP_CONNECTION_GITHUB_APP_ID || !appCfg.INF_APP_CONNECTION_GITHUB_APP_PRIVATE_KEY) { + throw new InternalServerError({ + message: `GitHub ${getAppConnectionMethodName(method).replace( + "GitHub", + "" + )} environment variables have not been configured` + }); + } + + client = new Octokit({ + authStrategy: createAppAuth, + auth: { + appId: appCfg.INF_APP_CONNECTION_GITHUB_APP_ID, + privateKey: appCfg.INF_APP_CONNECTION_GITHUB_APP_PRIVATE_KEY, + installationId: credentials.installationId + } + }); + break; + case GitHubConnectionMethod.OAuth: + client = new Octokit({ + auth: credentials.accessToken + }); + break; + default: + throw new InternalServerError({ + message: `Unhandled GitHub connection method: ${method as GitHubConnectionMethod}` + }); + } + + return client; +}; + +type GitHubOrganization = { + login: string; + id: number; +}; + +type GitHubRepository = { + id: number; + name: string; + owner: GitHubOrganization; +}; + +export const getGitHubRepositories = async (appConnection: TGitHubConnection) => { + const client = getGitHubClient(appConnection); + + let repositories: GitHubRepository[]; + + switch (appConnection.method) { + case GitHubConnectionMethod.App: + repositories = await client.paginate("GET /installation/repositories"); + break; + case GitHubConnectionMethod.OAuth: + default: + repositories = (await client.paginate("GET /user/repos")).filter((repo) => repo.permissions?.admin); + break; + } + + return repositories; +}; + +export const getGitHubOrganizations = async (appConnection: TGitHubConnection) => { + const client = getGitHubClient(appConnection); + + let organizations: GitHubOrganization[]; + + switch (appConnection.method) { + case GitHubConnectionMethod.App: { + const installationRepositories = await client.paginate("GET /installation/repositories"); + + const organizationMap: Record = {}; + + installationRepositories.forEach((repo) => { + if (repo.owner.type === "Organization") { + organizationMap[repo.owner.id] = repo.owner; + } + }); + + organizations = Object.values(organizationMap); + + break; + } + case GitHubConnectionMethod.OAuth: + default: + organizations = await client.paginate("GET /user/orgs"); + break; + } + + return organizations; +}; + +export const getGitHubEnvironments = async (appConnection: TGitHubConnection, owner: string, repo: string) => { + const client = getGitHubClient(appConnection); + + try { + const environments = await client.paginate("GET /repos/{owner}/{repo}/environments", { + owner, + repo + }); + + return environments; + } catch (e) { + // repo doesn't have envs + if ((e as { status: number }).status === 404) { + return []; + } + + throw e; + } +}; + type TokenRespData = { access_token: string; scope: string; token_type: string; + error?: string; }; export const validateGitHubConnectionCredentials = async (config: TGitHubConnectionConfig) => { @@ -53,7 +176,10 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect if (!clientId || !clientSecret) { throw new InternalServerError({ - message: `GitHub ${getAppConnectionMethodName(method)} environment variables have not been configured` + message: `GitHub ${getAppConnectionMethodName(method).replace( + "GitHub", + "" + )} environment variables have not been configured` }); } @@ -65,7 +191,7 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect client_id: clientId, client_secret: clientSecret, code: credentials.code, - redirect_uri: `${SITE_URL}/app-connections/github/oauth/callback` + redirect_uri: `${SITE_URL}/organization/app-connections/github/oauth/callback` }, headers: { Accept: "application/json", @@ -90,6 +216,8 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect id: number; account: { login: string; + type: string; + id: number; }; }[]; }>(IntegrationUrls.GITHUB_USER_INSTALLATIONS, { @@ -111,10 +239,13 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect } } + if (!tokenResp.data.access_token) { + throw new InternalServerError({ message: `Missing access token: ${tokenResp.data.error}` }); + } + switch (method) { case GitHubConnectionMethod.App: return { - // access token not needed for GitHub App installationId: credentials.installationId }; case GitHubConnectionMethod.OAuth: diff --git a/backend/src/services/app-connection/github/github-connection-schemas.ts b/backend/src/services/app-connection/github/github-connection-schemas.ts index 5adb211ba..e98b9169d 100644 --- a/backend/src/services/app-connection/github/github-connection-schemas.ts +++ b/backend/src/services/app-connection/github/github-connection-schemas.ts @@ -57,7 +57,7 @@ export const UpdateGitHubConnectionSchema = z const BaseGitHubConnectionSchema = BaseAppConnectionSchema.extend({ app: z.literal(AppConnection.GitHub) }); -export const GitHubAppConnectionSchema = z.intersection( +export const GitHubConnectionSchema = z.intersection( BaseGitHubConnectionSchema, z.discriminatedUnion("method", [ z.object({ @@ -74,19 +74,19 @@ export const GitHubAppConnectionSchema = z.intersection( export const SanitizedGitHubConnectionSchema = z.discriminatedUnion("method", [ BaseGitHubConnectionSchema.extend({ method: z.literal(GitHubConnectionMethod.App), - credentials: GitHubConnectionAppOutputCredentialsSchema.omit({ installationId: true }) + credentials: GitHubConnectionAppOutputCredentialsSchema.pick({}) }), BaseGitHubConnectionSchema.extend({ method: z.literal(GitHubConnectionMethod.OAuth), - credentials: GitHubConnectionOAuthOutputCredentialsSchema.omit({ accessToken: true }) + credentials: GitHubConnectionOAuthOutputCredentialsSchema.pick({}) }) ]); export const GitHubConnectionListItemSchema = z.object({ name: z.literal("GitHub"), app: z.literal(AppConnection.GitHub), - // the below is preferable but currently breaks mintlify - // methods: z.tuple([z.literal(GitHubConnectionMethod.GitHubApp), z.literal(GitHubConnectionMethod.OAuth)]), + // the below is preferable but currently breaks with our zod to json schema parser + // methods: z.tuple([z.literal(GitHubConnectionMethod.App), z.literal(GitHubConnectionMethod.OAuth)]), methods: z.nativeEnum(GitHubConnectionMethod).array(), oauthClientId: z.string().optional(), appClientSlug: z.string().optional() diff --git a/backend/src/services/app-connection/github/github-connection-service.ts b/backend/src/services/app-connection/github/github-connection-service.ts new file mode 100644 index 000000000..b4e95c5a7 --- /dev/null +++ b/backend/src/services/app-connection/github/github-connection-service.ts @@ -0,0 +1,55 @@ +import { OrgServiceActor } from "@app/lib/types"; +import { AppConnection } from "@app/services/app-connection/app-connection-enums"; +import { + getGitHubEnvironments, + getGitHubOrganizations, + getGitHubRepositories +} from "@app/services/app-connection/github/github-connection-fns"; +import { TGitHubConnection } from "@app/services/app-connection/github/github-connection-types"; + +type TGetAppConnectionFunc = ( + app: AppConnection, + connectionId: string, + actor: OrgServiceActor +) => Promise; + +type TListGitHubEnvironmentsDTO = { + connectionId: string; + repo: string; + owner: string; +}; + +export const githubConnectionService = (getAppConnection: TGetAppConnectionFunc) => { + const listRepositories = async (connectionId: string, actor: OrgServiceActor) => { + const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor); + + const repositories = await getGitHubRepositories(appConnection); + + return repositories; + }; + + const listOrganizations = async (connectionId: string, actor: OrgServiceActor) => { + const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor); + + const organizations = await getGitHubOrganizations(appConnection); + + return organizations; + }; + + const listEnvironments = async ( + { connectionId, repo, owner }: TListGitHubEnvironmentsDTO, + actor: OrgServiceActor + ) => { + const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor); + + const environments = await getGitHubEnvironments(appConnection, owner, repo); + + return environments; + }; + + return { + listRepositories, + listOrganizations, + listEnvironments + }; +}; diff --git a/backend/src/services/app-connection/github/github-connection-types.ts b/backend/src/services/app-connection/github/github-connection-types.ts index 5a9b13c00..714c87174 100644 --- a/backend/src/services/app-connection/github/github-connection-types.ts +++ b/backend/src/services/app-connection/github/github-connection-types.ts @@ -5,11 +5,11 @@ import { DiscriminativePick } from "@app/lib/types"; import { AppConnection } from "../app-connection-enums"; import { CreateGitHubConnectionSchema, - GitHubAppConnectionSchema, + GitHubConnectionSchema, ValidateGitHubConnectionCredentialsSchema } from "./github-connection-schemas"; -export type TGitHubConnection = z.infer; +export type TGitHubConnection = z.infer; export type TGitHubConnectionInput = z.infer & { app: AppConnection.GitHub; diff --git a/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-constants.ts b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-constants.ts new file mode 100644 index 000000000..37605442d --- /dev/null +++ b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-constants.ts @@ -0,0 +1,10 @@ +import { AppConnection } from "@app/services/app-connection/app-connection-enums"; +import { SecretSync } from "@app/services/secret-sync/secret-sync-enums"; +import { TSecretSyncListItem } from "@app/services/secret-sync/secret-sync-types"; + +export const AWS_PARAMETER_STORE_SYNC_LIST_OPTION: TSecretSyncListItem = { + name: "AWS Parameter Store", + destination: SecretSync.AWSParameterStore, + connection: AppConnection.AWS, + canImportSecrets: true +}; diff --git a/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts new file mode 100644 index 000000000..a495f9e83 --- /dev/null +++ b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts @@ -0,0 +1,207 @@ +import AWS, { AWSError } from "aws-sdk"; + +import { getAwsConnectionConfig } from "@app/services/app-connection/aws/aws-connection-fns"; +import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors"; +import { TSecretMap } from "@app/services/secret-sync/secret-sync-types"; + +import { TAwsParameterStoreSyncWithCredentials } from "./aws-parameter-store-sync-types"; + +type TAWSParameterStoreRecord = Record; + +const MAX_RETRIES = 5; +const BATCH_SIZE = 10; + +const getSSM = async (secretSync: TAwsParameterStoreSyncWithCredentials) => { + const { destinationConfig, connection } = secretSync; + + const config = await getAwsConnectionConfig(connection, destinationConfig.region); + + const ssm = new AWS.SSM({ + apiVersion: "2014-11-06", + region: destinationConfig.region + }); + + ssm.config.update(config); + + return ssm; +}; + +const sleep = async () => + new Promise((resolve) => { + setTimeout(resolve, 1000); + }); + +const getParametersByPath = async (ssm: AWS.SSM, path: string): Promise => { + const awsParameterStoreSecretsRecord: TAWSParameterStoreRecord = {}; + let hasNext = true; + let nextToken: string | undefined; + let attempt = 0; + + while (hasNext) { + try { + // eslint-disable-next-line no-await-in-loop + const parameters = await ssm + .getParametersByPath({ + Path: path, + Recursive: false, + WithDecryption: true, + MaxResults: BATCH_SIZE, + NextToken: nextToken + }) + .promise(); + + attempt = 0; + + if (parameters.Parameters) { + parameters.Parameters.forEach((parameter) => { + if (parameter.Name) { + // no leading slash if path is '/' + const secKey = path.length > 1 ? parameter.Name.substring(path.length) : parameter.Name; + awsParameterStoreSecretsRecord[secKey] = parameter; + } + }); + } + + hasNext = Boolean(parameters.NextToken); + nextToken = parameters.NextToken; + } catch (e) { + if ((e as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) { + attempt += 1; + // eslint-disable-next-line no-await-in-loop + await sleep(); + } + + throw e; + } + } + + return awsParameterStoreSecretsRecord; +}; + +const putParameter = async ( + ssm: AWS.SSM, + params: AWS.SSM.PutParameterRequest, + attempt = 0 +): Promise => { + try { + return await ssm.putParameter(params).promise(); + } catch (error) { + if ((error as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) { + await sleep(); + + // retry + return putParameter(ssm, params, attempt + 1); + } + throw error; + } +}; + +const deleteParametersBatch = async ( + ssm: AWS.SSM, + parameters: AWS.SSM.Parameter[], + attempt = 0 +): Promise => { + const results: AWS.SSM.DeleteParameterResult[] = []; + let remainingParams = [...parameters]; + + while (remainingParams.length > 0) { + const batch = remainingParams.slice(0, BATCH_SIZE); + + try { + // eslint-disable-next-line no-await-in-loop + const result = await ssm.deleteParameters({ Names: batch.map((param) => param.Name!) }).promise(); + results.push(result); + remainingParams = remainingParams.slice(BATCH_SIZE); + } catch (error) { + if ((error as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) { + // eslint-disable-next-line no-await-in-loop + await sleep(); + + // Retry the current batch + // eslint-disable-next-line no-await-in-loop + return [...results, ...(await deleteParametersBatch(ssm, remainingParams, attempt + 1))]; + } + throw error; + } + } + + return results; +}; + +export const AwsParameterStoreSyncFns = { + syncSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials, secretMap: TSecretMap) => { + const { destinationConfig } = secretSync; + + const ssm = await getSSM(secretSync); + + // TODO(scott): KMS Key ID, Tags + + const awsParameterStoreSecretsRecord = await getParametersByPath(ssm, destinationConfig.path); + + for await (const entry of Object.entries(secretMap)) { + const [key, { value }] = entry; + + // skip empty values (not allowed by AWS) or secrets that haven't changed + if (!value || (key in awsParameterStoreSecretsRecord && awsParameterStoreSecretsRecord[key].Value === value)) { + // eslint-disable-next-line no-continue + continue; + } + + try { + await putParameter(ssm, { + Name: `${destinationConfig.path}${key}`, + Type: "SecureString", + Value: value, + Overwrite: true + }); + } catch (error) { + throw new SecretSyncError({ + error, + secretKey: key + }); + } + } + + const parametersToDelete: AWS.SSM.Parameter[] = []; + + for (const entry of Object.entries(awsParameterStoreSecretsRecord)) { + const [key, parameter] = entry; + + if (!(key in secretMap) || !secretMap[key].value) { + parametersToDelete.push(parameter); + } + } + + await deleteParametersBatch(ssm, parametersToDelete); + }, + getSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials): Promise => { + const { destinationConfig } = secretSync; + + const ssm = await getSSM(secretSync); + + const awsParameterStoreSecretsRecord = await getParametersByPath(ssm, destinationConfig.path); + + return Object.fromEntries( + Object.entries(awsParameterStoreSecretsRecord).map(([key, value]) => [key, { value: value.Value ?? "" }]) + ); + }, + removeSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials, secretMap: TSecretMap) => { + const { destinationConfig } = secretSync; + + const ssm = await getSSM(secretSync); + + const awsParameterStoreSecretsRecord = await getParametersByPath(ssm, destinationConfig.path); + + const parametersToDelete: AWS.SSM.Parameter[] = []; + + for (const entry of Object.entries(awsParameterStoreSecretsRecord)) { + const [key, param] = entry; + + if (key in secretMap) { + parametersToDelete.push(param); + } + } + + await deleteParametersBatch(ssm, parametersToDelete); + } +}; diff --git a/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-schemas.ts b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-schemas.ts new file mode 100644 index 000000000..e89096baa --- /dev/null +++ b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-schemas.ts @@ -0,0 +1,45 @@ +import { z } from "zod"; + +import { SecretSyncs } from "@app/lib/api-docs"; +import { AppConnection, AWSRegion } from "@app/services/app-connection/app-connection-enums"; +import { SecretSync } from "@app/services/secret-sync/secret-sync-enums"; +import { + BaseSecretSyncSchema, + GenericCreateSecretSyncFieldsSchema, + GenericUpdateSecretSyncFieldsSchema +} from "@app/services/secret-sync/secret-sync-schemas"; + +const AwsParameterStoreSyncDestinationConfigSchema = z.object({ + region: z.nativeEnum(AWSRegion).describe(SecretSyncs.DESTINATION_CONFIG.AWS_PARAMETER_STORE.REGION), + path: z + .string() + .trim() + .min(1, "Parameter Store Path required") + .max(2048, "Cannot exceed 2048 characters") + .regex(/^\/([/]|(([\w-]+\/)+))?$/, 'Invalid path - must follow "/example/path/" format') + .describe(SecretSyncs.DESTINATION_CONFIG.AWS_PARAMETER_STORE.PATH) +}); + +export const AwsParameterStoreSyncSchema = BaseSecretSyncSchema(SecretSync.AWSParameterStore).extend({ + destination: z.literal(SecretSync.AWSParameterStore), + destinationConfig: AwsParameterStoreSyncDestinationConfigSchema +}); + +export const CreateAwsParameterStoreSyncSchema = GenericCreateSecretSyncFieldsSchema( + SecretSync.AWSParameterStore +).extend({ + destinationConfig: AwsParameterStoreSyncDestinationConfigSchema +}); + +export const UpdateAwsParameterStoreSyncSchema = GenericUpdateSecretSyncFieldsSchema( + SecretSync.AWSParameterStore +).extend({ + destinationConfig: AwsParameterStoreSyncDestinationConfigSchema.optional() +}); + +export const AwsParameterStoreSyncListItemSchema = z.object({ + name: z.literal("AWS Parameter Store"), + connection: z.literal(AppConnection.AWS), + destination: z.literal(SecretSync.AWSParameterStore), + canImportSecrets: z.literal(true) +}); diff --git a/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-types.ts b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-types.ts new file mode 100644 index 000000000..dada28435 --- /dev/null +++ b/backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-types.ts @@ -0,0 +1,19 @@ +import { z } from "zod"; + +import { TAwsConnection } from "@app/services/app-connection/aws"; + +import { + AwsParameterStoreSyncListItemSchema, + AwsParameterStoreSyncSchema, + CreateAwsParameterStoreSyncSchema +} from "./aws-parameter-store-sync-schemas"; + +export type TAwsParameterStoreSync = z.infer; + +export type TAwsParameterStoreSyncInput = z.infer; + +export type TAwsParameterStoreSyncListItem = z.infer; + +export type TAwsParameterStoreSyncWithCredentials = TAwsParameterStoreSync & { + connection: TAwsConnection; +}; diff --git a/backend/src/services/secret-sync/aws-parameter-store/index.ts b/backend/src/services/secret-sync/aws-parameter-store/index.ts new file mode 100644 index 000000000..20728cd8f --- /dev/null +++ b/backend/src/services/secret-sync/aws-parameter-store/index.ts @@ -0,0 +1,4 @@ +export * from "./aws-parameter-store-sync-constants"; +export * from "./aws-parameter-store-sync-fns"; +export * from "./aws-parameter-store-sync-schemas"; +export * from "./aws-parameter-store-sync-types"; diff --git a/backend/src/services/secret-sync/github/github-sync-constants.ts b/backend/src/services/secret-sync/github/github-sync-constants.ts new file mode 100644 index 000000000..f97b96d9e --- /dev/null +++ b/backend/src/services/secret-sync/github/github-sync-constants.ts @@ -0,0 +1,10 @@ +import { AppConnection } from "@app/services/app-connection/app-connection-enums"; +import { SecretSync } from "@app/services/secret-sync/secret-sync-enums"; +import { TSecretSyncListItem } from "@app/services/secret-sync/secret-sync-types"; + +export const GITHUB_SYNC_LIST_OPTION: TSecretSyncListItem = { + name: "GitHub", + destination: SecretSync.GitHub, + connection: AppConnection.GitHub, + canImportSecrets: false +}; diff --git a/backend/src/services/secret-sync/github/github-sync-enums.ts b/backend/src/services/secret-sync/github/github-sync-enums.ts new file mode 100644 index 000000000..c0109370e --- /dev/null +++ b/backend/src/services/secret-sync/github/github-sync-enums.ts @@ -0,0 +1,11 @@ +export enum GitHubSyncScope { + Repository = "repository", + Organization = "organization", + RepositoryEnvironment = "repository-environment" +} + +export enum GitHubSyncVisibility { + All = "all", + Private = "private", + Selected = "selected" +} diff --git a/backend/src/services/secret-sync/github/github-sync-fns.ts b/backend/src/services/secret-sync/github/github-sync-fns.ts new file mode 100644 index 000000000..a09a41163 --- /dev/null +++ b/backend/src/services/secret-sync/github/github-sync-fns.ts @@ -0,0 +1,242 @@ +import { Octokit } from "@octokit/rest"; +import sodium from "libsodium-wrappers"; + +import { getGitHubClient } from "@app/services/app-connection/github"; +import { GitHubSyncScope, GitHubSyncVisibility } from "@app/services/secret-sync/github/github-sync-enums"; +import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors"; +import { SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps"; +import { TSecretMap } from "@app/services/secret-sync/secret-sync-types"; + +import { TGitHubPublicKey, TGitHubSecret, TGitHubSecretPayload, TGitHubSyncWithCredentials } from "./github-sync-types"; + +// TODO: rate limit handling + +const getEncryptedSecrets = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => { + let encryptedSecrets: TGitHubSecret[]; + + const { destinationConfig } = secretSync; + + switch (destinationConfig.scope) { + case GitHubSyncScope.Organization: { + encryptedSecrets = await client.paginate("GET /orgs/{org}/actions/secrets", { + org: destinationConfig.org + }); + break; + } + case GitHubSyncScope.Repository: { + encryptedSecrets = await client.paginate("GET /repos/{owner}/{repo}/actions/secrets", { + owner: destinationConfig.owner, + repo: destinationConfig.repo + }); + + break; + } + case GitHubSyncScope.RepositoryEnvironment: + default: { + encryptedSecrets = await client.paginate("GET /repos/{owner}/{repo}/environments/{environment_name}/secrets", { + owner: destinationConfig.owner, + repo: destinationConfig.repo, + environment_name: destinationConfig.env + }); + break; + } + } + + return encryptedSecrets; +}; + +const getPublicKey = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => { + let publicKey: TGitHubPublicKey; + + const { destinationConfig } = secretSync; + + switch (destinationConfig.scope) { + case GitHubSyncScope.Organization: { + publicKey = ( + await client.request("GET /orgs/{org}/actions/secrets/public-key", { + org: destinationConfig.org + }) + ).data; + break; + } + case GitHubSyncScope.Repository: { + publicKey = ( + await client.request("GET /repos/{owner}/{repo}/actions/secrets/public-key", { + owner: destinationConfig.owner, + repo: destinationConfig.repo + }) + ).data; + break; + } + case GitHubSyncScope.RepositoryEnvironment: + default: { + publicKey = ( + await client.request("GET /repos/{owner}/{repo}/environments/{environment_name}/secrets/public-key", { + owner: destinationConfig.owner, + repo: destinationConfig.repo, + environment_name: destinationConfig.env + }) + ).data; + break; + } + } + + return publicKey; +}; + +const deleteSecret = async ( + client: Octokit, + secretSync: TGitHubSyncWithCredentials, + encryptedSecret: TGitHubSecret +) => { + const { destinationConfig } = secretSync; + + switch (destinationConfig.scope) { + case GitHubSyncScope.Organization: { + await client.request(`DELETE /orgs/{org}/actions/secrets/{secret_name}`, { + org: destinationConfig.org, + secret_name: encryptedSecret.name + }); + break; + } + case GitHubSyncScope.Repository: { + await client.request("DELETE /repos/{owner}/{repo}/actions/secrets/{secret_name}", { + owner: destinationConfig.owner, + repo: destinationConfig.repo, + secret_name: encryptedSecret.name + }); + break; + } + case GitHubSyncScope.RepositoryEnvironment: + default: { + await client.request("DELETE /repos/{owner}/{repo}/environments/{environment_name}/secrets/{secret_name}", { + owner: destinationConfig.owner, + repo: destinationConfig.repo, + environment_name: destinationConfig.env, + secret_name: encryptedSecret.name + }); + break; + } + } +}; + +const putSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredentials, payload: TGitHubSecretPayload) => { + const { destinationConfig } = secretSync; + + switch (destinationConfig.scope) { + case GitHubSyncScope.Organization: { + const { visibility, selectedRepositoryIds } = destinationConfig; + + await client.request(`PUT /orgs/{org}/actions/secrets/{secret_name}`, { + org: destinationConfig.org, + ...payload, + visibility, + ...(visibility === GitHubSyncVisibility.Selected && { + selected_repository_ids: selectedRepositoryIds + }) + }); + break; + } + case GitHubSyncScope.Repository: { + await client.request("PUT /repos/{owner}/{repo}/actions/secrets/{secret_name}", { + owner: destinationConfig.owner, + repo: destinationConfig.repo, + ...payload + }); + break; + } + case GitHubSyncScope.RepositoryEnvironment: + default: { + await client.request("PUT /repos/{owner}/{repo}/environments/{environment_name}/secrets/{secret_name}", { + owner: destinationConfig.owner, + repo: destinationConfig.repo, + environment_name: destinationConfig.env, + ...payload + }); + break; + } + } +}; + +export const GithubSyncFns = { + syncSecrets: async (secretSync: TGitHubSyncWithCredentials, secretMap: TSecretMap) => { + switch (secretSync.destinationConfig.scope) { + case GitHubSyncScope.Organization: + if (Object.values(secretMap).length > 1000) { + throw new SecretSyncError({ + message: "GitHub does not support storing more than 1,000 secrets at the organization level.", + shouldRetry: false + }); + } + break; + case GitHubSyncScope.Repository: + case GitHubSyncScope.RepositoryEnvironment: + if (Object.values(secretMap).length > 100) { + throw new SecretSyncError({ + message: "GitHub does not support storing more than 100 secrets at the repository level.", + shouldRetry: false + }); + } + break; + default: + throw new Error( + `Unsupported GitHub Sync scope ${ + (secretSync.destinationConfig as TGitHubSyncWithCredentials["destinationConfig"]).scope + }` + ); + } + + const client = getGitHubClient(secretSync.connection); + + const encryptedSecrets = await getEncryptedSecrets(client, secretSync); + + const publicKey = await getPublicKey(client, secretSync); + + for await (const encryptedSecret of encryptedSecrets) { + if (!(encryptedSecret.name in secretMap)) { + await deleteSecret(client, secretSync, encryptedSecret); + } + } + + await sodium.ready.then(async () => { + for await (const key of Object.keys(secretMap)) { + // convert secret & base64 key to Uint8Array. + const binaryKey = sodium.from_base64(publicKey.key, sodium.base64_variants.ORIGINAL); + const binarySecretValue = sodium.from_string(secretMap[key].value); + + // encrypt secret using libsodium + const encryptedBytes = sodium.crypto_box_seal(binarySecretValue, binaryKey); + + // convert encrypted Uint8Array to base64 + const encryptedSecretValue = sodium.to_base64(encryptedBytes, sodium.base64_variants.ORIGINAL); + + try { + await putSecret(client, secretSync, { + secret_name: key, + encrypted_value: encryptedSecretValue, + key_id: publicKey.key_id + }); + } catch (error) { + throw new SecretSyncError({ + error, + secretKey: key + }); + } + } + }); + }, + getSecrets: async (secretSync: TGitHubSyncWithCredentials) => { + throw new Error(`${SECRET_SYNC_NAME_MAP[secretSync.destination]} does not support importing secrets.`); + }, + removeSecrets: async (secretSync: TGitHubSyncWithCredentials, secretMap: TSecretMap) => { + const client = getGitHubClient(secretSync.connection); + + const encryptedSecrets = await getEncryptedSecrets(client, secretSync); + + for await (const encryptedSecret of encryptedSecrets) { + if (encryptedSecret.name in secretMap) { + await deleteSecret(client, secretSync, encryptedSecret); + } + } + } +}; diff --git a/backend/src/services/secret-sync/github/github-sync-schemas.ts b/backend/src/services/secret-sync/github/github-sync-schemas.ts new file mode 100644 index 000000000..37a294a1b --- /dev/null +++ b/backend/src/services/secret-sync/github/github-sync-schemas.ts @@ -0,0 +1,82 @@ +import { z } from "zod"; + +import { SecretSyncs } from "@app/lib/api-docs"; +import { AppConnection } from "@app/services/app-connection/app-connection-enums"; +import { GitHubSyncScope, GitHubSyncVisibility } from "@app/services/secret-sync/github/github-sync-enums"; +import { SecretSync } from "@app/services/secret-sync/secret-sync-enums"; +import { + BaseSecretSyncSchema, + GenericCreateSecretSyncFieldsSchema, + GenericUpdateSecretSyncFieldsSchema +} from "@app/services/secret-sync/secret-sync-schemas"; +import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types"; + +const GitHubSyncDestinationConfigSchema = z + .discriminatedUnion("scope", [ + z.object({ + scope: z.literal(GitHubSyncScope.Organization), + org: z.string().min(1, "Organization name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.ORG), + visibility: z.nativeEnum(GitHubSyncVisibility), + selectedRepositoryIds: z.number().array().optional() + }), + z.object({ + scope: z.literal(GitHubSyncScope.Repository), + owner: z.string().min(1, "Repository owner name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.OWNER), + repo: z.string().min(1, "Repository name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.REPO) + }), + z.object({ + scope: z.literal(GitHubSyncScope.RepositoryEnvironment), + owner: z.string().min(1, "Repository owner name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.OWNER), + repo: z.string().min(1, "Repository name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.REPO), + env: z.string().min(1, "Environment name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.ENV) + }) + ]) + .superRefine((options, ctx) => { + if (options.scope === GitHubSyncScope.Organization) { + if (options.visibility === GitHubSyncVisibility.Selected) { + if (!options.selectedRepositoryIds?.length) + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "Select at least 1 repository", + path: ["selectedRepositoryIds"] + }); + return; + } + + if (options.selectedRepositoryIds?.length) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Selected repositories is only supported for visibility "Selected"`, + path: ["selectedRepositoryIds"] + }); + } + } + }); + +const GitHubSyncOptionsConfig: TSyncOptionsConfig = { canImportSecrets: false }; + +export const GitHubSyncSchema = BaseSecretSyncSchema(SecretSync.GitHub, GitHubSyncOptionsConfig).extend({ + destination: z.literal(SecretSync.GitHub), + destinationConfig: GitHubSyncDestinationConfigSchema +}); + +export const CreateGitHubSyncSchema = GenericCreateSecretSyncFieldsSchema( + SecretSync.GitHub, + GitHubSyncOptionsConfig +).extend({ + destinationConfig: GitHubSyncDestinationConfigSchema +}); + +export const UpdateGitHubSyncSchema = GenericUpdateSecretSyncFieldsSchema( + SecretSync.GitHub, + GitHubSyncOptionsConfig +).extend({ + destinationConfig: GitHubSyncDestinationConfigSchema.optional() +}); + +export const GitHubSyncListItemSchema = z.object({ + name: z.literal("GitHub"), + connection: z.literal(AppConnection.GitHub), + destination: z.literal(SecretSync.GitHub), + canImportSecrets: z.literal(false) +}); diff --git a/backend/src/services/secret-sync/github/github-sync-types.ts b/backend/src/services/secret-sync/github/github-sync-types.ts new file mode 100644 index 000000000..c917a9fa4 --- /dev/null +++ b/backend/src/services/secret-sync/github/github-sync-types.ts @@ -0,0 +1,38 @@ +import { z } from "zod"; + +import { TGitHubConnection } from "@app/services/app-connection/github"; + +import { CreateGitHubSyncSchema, GitHubSyncListItemSchema, GitHubSyncSchema } from "./github-sync-schemas"; + +export type TGitHubSync = z.infer; + +export type TGitHubSyncInput = z.infer; + +export type TGitHubSyncListItem = z.infer; + +export type TGitHubSyncWithCredentials = TGitHubSync & { + connection: TGitHubConnection; +}; + +export type TGitHubSecret = { + name: string; + created_at: string; + updated_at: string; + visibility?: "all" | "private" | "selected"; + selected_repositories_url?: string | undefined; +}; + +export type TGitHubPublicKey = { + key_id: string; + key: string; + id?: number | undefined; + url?: string | undefined; + title?: string | undefined; + created_at?: string | undefined; +}; + +export type TGitHubSecretPayload = { + key_id: string; + secret_name: string; + encrypted_value: string; +}; diff --git a/backend/src/services/secret-sync/github/index.ts b/backend/src/services/secret-sync/github/index.ts new file mode 100644 index 000000000..a136d7780 --- /dev/null +++ b/backend/src/services/secret-sync/github/index.ts @@ -0,0 +1,4 @@ +export * from "./github-sync-constants"; +export * from "./github-sync-fns"; +export * from "./github-sync-schemas"; +export * from "./github-sync-types"; diff --git a/backend/src/services/secret-sync/secret-sync-dal.ts b/backend/src/services/secret-sync/secret-sync-dal.ts new file mode 100644 index 000000000..8d99f8637 --- /dev/null +++ b/backend/src/services/secret-sync/secret-sync-dal.ts @@ -0,0 +1,212 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName } from "@app/db/schemas"; +import { TSecretSyncs } from "@app/db/schemas/secret-syncs"; +import { DatabaseError } from "@app/lib/errors"; +import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex"; +import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; + +export type TSecretSyncDALFactory = ReturnType; + +type SecretSyncFindFilter = Parameters>[0]; + +const baseSecretSyncQuery = ({ filter, db, tx }: { db: TDbClient; filter?: SecretSyncFindFilter; tx?: Knex }) => { + const query = (tx || db.replicaNode())(TableName.SecretSync) + .leftJoin(TableName.SecretFolder, `${TableName.SecretSync}.folderId`, `${TableName.SecretFolder}.id`) + .leftJoin(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`) + .join(TableName.AppConnection, `${TableName.SecretSync}.connectionId`, `${TableName.AppConnection}.id`) + .select(selectAllTableCols(TableName.SecretSync)) + .select( + // environment + db.ref("name").withSchema(TableName.Environment).as("envName"), + db.ref("id").withSchema(TableName.Environment).as("envId"), + db.ref("slug").withSchema(TableName.Environment).as("envSlug"), + // entire connection + db.ref("name").withSchema(TableName.AppConnection).as("connectionName"), + db.ref("method").withSchema(TableName.AppConnection).as("connectionMethod"), + db.ref("app").withSchema(TableName.AppConnection).as("connectionApp"), + db.ref("orgId").withSchema(TableName.AppConnection).as("connectionOrgId"), + db.ref("encryptedCredentials").withSchema(TableName.AppConnection).as("connectionEncryptedCredentials"), + db.ref("description").withSchema(TableName.AppConnection).as("connectionDescription"), + db.ref("version").withSchema(TableName.AppConnection).as("connectionVersion"), + db.ref("createdAt").withSchema(TableName.AppConnection).as("connectionCreatedAt"), + db.ref("updatedAt").withSchema(TableName.AppConnection).as("connectionUpdatedAt") + ); + + // prepends table name to filter keys to avoid ambiguous col references, skipping utility filters like $in, etc. + const prependTableName = (filterObj: object): SecretSyncFindFilter => + Object.fromEntries( + Object.entries(filterObj).map(([key, value]) => + key.startsWith("$") ? [key, prependTableName(value as object)] : [`${TableName.SecretSync}.${key}`, value] + ) + ); + + if (filter) { + /* eslint-disable @typescript-eslint/no-misused-promises */ + void query.where(buildFindFilter(prependTableName(filter))); + } + + return query; +}; + +const expandSecretSync = ( + secretSync: Awaited>[number], + folder?: Awaited>[number] +) => { + const { + envId, + envName, + envSlug, + connectionApp, + connectionName, + connectionId, + connectionOrgId, + connectionEncryptedCredentials, + connectionMethod, + connectionDescription, + connectionCreatedAt, + connectionUpdatedAt, + connectionVersion, + ...el + } = secretSync; + + return { + ...el, + connectionId, + environment: envId ? { id: envId, name: envName, slug: envSlug } : null, + connection: { + app: connectionApp, + id: connectionId, + name: connectionName, + orgId: connectionOrgId, + encryptedCredentials: connectionEncryptedCredentials, + method: connectionMethod, + description: connectionDescription, + createdAt: connectionCreatedAt, + updatedAt: connectionUpdatedAt, + version: connectionVersion + }, + folder: folder + ? { + id: folder.id, + path: folder.path + } + : null + }; +}; + +export const secretSyncDALFactory = ( + db: TDbClient, + folderDAL: Pick +) => { + const secretSyncOrm = ormify(db, TableName.SecretSync); + + const findById = async (id: string, tx?: Knex) => { + try { + const secretSync = await baseSecretSyncQuery({ + filter: { id }, + db, + tx + }).first(); + + if (secretSync) { + // TODO (scott): replace with cached folder path once implemented + const [folderWithPath] = secretSync.folderId + ? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]) + : []; + return expandSecretSync(secretSync, folderWithPath); + } + } catch (error) { + throw new DatabaseError({ error, name: "Find by ID - Secret Sync" }); + } + }; + + const create = async (data: Parameters<(typeof secretSyncOrm)["create"]>[0]) => { + try { + const secretSync = (await secretSyncOrm.transaction(async (tx) => { + const sync = await secretSyncOrm.create(data, tx); + + return baseSecretSyncQuery({ + filter: { id: sync.id }, + db, + tx + }).first(); + }))!; + + // TODO (scott): replace with cached folder path once implemented + const [folderWithPath] = secretSync.folderId + ? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]) + : []; + return expandSecretSync(secretSync, folderWithPath); + } catch (error) { + throw new DatabaseError({ error, name: "Create - Secret Sync" }); + } + }; + + const updateById = async (syncId: string, data: Parameters<(typeof secretSyncOrm)["updateById"]>[1]) => { + try { + const secretSync = (await secretSyncOrm.transaction(async (tx) => { + const sync = await secretSyncOrm.updateById(syncId, data, tx); + + return baseSecretSyncQuery({ + filter: { id: sync.id }, + db, + tx + }).first(); + }))!; + + // TODO (scott): replace with cached folder path once implemented + const [folderWithPath] = secretSync.folderId + ? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]) + : []; + return expandSecretSync(secretSync, folderWithPath); + } catch (error) { + throw new DatabaseError({ error, name: "Update by ID - Secret Sync" }); + } + }; + + const findOne = async (filter: Parameters<(typeof secretSyncOrm)["findOne"]>[0], tx?: Knex) => { + try { + const secretSync = await baseSecretSyncQuery({ filter, db, tx }).first(); + + if (secretSync) { + // TODO (scott): replace with cached folder path once implemented + const [folderWithPath] = secretSync.folderId + ? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]) + : []; + return expandSecretSync(secretSync, folderWithPath); + } + } catch (error) { + throw new DatabaseError({ error, name: "Find One - Secret Sync" }); + } + }; + + const find = async (filter: Parameters<(typeof secretSyncOrm)["find"]>[0], tx?: Knex) => { + try { + const secretSyncs = await baseSecretSyncQuery({ filter, db, tx }); + + if (!secretSyncs.length) return []; + + const foldersWithPath = await folderDAL.findSecretPathByFolderIds( + secretSyncs[0].projectId, + secretSyncs.filter((sync) => Boolean(sync.folderId)).map((sync) => sync.folderId!) + ); + + // TODO (scott): replace with cached folder path once implemented + const folderRecord: Record = {}; + + foldersWithPath.forEach((folder) => { + if (folder) folderRecord[folder.id] = folder; + }); + + return secretSyncs.map((secretSync) => + expandSecretSync(secretSync, secretSync.folderId ? folderRecord[secretSync.folderId] : undefined) + ); + } catch (error) { + throw new DatabaseError({ error, name: "Find - Secret Sync" }); + } + }; + + return { ...secretSyncOrm, findById, findOne, find, create, updateById }; +}; diff --git a/backend/src/services/secret-sync/secret-sync-enums.ts b/backend/src/services/secret-sync/secret-sync-enums.ts new file mode 100644 index 000000000..406a3a161 --- /dev/null +++ b/backend/src/services/secret-sync/secret-sync-enums.ts @@ -0,0 +1,15 @@ +export enum SecretSync { + AWSParameterStore = "aws-parameter-store", + GitHub = "github" +} + +export enum SecretSyncInitialSyncBehavior { + OverwriteDestination = "overwrite-destination", + ImportPrioritizeSource = "import-prioritize-source", + ImportPrioritizeDestination = "import-prioritize-destination" +} + +export enum SecretSyncImportBehavior { + PrioritizeSource = "prioritize-source", + PrioritizeDestination = "prioritize-destination" +} diff --git a/backend/src/services/secret-sync/secret-sync-errors.ts b/backend/src/services/secret-sync/secret-sync-errors.ts new file mode 100644 index 000000000..859fbb00d --- /dev/null +++ b/backend/src/services/secret-sync/secret-sync-errors.ts @@ -0,0 +1,23 @@ +export class SecretSyncError extends Error { + name: string; + + error?: unknown; + + secretKey?: string; + + shouldRetry?: boolean; + + constructor({ + name, + error, + secretKey, + message, + shouldRetry = true + }: { name?: string; error?: unknown; secretKey?: string; shouldRetry?: boolean; message?: string } = {}) { + super(message); + this.name = name || "SecretSyncError"; + this.error = error; + this.secretKey = secretKey; + this.shouldRetry = shouldRetry; + } +} diff --git a/backend/src/services/secret-sync/secret-sync-fns.ts b/backend/src/services/secret-sync/secret-sync-fns.ts new file mode 100644 index 000000000..de39fef02 --- /dev/null +++ b/backend/src/services/secret-sync/secret-sync-fns.ts @@ -0,0 +1,127 @@ +import { AxiosError } from "axios"; + +import { + AWS_PARAMETER_STORE_SYNC_LIST_OPTION, + AwsParameterStoreSyncFns +} from "@app/services/secret-sync/aws-parameter-store"; +import { GITHUB_SYNC_LIST_OPTION, GithubSyncFns } from "@app/services/secret-sync/github"; +import { SecretSync } from "@app/services/secret-sync/secret-sync-enums"; +import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors"; +import { + TSecretMap, + TSecretSyncListItem, + TSecretSyncWithCredentials +} from "@app/services/secret-sync/secret-sync-types"; + +const SECRET_SYNC_LIST_OPTIONS: Record = { + [SecretSync.AWSParameterStore]: AWS_PARAMETER_STORE_SYNC_LIST_OPTION, + [SecretSync.GitHub]: GITHUB_SYNC_LIST_OPTION +}; + +export const listSecretSyncOptions = () => { + return Object.values(SECRET_SYNC_LIST_OPTIONS).sort((a, b) => a.name.localeCompare(b.name)); +}; + +// const addAffixes = (secretSync: TSecretSyncWithCredentials, unprocessedSecretMap: TSecretMap) => { +// let secretMap = { ...unprocessedSecretMap }; +// +// const { appendSuffix, prependPrefix } = secretSync.syncOptions; +// +// if (appendSuffix || prependPrefix) { +// secretMap = {}; +// Object.entries(unprocessedSecretMap).forEach(([key, value]) => { +// secretMap[`${prependPrefix || ""}${key}${appendSuffix || ""}`] = value; +// }); +// } +// +// return secretMap; +// }; +// +// const stripAffixes = (secretSync: TSecretSyncWithCredentials, unprocessedSecretMap: TSecretMap) => { +// let secretMap = { ...unprocessedSecretMap }; +// +// const { appendSuffix, prependPrefix } = secretSync.syncOptions; +// +// if (appendSuffix || prependPrefix) { +// secretMap = {}; +// Object.entries(unprocessedSecretMap).forEach(([key, value]) => { +// let processedKey = key; +// +// if (prependPrefix && processedKey.startsWith(prependPrefix)) { +// processedKey = processedKey.slice(prependPrefix.length); +// } +// +// if (appendSuffix && processedKey.endsWith(appendSuffix)) { +// processedKey = processedKey.slice(0, -appendSuffix.length); +// } +// +// secretMap[processedKey] = value; +// }); +// } +// +// return secretMap; +// }; + +export const SecretSyncFns = { + syncSecrets: (secretSync: TSecretSyncWithCredentials, secretMap: TSecretMap): Promise => { + // const affixedSecretMap = addAffixes(secretSync, secretMap); + + switch (secretSync.destination) { + case SecretSync.AWSParameterStore: + return AwsParameterStoreSyncFns.syncSecrets(secretSync, secretMap); + case SecretSync.GitHub: + return GithubSyncFns.syncSecrets(secretSync, secretMap); + default: + throw new Error( + `Unhandled sync destination for sync secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}` + ); + } + }, + getSecrets: async (secretSync: TSecretSyncWithCredentials): Promise => { + let secretMap: TSecretMap; + switch (secretSync.destination) { + case SecretSync.AWSParameterStore: + secretMap = await AwsParameterStoreSyncFns.getSecrets(secretSync); + break; + case SecretSync.GitHub: + secretMap = await GithubSyncFns.getSecrets(secretSync); + break; + default: + throw new Error( + `Unhandled sync destination for get secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}` + ); + } + + return secretMap; + // return stripAffixes(secretSync, secretMap); + }, + removeSecrets: (secretSync: TSecretSyncWithCredentials, secretMap: TSecretMap): Promise => { + // const affixedSecretMap = addAffixes(secretSync, secretMap); + + switch (secretSync.destination) { + case SecretSync.AWSParameterStore: + return AwsParameterStoreSyncFns.removeSecrets(secretSync, secretMap); + case SecretSync.GitHub: + return GithubSyncFns.removeSecrets(secretSync, secretMap); + default: + throw new Error( + `Unhandled sync destination for remove secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}` + ); + } + } +}; + +export const parseSyncErrorMessage = (err: unknown): string => { + if (err instanceof SecretSyncError) { + return JSON.stringify({ + secretKey: err.secretKey, + error: err.message ?? parseSyncErrorMessage(err.error) + }); + } + + if (err instanceof AxiosError) { + return err?.response?.data ? JSON.stringify(err?.response?.data) : err?.message ?? "An unknown error occurred."; + } + + return (err as Error)?.message || "An unknown error occurred."; +}; diff --git a/backend/src/services/secret-sync/secret-sync-maps.ts b/backend/src/services/secret-sync/secret-sync-maps.ts new file mode 100644 index 000000000..67ba7b690 --- /dev/null +++ b/backend/src/services/secret-sync/secret-sync-maps.ts @@ -0,0 +1,12 @@ +import { AppConnection } from "@app/services/app-connection/app-connection-enums"; +import { SecretSync } from "@app/services/secret-sync/secret-sync-enums"; + +export const SECRET_SYNC_NAME_MAP: Record = { + [SecretSync.AWSParameterStore]: "AWS Parameter Store", + [SecretSync.GitHub]: "GitHub" +}; + +export const SECRET_SYNC_CONNECTION_MAP: Record = { + [SecretSync.AWSParameterStore]: AppConnection.AWS, + [SecretSync.GitHub]: AppConnection.GitHub +}; diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.ts new file mode 100644 index 000000000..d2bcdb590 --- /dev/null +++ b/backend/src/services/secret-sync/secret-sync-queue.ts @@ -0,0 +1,955 @@ +import opentelemetry from "@opentelemetry/api"; +import { AxiosError } from "axios"; +import { Job } from "bullmq"; + +import { ProjectMembershipRole, SecretType } from "@app/db/schemas"; +import { TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-service"; +import { EventType } from "@app/ee/services/audit-log/audit-log-types"; +import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore"; +import { getConfig } from "@app/lib/config/env"; +import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; +import { decryptAppConnectionCredentials } from "@app/services/app-connection/app-connection-fns"; +import { ActorType } from "@app/services/auth/auth-type"; +import { TKmsServiceFactory } from "@app/services/kms/kms-service"; +import { KmsDataKey } from "@app/services/kms/kms-types"; +import { TProjectDALFactory } from "@app/services/project/project-dal"; +import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal"; +import { TProjectMembershipDALFactory } from "@app/services/project-membership/project-membership-dal"; +import { TResourceMetadataDALFactory } from "@app/services/resource-metadata/resource-metadata-dal"; +import { TSecretDALFactory } from "@app/services/secret/secret-dal"; +import { createManySecretsRawFnFactory, updateManySecretsRawFnFactory } from "@app/services/secret/secret-fns"; +import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; +import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; +import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal"; +import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal"; +import { fnSecretsV2FromImports } from "@app/services/secret-import/secret-import-fns"; +import { TSecretSyncDALFactory } from "@app/services/secret-sync/secret-sync-dal"; +import { + SecretSync, + SecretSyncImportBehavior, + SecretSyncInitialSyncBehavior +} from "@app/services/secret-sync/secret-sync-enums"; +import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors"; +import { parseSyncErrorMessage, SecretSyncFns } from "@app/services/secret-sync/secret-sync-fns"; +import { SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps"; +import { + SecretSyncAction, + SecretSyncStatus, + TQueueSecretSyncImportSecretsByIdDTO, + TQueueSecretSyncRemoveSecretsByIdDTO, + TQueueSecretSyncsByPathDTO, + TQueueSecretSyncSyncSecretsByIdDTO, + TQueueSendSecretSyncActionFailedNotificationsDTO, + TSecretMap, + TSecretSyncImportSecretsDTO, + TSecretSyncRaw, + TSecretSyncRemoveSecretsDTO, + TSecretSyncSyncSecretsDTO, + TSecretSyncWithCredentials, + TSendSecretSyncFailedNotificationsJobDTO +} from "@app/services/secret-sync/secret-sync-types"; +import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; +import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal"; +import { expandSecretReferencesFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-fns"; +import { TSecretVersionV2DALFactory } from "@app/services/secret-v2-bridge/secret-version-dal"; +import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal"; +import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; + +export type TSecretSyncQueueFactory = ReturnType; + +type TSecretSyncQueueFactoryDep = { + queueService: Pick; + kmsService: Pick; + keyStore: Pick; + folderDAL: TSecretFolderDALFactory; + secretV2BridgeDAL: Pick< + TSecretV2BridgeDALFactory, + | "findByFolderId" + | "find" + | "insertMany" + | "upsertSecretReferences" + | "findBySecretKeys" + | "bulkUpdate" + | "deleteMany" + >; + secretImportDAL: Pick; + secretSyncDAL: Pick; + auditLogService: Pick; + projectMembershipDAL: Pick; + projectDAL: TProjectDALFactory; + smtpService: Pick; + projectBotDAL: TProjectBotDALFactory; + secretDAL: TSecretDALFactory; + secretVersionDAL: TSecretVersionDALFactory; + secretBlindIndexDAL: TSecretBlindIndexDALFactory; + secretTagDAL: TSecretTagDALFactory; + secretVersionTagDAL: TSecretVersionTagDALFactory; + secretVersionV2BridgeDAL: Pick; + secretVersionTagV2BridgeDAL: Pick; + resourceMetadataDAL: Pick; +}; + +type SecretSyncActionJob = Job< + TQueueSecretSyncSyncSecretsByIdDTO | TQueueSecretSyncImportSecretsByIdDTO | TQueueSecretSyncRemoveSecretsByIdDTO +>; + +const getRequeueDelay = (failureCount?: number) => { + if (!failureCount) return 0; + + const baseDelay = 1000; + const maxDelay = 30000; + + const delay = Math.min(baseDelay * 2 ** failureCount, maxDelay); + + const jitter = delay * (0.5 + Math.random() * 0.5); + + return jitter; +}; + +export const secretSyncQueueFactory = ({ + queueService, + kmsService, + keyStore, + folderDAL, + secretV2BridgeDAL, + secretImportDAL, + secretSyncDAL, + auditLogService, + projectMembershipDAL, + projectDAL, + smtpService, + projectBotDAL, + secretDAL, + secretVersionDAL, + secretBlindIndexDAL, + secretTagDAL, + secretVersionTagDAL, + secretVersionV2BridgeDAL, + secretVersionTagV2BridgeDAL, + resourceMetadataDAL +}: TSecretSyncQueueFactoryDep) => { + const appCfg = getConfig(); + + const integrationMeter = opentelemetry.metrics.getMeter("SecretSyncs"); + const syncSecretsErrorHistogram = integrationMeter.createHistogram("secret_sync_sync_secrets_errors", { + description: "Secret Sync - sync secrets errors", + unit: "1" + }); + const importSecretsErrorHistogram = integrationMeter.createHistogram("secret_sync_import_secrets_errors", { + description: "Secret Sync - import secrets errors", + unit: "1" + }); + const removeSecretsErrorHistogram = integrationMeter.createHistogram("secret_sync_remove_secrets_errors", { + description: "Secret Sync - remove secrets errors", + unit: "1" + }); + + const $createManySecretsRawFn = createManySecretsRawFnFactory({ + projectDAL, + projectBotDAL, + secretDAL, + secretVersionDAL, + secretBlindIndexDAL, + secretTagDAL, + secretVersionTagDAL, + folderDAL, + kmsService, + secretVersionV2BridgeDAL, + secretV2BridgeDAL, + secretVersionTagV2BridgeDAL, + resourceMetadataDAL + }); + + const $updateManySecretsRawFn = updateManySecretsRawFnFactory({ + projectDAL, + projectBotDAL, + secretDAL, + secretVersionDAL, + secretBlindIndexDAL, + secretTagDAL, + secretVersionTagDAL, + folderDAL, + kmsService, + secretVersionV2BridgeDAL, + secretV2BridgeDAL, + secretVersionTagV2BridgeDAL, + resourceMetadataDAL + }); + + const $getInfisicalSecrets = async ( + secretSync: TSecretSyncRaw | TSecretSyncWithCredentials, + includeImports = true + ) => { + const { projectId, folderId, environment, folder } = secretSync; + + if (!folderId || !environment || !folder) + throw new SecretSyncError({ + message: + "Invalid Secret Sync source configuration: folder no longer exists. Please update source environment and secret path.", + shouldRetry: false + }); + + const secretMap: TSecretMap = {}; + + const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ + type: KmsDataKey.SecretManager, + projectId + }); + + const decryptSecretValue = (value?: Buffer | undefined | null) => + value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : ""; + + const { expandSecretReferences } = expandSecretReferencesFactory({ + decryptSecretValue, + secretDAL: secretV2BridgeDAL, + folderDAL, + projectId, + canExpandValue: () => true + }); + + const secrets = await secretV2BridgeDAL.findByFolderId(folderId); + + await Promise.allSettled( + secrets.map(async (secret) => { + const secretKey = secret.key; + const secretValue = decryptSecretValue(secret.encryptedValue); + const expandedSecretValue = await expandSecretReferences({ + environment: environment.slug, + secretPath: folder.path, + skipMultilineEncoding: secret.skipMultilineEncoding, + value: secretValue + }); + secretMap[secretKey] = { value: expandedSecretValue || "" }; + + if (secret.encryptedComment) { + const commentValue = decryptSecretValue(secret.encryptedComment); + secretMap[secretKey].comment = commentValue; + } + + secretMap[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding); + }) + ); + + if (!includeImports) return secretMap; + + const secretImports = await secretImportDAL.find({ folderId, isReplication: false }); + + if (secretImports.length) { + const importedSecrets = await fnSecretsV2FromImports({ + decryptor: decryptSecretValue, + folderDAL, + secretDAL: secretV2BridgeDAL, + expandSecretReferences, + secretImportDAL, + secretImports, + hasSecretAccess: () => true + }); + + for (let i = importedSecrets.length - 1; i >= 0; i -= 1) { + for (let j = 0; j < importedSecrets[i].secrets.length; j += 1) { + const importedSecret = importedSecrets[i].secrets[j]; + if (!secretMap[importedSecret.key]) { + secretMap[importedSecret.key] = { + skipMultilineEncoding: importedSecret.skipMultilineEncoding, + comment: importedSecret.secretComment, + value: importedSecret.secretValue || "" + }; + } + } + } + } + + return secretMap; + }; + + const queueSecretSyncSyncSecretsById = async (payload: TQueueSecretSyncSyncSecretsByIdDTO) => + queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncSyncSecrets, payload, { + delay: getRequeueDelay(payload.failedToAcquireLockCount), // this is for delaying re-queued jobs if sync is locked + attempts: 5, + backoff: { + type: "exponential", + delay: 3000 + }, + removeOnComplete: true, + removeOnFail: true + }); + + const queueSecretSyncImportSecretsById = async (payload: TQueueSecretSyncImportSecretsByIdDTO) => + queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncImportSecrets, payload, { + attempts: 1, + removeOnComplete: true, + removeOnFail: true + }); + + const queueSecretSyncRemoveSecretsById = async (payload: TQueueSecretSyncRemoveSecretsByIdDTO) => + queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncRemoveSecrets, payload, { + attempts: 1, + removeOnComplete: true, + removeOnFail: true + }); + + const $queueSendSecretSyncFailedNotifications = async (payload: TQueueSendSecretSyncActionFailedNotificationsDTO) => { + if (!appCfg.isSmtpConfigured) return; + + await queueService.queue( + QueueName.AppConnectionSecretSync, + QueueJobs.SecretSyncSendActionFailedNotifications, + payload, + { + jobId: `secret-sync-${payload.secretSync.id}-failed-notifications`, + attempts: 5, + delay: 1000 * 60, + backoff: { + type: "exponential", + delay: 3000 + }, + removeOnFail: true, + removeOnComplete: true + } + ); + }; + + const $importSecrets = async ( + secretSync: TSecretSyncWithCredentials, + importBehavior: SecretSyncImportBehavior + ): Promise => { + const { projectId, environment, folder } = secretSync; + + if (!environment || !folder) + throw new Error( + "Invalid Secret Sync source configuration: folder no longer exists. Please update source environment and secret path." + ); + + const importedSecrets = await SecretSyncFns.getSecrets(secretSync); + + if (!Object.keys(importedSecrets).length) return {}; + + const importedSecretMap: TSecretMap = {}; + + const secretMap = await $getInfisicalSecrets(secretSync, false); + + const secretsToCreate: Parameters[0]["secrets"] = []; + const secretsToUpdate: Parameters[0]["secrets"] = []; + + Object.entries(importedSecrets).forEach(([key, secretData]) => { + const { value, comment = "", skipMultilineEncoding } = secretData; + + const secret = { + secretName: key, + secretValue: value, + type: SecretType.Shared, + secretComment: comment, + skipMultilineEncoding: skipMultilineEncoding ?? undefined + }; + + if (Object.hasOwn(secretMap, key)) { + secretsToUpdate.push(secret); + if (importBehavior === SecretSyncImportBehavior.PrioritizeDestination) importedSecretMap[key] = secretData; + } else { + secretsToCreate.push(secret); + importedSecretMap[key] = secretData; + } + }); + + if (secretsToCreate.length) { + await $createManySecretsRawFn({ + projectId, + path: folder.path, + environment: environment.slug, + secrets: secretsToCreate + }); + } + + if (importBehavior === SecretSyncImportBehavior.PrioritizeDestination && secretsToUpdate.length) { + await $updateManySecretsRawFn({ + projectId, + path: folder.path, + environment: environment.slug, + secrets: secretsToUpdate + }); + } + + return importedSecretMap; + }; + + const $handleSyncSecretsJob = async (job: TSecretSyncSyncSecretsDTO) => { + const { + data: { syncId, auditLogInfo } + } = job; + + const secretSync = await secretSyncDAL.findById(syncId); + + if (!secretSync) throw new Error(`Cannot find secret sync with ID ${syncId}`); + + await secretSyncDAL.updateById(syncId, { + syncStatus: SecretSyncStatus.Running + }); + + logger.info( + `SecretSync Sync [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]` + ); + + let isSynced = false; + let syncMessage: string | null = null; + let isFinalAttempt = job.attemptsStarted === job.opts.attempts; + + try { + const { + connection: { orgId, encryptedCredentials } + } = secretSync; + + const credentials = await decryptAppConnectionCredentials({ + orgId, + encryptedCredentials, + kmsService + }); + + const secretSyncWithCredentials = { + ...secretSync, + connection: { + ...secretSync.connection, + credentials + } + } as TSecretSyncWithCredentials; + + const { + lastSyncedAt, + syncOptions: { initialSyncBehavior } + } = secretSyncWithCredentials; + + const secretMap = await $getInfisicalSecrets(secretSync); + + if (!lastSyncedAt && initialSyncBehavior !== SecretSyncInitialSyncBehavior.OverwriteDestination) { + const importedSecretMap = await $importSecrets( + secretSyncWithCredentials, + initialSyncBehavior === SecretSyncInitialSyncBehavior.ImportPrioritizeSource + ? SecretSyncImportBehavior.PrioritizeSource + : SecretSyncImportBehavior.PrioritizeDestination + ); + + Object.entries(importedSecretMap).forEach(([key, secretData]) => { + secretMap[key] = secretData; + }); + } + + await SecretSyncFns.syncSecrets(secretSyncWithCredentials, secretMap); + + isSynced = true; + } catch (err) { + logger.error( + err, + `SecretSync Sync Error [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]` + ); + + if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) { + syncSecretsErrorHistogram.record(1, { + version: 1, + destination: secretSync.destination, + syncId: secretSync.id, + projectId: secretSync.projectId, + type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError", + status: err instanceof AxiosError ? err.response?.status : undefined, + name: err instanceof Error ? err.name : undefined + }); + } + + syncMessage = parseSyncErrorMessage(err); + + if (err instanceof SecretSyncError && !err.shouldRetry) { + isFinalAttempt = true; + } else { + // re-throw so job fails + throw err; + } + } finally { + const ranAt = new Date(); + const syncStatus = isSynced ? SecretSyncStatus.Succeeded : SecretSyncStatus.Failed; + + await auditLogService.createAuditLog({ + projectId: secretSync.projectId, + ...(auditLogInfo ?? { + actor: { + type: ActorType.PLATFORM, + metadata: {} + } + }), + event: { + type: EventType.SECRET_SYNC_SYNC_SECRETS, + metadata: { + syncId: secretSync.id, + syncOptions: secretSync.syncOptions, + destination: secretSync.destination, + destinationConfig: secretSync.destinationConfig, + folderId: secretSync.folderId, + connectionId: secretSync.connectionId, + jobRanAt: ranAt, + jobId: job.id!, + syncStatus, + syncMessage + } + } + }); + + if (isSynced || isFinalAttempt) { + const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, { + syncStatus, + lastSyncJobId: job.id, + lastSyncMessage: syncMessage, + lastSyncedAt: isSynced ? ranAt : undefined + }); + + if (!isSynced) { + await $queueSendSecretSyncFailedNotifications({ + secretSync: updatedSecretSync, + action: SecretSyncAction.SyncSecrets, + auditLogInfo + }); + } + } + } + + logger.info("SecretSync Sync Job with ID %s Completed", job.id); + }; + + const $handleImportSecretsJob = async (job: TSecretSyncImportSecretsDTO) => { + const { + data: { syncId, auditLogInfo, importBehavior } + } = job; + + const secretSync = await secretSyncDAL.findById(syncId); + + if (!secretSync) throw new Error(`Cannot find secret sync with ID ${syncId}`); + + await secretSyncDAL.updateById(syncId, { + importStatus: SecretSyncStatus.Running + }); + + logger.info( + `SecretSync Import [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]` + ); + + let isSuccess = false; + let importMessage: string | null = null; + const isFinalAttempt = job.attemptsStarted === job.opts.attempts; + + try { + const { + connection: { orgId, encryptedCredentials } + } = secretSync; + + const credentials = await decryptAppConnectionCredentials({ + orgId, + encryptedCredentials, + kmsService + }); + + await $importSecrets( + { + ...secretSync, + connection: { + ...secretSync.connection, + credentials + } + } as TSecretSyncWithCredentials, + importBehavior + ); + + isSuccess = true; + } catch (err) { + logger.error( + err, + `SecretSync Import Error [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]` + ); + + if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) { + importSecretsErrorHistogram.record(1, { + version: 1, + destination: secretSync.destination, + syncId: secretSync.id, + projectId: secretSync.projectId, + type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError", + status: err instanceof AxiosError ? err.response?.status : undefined, + name: err instanceof Error ? err.name : undefined + }); + } + + importMessage = parseSyncErrorMessage(err); + + // re-throw so job fails + throw err; + } finally { + const ranAt = new Date(); + const importStatus = isSuccess ? SecretSyncStatus.Succeeded : SecretSyncStatus.Failed; + + await auditLogService.createAuditLog({ + projectId: secretSync.projectId, + ...(auditLogInfo ?? { + actor: { + type: ActorType.PLATFORM, + metadata: {} + } + }), + event: { + type: EventType.SECRET_SYNC_IMPORT_SECRETS, + metadata: { + syncId: secretSync.id, + syncOptions: secretSync.syncOptions, + destination: secretSync.destination, + destinationConfig: secretSync.destinationConfig, + folderId: secretSync.folderId, + connectionId: secretSync.connectionId, + jobRanAt: ranAt, + jobId: job.id!, + importStatus, + importMessage, + importBehavior + } + } + }); + + if (isSuccess || isFinalAttempt) { + const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, { + importStatus, + lastImportJobId: job.id, + lastImportMessage: importMessage, + lastImportedAt: isSuccess ? ranAt : undefined + }); + + if (!isSuccess) { + await $queueSendSecretSyncFailedNotifications({ + secretSync: updatedSecretSync, + action: SecretSyncAction.ImportSecrets, + auditLogInfo + }); + } + } + } + + logger.info("SecretSync Import Job with ID %s Completed", job.id); + }; + + const $handleRemoveSecretsJob = async (job: TSecretSyncRemoveSecretsDTO) => { + const { + data: { syncId, auditLogInfo, deleteSyncOnComplete } + } = job; + + const secretSync = await secretSyncDAL.findById(syncId); + + if (!secretSync) throw new Error(`Cannot find secret sync with ID ${syncId}`); + + await secretSyncDAL.updateById(syncId, { + removeStatus: SecretSyncStatus.Running + }); + + logger.info( + `SecretSync Remove [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]` + ); + + let isSuccess = false; + let removeMessage: string | null = null; + const isFinalAttempt = job.attemptsStarted === job.opts.attempts; + + try { + const { + connection: { orgId, encryptedCredentials } + } = secretSync; + + const credentials = await decryptAppConnectionCredentials({ + orgId, + encryptedCredentials, + kmsService + }); + + const secretMap = await $getInfisicalSecrets(secretSync); + + await SecretSyncFns.removeSecrets( + { + ...secretSync, + connection: { + ...secretSync.connection, + credentials + } + } as TSecretSyncWithCredentials, + secretMap + ); + + isSuccess = true; + } catch (err) { + logger.error( + err, + `SecretSync Remove Error [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]` + ); + + if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) { + removeSecretsErrorHistogram.record(1, { + version: 1, + destination: secretSync.destination, + syncId: secretSync.id, + projectId: secretSync.projectId, + type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError", + status: err instanceof AxiosError ? err.response?.status : undefined, + name: err instanceof Error ? err.name : undefined + }); + } + + removeMessage = parseSyncErrorMessage(err); + + // re-throw so job fails + throw err; + } finally { + const ranAt = new Date(); + const removeStatus = isSuccess ? SecretSyncStatus.Succeeded : SecretSyncStatus.Failed; + + await auditLogService.createAuditLog({ + projectId: secretSync.projectId, + ...(auditLogInfo ?? { + actor: { + type: ActorType.PLATFORM, + metadata: {} + } + }), + event: { + type: EventType.SECRET_SYNC_REMOVE_SECRETS, + metadata: { + syncId: secretSync.id, + syncOptions: secretSync.syncOptions, + destination: secretSync.destination, + destinationConfig: secretSync.destinationConfig, + folderId: secretSync.folderId, + connectionId: secretSync.connectionId, + jobRanAt: ranAt, + jobId: job.id!, + removeStatus, + removeMessage + } + } + }); + + if (isSuccess || isFinalAttempt) { + if (isSuccess && deleteSyncOnComplete) { + await secretSyncDAL.deleteById(secretSync.id); + } else { + const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, { + removeStatus, + lastRemoveJobId: job.id, + lastRemoveMessage: removeMessage, + lastRemovedAt: isSuccess ? ranAt : undefined + }); + + if (!isSuccess) { + await $queueSendSecretSyncFailedNotifications({ + secretSync: updatedSecretSync, + action: SecretSyncAction.RemoveSecrets, + auditLogInfo + }); + } + } + } + } + + logger.info("SecretSync Remove Job with ID %s Completed", job.id); + }; + + const $sendSecretSyncFailedNotifications = async (job: TSendSecretSyncFailedNotificationsJobDTO) => { + const { + data: { secretSync, auditLogInfo, action } + } = job; + + const { projectId, destination, name, folder, lastSyncMessage, lastRemoveMessage, lastImportMessage, environment } = + secretSync; + + const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId); + const project = await projectDAL.findById(projectId); + + let projectAdmins = projectMembers.filter((member) => + member.roles.some((role) => role.role === ProjectMembershipRole.Admin) + ); + + const triggeredByUserId = + auditLogInfo && auditLogInfo.actor.type === ActorType.USER && auditLogInfo.actor.metadata.userId; + + // only notify triggering user if triggered by admin + if (triggeredByUserId && projectAdmins.map((admin) => admin.userId).includes(triggeredByUserId)) { + projectAdmins = projectAdmins.filter((admin) => admin.userId === triggeredByUserId); + } + + const syncDestination = SECRET_SYNC_NAME_MAP[destination as SecretSync]; + + let actionLabel: string; + let failureMessage: string | null | undefined; + + switch (action) { + case SecretSyncAction.ImportSecrets: + actionLabel = "Import"; + failureMessage = lastImportMessage; + + break; + case SecretSyncAction.RemoveSecrets: + actionLabel = "Remove"; + failureMessage = lastRemoveMessage; + + break; + case SecretSyncAction.SyncSecrets: + default: + actionLabel = `Sync`; + failureMessage = lastSyncMessage; + break; + } + + await smtpService.sendMail({ + recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean), + template: SmtpTemplates.SecretSyncFailed, + subjectLine: `Secret Sync Failed to ${actionLabel} Secrets`, + substitutions: { + syncName: name, + syncDestination, + content: `Your ${syncDestination} Sync named "${name}" failed while attempting to ${action.toLowerCase()} secrets.`, + failureMessage, + secretPath: folder?.path, + environment: environment?.name, + projectName: project.name, + syncUrl: `${appCfg.SITE_URL}/integrations/secret-syncs/${destination}/${secretSync.id}` + } + }); + }; + + const queueSecretSyncsSyncSecretsByPath = async ({ + secretPath, + projectId, + environmentSlug + }: TQueueSecretSyncsByPathDTO) => { + const folder = await folderDAL.findBySecretPath(projectId, environmentSlug, secretPath); + + if (!folder) + throw new Error( + `Could not find folder at path "${secretPath}" for environment with slug "${environmentSlug}" in project with ID "${projectId}"` + ); + + const secretSyncs = await secretSyncDAL.find({ folderId: folder.id, isAutoSyncEnabled: true }); + + await Promise.all(secretSyncs.map((secretSync) => queueSecretSyncSyncSecretsById({ syncId: secretSync.id }))); + }; + + const $handleAcquireLockFailure = async (job: SecretSyncActionJob) => { + const { syncId, auditLogInfo } = job.data; + + switch (job.name) { + case QueueJobs.SecretSyncSyncSecrets: { + const { failedToAcquireLockCount = 0, ...rest } = job.data as TQueueSecretSyncSyncSecretsByIdDTO; + + if (failedToAcquireLockCount < 10) { + await queueSecretSyncSyncSecretsById({ ...rest, failedToAcquireLockCount: failedToAcquireLockCount + 1 }); + return; + } + + const secretSync = await secretSyncDAL.updateById(syncId, { + syncStatus: SecretSyncStatus.Failed, + lastSyncMessage: + "Failed to run job. This typically happens when a sync is already in progress. Please try again.", + lastSyncJobId: job.id + }); + + await $queueSendSecretSyncFailedNotifications({ + secretSync, + action: SecretSyncAction.SyncSecrets, + auditLogInfo + }); + + break; + } + // Scott: the two cases below are unlikely to happen as we check the lock at the API level but including this as a fallback + case QueueJobs.SecretSyncImportSecrets: { + const secretSync = await secretSyncDAL.updateById(syncId, { + importStatus: SecretSyncStatus.Failed, + lastImportMessage: + "Failed to run job. This typically happens when a sync is already in progress. Please try again.", + lastImportJobId: job.id + }); + + await $queueSendSecretSyncFailedNotifications({ + secretSync, + action: SecretSyncAction.ImportSecrets, + auditLogInfo + }); + + break; + } + case QueueJobs.SecretSyncRemoveSecrets: { + const secretSync = await secretSyncDAL.updateById(syncId, { + removeStatus: SecretSyncStatus.Failed, + lastRemoveMessage: + "Failed to run job. This typically happens when a sync is already in progress. Please try again.", + lastRemoveJobId: job.id + }); + + await $queueSendSecretSyncFailedNotifications({ + secretSync, + action: SecretSyncAction.RemoveSecrets, + auditLogInfo + }); + + break; + } + default: + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + throw new Error(`Unhandled Secret Sync Job ${job.name}`); + } + }; + + queueService.start(QueueName.AppConnectionSecretSync, async (job) => { + if (job.name === QueueJobs.SecretSyncSendActionFailedNotifications) { + await $sendSecretSyncFailedNotifications(job as TSendSecretSyncFailedNotificationsJobDTO); + return; + } + + const { syncId } = job.data as + | TQueueSecretSyncSyncSecretsByIdDTO + | TQueueSecretSyncImportSecretsByIdDTO + | TQueueSecretSyncRemoveSecretsByIdDTO; + + let lock: Awaited>; + + try { + lock = await keyStore.acquireLock( + [KeyStorePrefixes.SecretSyncLock(syncId)], + // scott: not sure on this duration; syncs can take excessive amounts of time so we need to keep it locked, + // but should always release below... + 5 * 60 * 1000 + ); + } catch (e) { + logger.info(`SecretSync Failed to acquire lock [syncId=${syncId}] [job=${job.name}]`); + + await $handleAcquireLockFailure(job as SecretSyncActionJob); + + return; + } + + try { + switch (job.name) { + case QueueJobs.SecretSyncSyncSecrets: + await $handleSyncSecretsJob(job as TSecretSyncSyncSecretsDTO); + break; + case QueueJobs.SecretSyncImportSecrets: + await $handleImportSecretsJob(job as TSecretSyncImportSecretsDTO); + break; + case QueueJobs.SecretSyncRemoveSecrets: + await $handleRemoveSecretsJob(job as TSecretSyncRemoveSecretsDTO); + break; + default: + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + throw new Error(`Unhandled Secret Sync Job ${job.name}`); + } + } finally { + await lock.release(); + } + }); + + return { + queueSecretSyncSyncSecretsById, + queueSecretSyncImportSecretsById, + queueSecretSyncRemoveSecretsById, + queueSecretSyncsSyncSecretsByPath + }; +}; diff --git a/backend/src/services/secret-sync/secret-sync-schemas.ts b/backend/src/services/secret-sync/secret-sync-schemas.ts new file mode 100644 index 000000000..9821c4e1d --- /dev/null +++ b/backend/src/services/secret-sync/secret-sync-schemas.ts @@ -0,0 +1,96 @@ +import { z } from "zod"; + +import { SecretSyncsSchema } from "@app/db/schemas/secret-syncs"; +import { SecretSyncs } from "@app/lib/api-docs"; +import { removeTrailingSlash } from "@app/lib/fn"; +import { slugSchema } from "@app/server/lib/schemas"; +import { SecretSync, SecretSyncInitialSyncBehavior } from "@app/services/secret-sync/secret-sync-enums"; +import { SECRET_SYNC_CONNECTION_MAP } from "@app/services/secret-sync/secret-sync-maps"; +import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types"; + +const SyncOptionsSchema = (secretSync: SecretSync, options: TSyncOptionsConfig = { canImportSecrets: true }) => + z.object({ + initialSyncBehavior: (options.canImportSecrets + ? z.nativeEnum(SecretSyncInitialSyncBehavior) + : z.literal(SecretSyncInitialSyncBehavior.OverwriteDestination) + ).describe(SecretSyncs.SYNC_OPTIONS(secretSync).INITIAL_SYNC_BEHAVIOR) + // prependPrefix: z + // .string() + // .trim() + // .transform((str) => str.toUpperCase()) + // .optional() + // .describe(SecretSyncs.SYNC_OPTIONS(secretSync).PREPEND_PREFIX), + // appendSuffix: z + // .string() + // .trim() + // .transform((str) => str.toUpperCase()) + // .optional() + // .describe(SecretSyncs.SYNC_OPTIONS(secretSync).APPEND_SUFFIX) + }); + +export const BaseSecretSyncSchema = (destination: SecretSync, syncOptionsConfig?: TSyncOptionsConfig) => + SecretSyncsSchema.omit({ + destination: true, + destinationConfig: true, + syncOptions: true + }).extend({ + // destination needs to be on the extended object for type differentiation + syncOptions: SyncOptionsSchema(destination, syncOptionsConfig), + // join properties + projectId: z.string(), + connection: z.object({ + app: z.literal(SECRET_SYNC_CONNECTION_MAP[destination]), + name: z.string(), + id: z.string().uuid() + }), + environment: z.object({ slug: z.string(), name: z.string(), id: z.string().uuid() }).nullable(), + folder: z.object({ id: z.string(), path: z.string() }).nullable() + }); + +export const GenericCreateSecretSyncFieldsSchema = (destination: SecretSync, syncOptionsConfig?: TSyncOptionsConfig) => + z.object({ + name: slugSchema({ field: "name" }).describe(SecretSyncs.CREATE(destination).name), + projectId: z.string().trim().min(1, "Project ID required").describe(SecretSyncs.CREATE(destination).projectId), + description: z + .string() + .trim() + .max(256, "Description cannot exceed 256 characters") + .nullish() + .describe(SecretSyncs.CREATE(destination).description), + connectionId: z.string().uuid().describe(SecretSyncs.CREATE(destination).connectionId), + environment: slugSchema({ field: "environment", max: 64 }).describe(SecretSyncs.CREATE(destination).environment), + secretPath: z + .string() + .trim() + .min(1, "Secret path required") + .transform(removeTrailingSlash) + .describe(SecretSyncs.CREATE(destination).secretPath), + isAutoSyncEnabled: z.boolean().default(true).describe(SecretSyncs.CREATE(destination).isAutoSyncEnabled), + syncOptions: SyncOptionsSchema(destination, syncOptionsConfig).describe(SecretSyncs.CREATE(destination).syncOptions) + }); + +export const GenericUpdateSecretSyncFieldsSchema = (destination: SecretSync, syncOptionsConfig?: TSyncOptionsConfig) => + z.object({ + name: slugSchema({ field: "name" }).describe(SecretSyncs.UPDATE(destination).name).optional(), + connectionId: z.string().uuid().describe(SecretSyncs.UPDATE(destination).connectionId).optional(), + description: z + .string() + .trim() + .max(256, "Description cannot exceed 256 characters") + .nullish() + .describe(SecretSyncs.UPDATE(destination).description), + environment: slugSchema({ field: "environment", max: 64 }) + .optional() + .describe(SecretSyncs.UPDATE(destination).environment), + secretPath: z + .string() + .trim() + .min(1, "Invalid secret path") + .transform(removeTrailingSlash) + .optional() + .describe(SecretSyncs.UPDATE(destination).secretPath), + isAutoSyncEnabled: z.boolean().optional().describe(SecretSyncs.UPDATE(destination).isAutoSyncEnabled), + syncOptions: SyncOptionsSchema(destination, syncOptionsConfig) + .optional() + .describe(SecretSyncs.UPDATE(destination).syncOptions) + }); diff --git a/backend/src/services/secret-sync/secret-sync-service.ts b/backend/src/services/secret-sync/secret-sync-service.ts new file mode 100644 index 000000000..4ce2fd2d5 --- /dev/null +++ b/backend/src/services/secret-sync/secret-sync-service.ts @@ -0,0 +1,562 @@ +import { ForbiddenError, subject } from "@casl/ability"; + +import { ActionProjectType } from "@app/db/schemas"; +import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { + ProjectPermissionActions, + ProjectPermissionSecretSyncActions, + ProjectPermissionSub +} from "@app/ee/services/permission/project-permission"; +import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore"; +import { BadRequestError, NotFoundError } from "@app/lib/errors"; +import { OrgServiceActor } from "@app/lib/types"; +import { TAppConnectionServiceFactory } from "@app/services/app-connection/app-connection-service"; +import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; +import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal"; +import { SecretSync } from "@app/services/secret-sync/secret-sync-enums"; +import { listSecretSyncOptions } from "@app/services/secret-sync/secret-sync-fns"; +import { + SecretSyncStatus, + TCreateSecretSyncDTO, + TDeleteSecretSyncDTO, + TFindSecretSyncByIdDTO, + TFindSecretSyncByNameDTO, + TListSecretSyncsByProjectId, + TSecretSync, + TTriggerSecretSyncImportSecretsByIdDTO, + TTriggerSecretSyncRemoveSecretsByIdDTO, + TTriggerSecretSyncSyncSecretsByIdDTO, + TUpdateSecretSyncDTO +} from "@app/services/secret-sync/secret-sync-types"; + +import { TSecretSyncDALFactory } from "./secret-sync-dal"; +import { SECRET_SYNC_CONNECTION_MAP, SECRET_SYNC_NAME_MAP } from "./secret-sync-maps"; +import { TSecretSyncQueueFactory } from "./secret-sync-queue"; + +type TSecretSyncServiceFactoryDep = { + secretSyncDAL: TSecretSyncDALFactory; + appConnectionService: Pick; + permissionService: Pick; + projectBotService: Pick; + folderDAL: Pick; + keyStore: Pick; + secretSyncQueue: Pick< + TSecretSyncQueueFactory, + "queueSecretSyncSyncSecretsById" | "queueSecretSyncImportSecretsById" | "queueSecretSyncRemoveSecretsById" + >; +}; + +export type TSecretSyncServiceFactory = ReturnType; + +export const secretSyncServiceFactory = ({ + secretSyncDAL, + folderDAL, + permissionService, + appConnectionService, + projectBotService, + secretSyncQueue, + keyStore +}: TSecretSyncServiceFactoryDep) => { + const listSecretSyncsByProjectId = async ( + { projectId, destination }: TListSecretSyncsByProjectId, + actor: OrgServiceActor + ) => { + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretManager, + projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretSyncActions.Read, + ProjectPermissionSub.SecretSyncs + ); + + const secretSyncs = await secretSyncDAL.find({ + ...(destination && { destination }), + projectId + }); + + return secretSyncs as TSecretSync[]; + }; + + const findSecretSyncById = async ({ destination, syncId }: TFindSecretSyncByIdDTO, actor: OrgServiceActor) => { + const secretSync = await secretSyncDAL.findById(syncId); + + if (!secretSync) + throw new NotFoundError({ + message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID "${syncId}"` + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretManager, + projectId: secretSync.projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretSyncActions.Read, + ProjectPermissionSub.SecretSyncs + ); + + if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination]) + throw new BadRequestError({ + message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}` + }); + + return secretSync as TSecretSync; + }; + + const findSecretSyncByName = async ( + { destination, syncName, projectId }: TFindSecretSyncByNameDTO, + actor: OrgServiceActor + ) => { + const folders = await folderDAL.findByProjectId(projectId); + + // we prevent conflicting names within a project so this will only return one at most + const [secretSync] = await secretSyncDAL.find({ + name: syncName, + $in: { + folderId: folders.map((folder) => folder.id) + } + }); + + if (!secretSync) + throw new NotFoundError({ + message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with name "${syncName}"` + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretManager, + projectId: secretSync.projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretSyncActions.Read, + ProjectPermissionSub.SecretSyncs + ); + + if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination]) + throw new BadRequestError({ + message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}` + }); + + return secretSync as TSecretSync; + }; + + const createSecretSync = async ( + { projectId, secretPath, environment, ...params }: TCreateSecretSyncDTO, + actor: OrgServiceActor + ) => { + const { permission: projectPermission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretManager, + projectId + }); + + const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId); + + if (!shouldUseSecretV2Bridge) + throw new BadRequestError({ message: "Project version does not support Secret Syncs" }); + + ForbiddenError.from(projectPermission).throwUnlessCan( + ProjectPermissionSecretSyncActions.Create, + ProjectPermissionSub.SecretSyncs + ); + + ForbiddenError.from(projectPermission).throwUnlessCan( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { + environment, + secretPath + }) + ); + + const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); + + if (!folder) + throw new BadRequestError({ + message: `Could not find folder with path "${secretPath}" in environment "${environment}" for project with ID "${projectId}"` + }); + + const destinationApp = SECRET_SYNC_CONNECTION_MAP[params.destination]; + + // validates permission to connect and app is valid for sync destination + await appConnectionService.connectAppConnectionById(destinationApp, params.connectionId, actor); + + const secretSync = await secretSyncDAL.transaction(async (tx) => { + const isConflictingName = Boolean( + ( + await secretSyncDAL.find( + { + name: params.name, + projectId + }, + tx + ) + ).length + ); + + if (isConflictingName) + throw new BadRequestError({ + message: `A Secret Sync with the name "${params.name}" already exists for the project with ID "${folder.projectId}"` + }); + + const sync = await secretSyncDAL.create({ + folderId: folder.id, + ...params, + ...(params.isAutoSyncEnabled && { syncStatus: SecretSyncStatus.Pending }), + projectId + }); + + return sync; + }); + + if (secretSync.isAutoSyncEnabled) await secretSyncQueue.queueSecretSyncSyncSecretsById({ syncId: secretSync.id }); + + return secretSync as TSecretSync; + }; + + const updateSecretSync = async ( + { destination, syncId, secretPath, environment, ...params }: TUpdateSecretSyncDTO, + actor: OrgServiceActor + ) => { + const secretSync = await secretSyncDAL.findById(syncId); + + if (!secretSync) + throw new NotFoundError({ + message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID ${syncId}` + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretManager, + projectId: secretSync.projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretSyncActions.Edit, + ProjectPermissionSub.SecretSyncs + ); + + if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination]) + throw new BadRequestError({ + message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}` + }); + + const updatedSecretSync = await secretSyncDAL.transaction(async (tx) => { + let { folderId } = secretSync; + + if (params.connectionId) { + const destinationApp = SECRET_SYNC_CONNECTION_MAP[secretSync.destination as SecretSync]; + + // validates permission to connect and app is valid for sync destination + await appConnectionService.connectAppConnectionById(destinationApp, params.connectionId, actor); + } + + if ( + (secretPath && secretPath !== secretSync.folder?.path) || + (environment && environment !== secretSync.environment?.slug) + ) { + const updatedEnvironment = environment ?? secretSync.environment?.slug; + const updatedSecretPath = secretPath ?? secretSync.folder?.path; + + if (!updatedEnvironment || !updatedSecretPath) + throw new BadRequestError({ message: "Must specify both source environment and secret path" }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { + environment: updatedEnvironment, + secretPath: updatedSecretPath + }) + ); + + const newFolder = await folderDAL.findBySecretPath(secretSync.projectId, updatedEnvironment, updatedSecretPath); + + if (!newFolder) + throw new BadRequestError({ + message: `Could not find folder with path "${secretPath}" in environment "${environment}" for project with ID "${secretSync.projectId}"` + }); + + folderId = newFolder.id; + } + + if (params.name && secretSync.name !== params.name) { + const isConflictingName = Boolean( + ( + await secretSyncDAL.find( + { + name: params.name, + projectId: secretSync.projectId + }, + tx + ) + ).length + ); + + if (isConflictingName) + throw new BadRequestError({ + message: `A Secret Sync with the name "${params.name}" already exists for project with ID "${secretSync.projectId}"` + }); + } + + const isAutoSyncEnabled = params.isAutoSyncEnabled ?? secretSync.isAutoSyncEnabled; + + const updatedSync = await secretSyncDAL.updateById(syncId, { + ...params, + ...(isAutoSyncEnabled && folderId && { syncStatus: SecretSyncStatus.Pending }), + folderId + }); + + return updatedSync; + }); + + if (updatedSecretSync.isAutoSyncEnabled) + await secretSyncQueue.queueSecretSyncSyncSecretsById({ syncId: secretSync.id }); + + return updatedSecretSync as TSecretSync; + }; + + const deleteSecretSync = async ( + { destination, syncId, removeSecrets }: TDeleteSecretSyncDTO, + actor: OrgServiceActor + ) => { + const secretSync = await secretSyncDAL.findById(syncId); + + if (!secretSync) + throw new NotFoundError({ + message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID "${syncId}"` + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretManager, + projectId: secretSync.projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretSyncActions.Delete, + ProjectPermissionSub.SecretSyncs + ); + + if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination]) + throw new BadRequestError({ + message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}` + }); + + if (removeSecrets) { + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretSyncActions.RemoveSecrets, + ProjectPermissionSub.SecretSyncs + ); + + if (!secretSync.folderId) + throw new BadRequestError({ + message: `Invalid source configuration: folder no longer exists. Please configure a valid source and try again.` + }); + + const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId))); + + if (isSyncJobRunning) + throw new BadRequestError({ message: `A job for this sync is already in progress. Please try again shortly.` }); + + await secretSyncQueue.queueSecretSyncRemoveSecretsById({ syncId, deleteSyncOnComplete: true }); + + const updatedSecretSync = await secretSyncDAL.updateById(syncId, { + removeStatus: SecretSyncStatus.Pending + }); + + return updatedSecretSync; + } + + await secretSyncDAL.deleteById(syncId); + + return secretSync as TSecretSync; + }; + + const triggerSecretSyncSyncSecretsById = async ( + { syncId, destination, ...params }: TTriggerSecretSyncSyncSecretsByIdDTO, + actor: OrgServiceActor + ) => { + const secretSync = await secretSyncDAL.findById(syncId); + + if (!secretSync) + throw new NotFoundError({ + message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID "${syncId}"` + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretManager, + projectId: secretSync.projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretSyncActions.SyncSecrets, + ProjectPermissionSub.SecretSyncs + ); + + if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination]) + throw new BadRequestError({ + message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}` + }); + + if (!secretSync.folderId) + throw new BadRequestError({ + message: `Invalid source configuration: folder no longer exists. Please configure a valid source and try again.` + }); + + const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId))); + + if (isSyncJobRunning) + throw new BadRequestError({ message: `A job for this sync is already in progress. Please try again shortly.` }); + + await secretSyncQueue.queueSecretSyncSyncSecretsById({ syncId, ...params }); + + const updatedSecretSync = await secretSyncDAL.updateById(syncId, { + syncStatus: SecretSyncStatus.Pending + }); + + return updatedSecretSync as TSecretSync; + }; + + const triggerSecretSyncImportSecretsById = async ( + { syncId, destination, ...params }: TTriggerSecretSyncImportSecretsByIdDTO, + actor: OrgServiceActor + ) => { + if (!listSecretSyncOptions().find((option) => option.destination === destination)?.canImportSecrets) { + throw new BadRequestError({ + message: `${SECRET_SYNC_NAME_MAP[destination]} does not support importing secrets.` + }); + } + + const secretSync = await secretSyncDAL.findById(syncId); + + if (!secretSync) + throw new NotFoundError({ + message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID "${syncId}"` + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretManager, + projectId: secretSync.projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretSyncActions.ImportSecrets, + ProjectPermissionSub.SecretSyncs + ); + + if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination]) + throw new BadRequestError({ + message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}` + }); + + if (!secretSync.folderId) + throw new BadRequestError({ + message: `Invalid source configuration: folder no longer exists. Please configure a valid source and try again.` + }); + + const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId))); + + if (isSyncJobRunning) + throw new BadRequestError({ message: `A job for this sync is already in progress. Please try again shortly.` }); + + await secretSyncQueue.queueSecretSyncImportSecretsById({ syncId, ...params }); + + const updatedSecretSync = await secretSyncDAL.updateById(syncId, { + importStatus: SecretSyncStatus.Pending + }); + + return updatedSecretSync as TSecretSync; + }; + + const triggerSecretSyncRemoveSecretsById = async ( + { syncId, destination, ...params }: TTriggerSecretSyncRemoveSecretsByIdDTO, + actor: OrgServiceActor + ) => { + const secretSync = await secretSyncDAL.findById(syncId); + + if (!secretSync) + throw new NotFoundError({ + message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID "${syncId}"` + }); + + const { permission } = await permissionService.getProjectPermission({ + actor: actor.type, + actorId: actor.id, + actorAuthMethod: actor.authMethod, + actorOrgId: actor.orgId, + actionProjectType: ActionProjectType.SecretManager, + projectId: secretSync.projectId + }); + + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionSecretSyncActions.RemoveSecrets, + ProjectPermissionSub.SecretSyncs + ); + + if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination]) + throw new BadRequestError({ + message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}` + }); + + if (!secretSync.folderId) + throw new BadRequestError({ + message: `Invalid source configuration: folder no longer exists. Please configure a valid source and try again.` + }); + + const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId))); + + if (isSyncJobRunning) + throw new BadRequestError({ message: `A job for this sync is already in progress. Please try again shortly.` }); + + await secretSyncQueue.queueSecretSyncRemoveSecretsById({ syncId, ...params }); + + const updatedSecretSync = await secretSyncDAL.updateById(syncId, { + removeStatus: SecretSyncStatus.Pending + }); + + return updatedSecretSync as TSecretSync; + }; + + return { + listSecretSyncOptions, + listSecretSyncsByProjectId, + findSecretSyncById, + findSecretSyncByName, + createSecretSync, + updateSecretSync, + deleteSecretSync, + triggerSecretSyncSyncSecretsById, + triggerSecretSyncImportSecretsById, + triggerSecretSyncRemoveSecretsById + }; +}; diff --git a/backend/src/services/secret-sync/secret-sync-types.ts b/backend/src/services/secret-sync/secret-sync-types.ts new file mode 100644 index 000000000..eade6671d --- /dev/null +++ b/backend/src/services/secret-sync/secret-sync-types.ts @@ -0,0 +1,148 @@ +import { Job } from "bullmq"; + +import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types"; +import { QueueJobs } from "@app/queue"; +import { + TGitHubSync, + TGitHubSyncInput, + TGitHubSyncListItem, + TGitHubSyncWithCredentials +} from "@app/services/secret-sync/github"; +import { TSecretSyncDALFactory } from "@app/services/secret-sync/secret-sync-dal"; +import { SecretSync, SecretSyncImportBehavior } from "@app/services/secret-sync/secret-sync-enums"; + +import { + TAwsParameterStoreSync, + TAwsParameterStoreSyncInput, + TAwsParameterStoreSyncListItem, + TAwsParameterStoreSyncWithCredentials +} from "./aws-parameter-store"; + +export type TSecretSync = TAwsParameterStoreSync | TGitHubSync; + +export type TSecretSyncWithCredentials = TAwsParameterStoreSyncWithCredentials | TGitHubSyncWithCredentials; + +export type TSecretSyncInput = TAwsParameterStoreSyncInput | TGitHubSyncInput; + +export type TSecretSyncListItem = TAwsParameterStoreSyncListItem | TGitHubSyncListItem; + +export type TSyncOptionsConfig = { + canImportSecrets: boolean; +}; + +export type TListSecretSyncsByProjectId = { + projectId: string; + destination?: SecretSync; +}; + +export type TFindSecretSyncByIdDTO = { + syncId: string; + destination: SecretSync; +}; + +export type TFindSecretSyncByNameDTO = { + syncName: string; + projectId: string; + destination: SecretSync; +}; + +export type TCreateSecretSyncDTO = Pick & { + destination: SecretSync; + projectId: string; + secretPath: string; + environment: string; + isAutoSyncEnabled?: boolean; +}; + +export type TUpdateSecretSyncDTO = Partial> & { + syncId: string; + destination: SecretSync; +}; + +export type TDeleteSecretSyncDTO = { + destination: SecretSync; + syncId: string; + removeSecrets: boolean; +}; + +type AuditLogInfo = Pick; + +export enum SecretSyncStatus { + Pending = "pending", + Running = "running", + Succeeded = "succeeded", + Failed = "failed" +} + +export enum SecretSyncAction { + SyncSecrets = "sync-secrets", + ImportSecrets = "import-secrets", + RemoveSecrets = "remove-secrets" +} + +export type TSecretSyncRaw = NonNullable>>; + +export type TQueueSecretSyncsByPathDTO = { + secretPath: string; + environmentSlug: string; + projectId: string; +}; + +export type TQueueSecretSyncSyncSecretsByIdDTO = { + syncId: string; + failedToAcquireLockCount?: number; + auditLogInfo?: AuditLogInfo; +}; + +export type TTriggerSecretSyncSyncSecretsByIdDTO = { + destination: SecretSync; +} & TQueueSecretSyncSyncSecretsByIdDTO; + +export type TQueueSecretSyncImportSecretsByIdDTO = { + syncId: string; + importBehavior: SecretSyncImportBehavior; + auditLogInfo?: AuditLogInfo; +}; + +export type TTriggerSecretSyncImportSecretsByIdDTO = { + destination: SecretSync; +} & TQueueSecretSyncImportSecretsByIdDTO; + +export type TQueueSecretSyncRemoveSecretsByIdDTO = { + syncId: string; + auditLogInfo?: AuditLogInfo; + deleteSyncOnComplete?: boolean; +}; + +export type TTriggerSecretSyncRemoveSecretsByIdDTO = { + destination: SecretSync; +} & TQueueSecretSyncRemoveSecretsByIdDTO; + +export type TQueueSendSecretSyncActionFailedNotificationsDTO = { + secretSync: TSecretSyncRaw; + auditLogInfo?: AuditLogInfo; + action: SecretSyncAction; +}; + +export type TSecretSyncSyncSecretsDTO = Job; +export type TSecretSyncImportSecretsDTO = Job< + TQueueSecretSyncImportSecretsByIdDTO, + void, + QueueJobs.SecretSyncSyncSecrets +>; +export type TSecretSyncRemoveSecretsDTO = Job< + TQueueSecretSyncRemoveSecretsByIdDTO, + void, + QueueJobs.SecretSyncSyncSecrets +>; + +export type TSendSecretSyncFailedNotificationsJobDTO = Job< + TQueueSendSecretSyncActionFailedNotificationsDTO, + void, + QueueJobs.SecretSyncSendActionFailedNotifications +>; + +export type TSecretMap = Record< + string, + { value: string; comment?: string; skipMultilineEncoding?: boolean | null | undefined } +>; diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 661da2362..dc973c0b1 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -29,6 +29,7 @@ import { createManySecretsRawFnFactory, updateManySecretsRawFnFactory } from "@a import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal"; import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal"; import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal"; +import { TSecretSyncQueueFactory } from "@app/services/secret-sync/secret-sync-queue"; import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal"; import { ActorType } from "../auth/auth-type"; @@ -107,6 +108,7 @@ type TSecretQueueFactoryDep = { orgService: Pick; projectUserMembershipRoleDAL: Pick; resourceMetadataDAL: Pick; + secretSyncQueue: Pick; }; export type TGetSecrets = { @@ -166,7 +168,8 @@ export const secretQueueFactory = ({ orgService, projectUserMembershipRoleDAL, projectKeyDAL, - resourceMetadataDAL + resourceMetadataDAL, + secretSyncQueue }: TSecretQueueFactoryDep) => { const integrationMeter = opentelemetry.metrics.getMeter("Integrations"); const errorHistogram = integrationMeter.createHistogram("integration_secret_sync_errors", { @@ -633,6 +636,9 @@ export const secretQueueFactory = ({ } } ); + + await secretSyncQueue.queueSecretSyncsSyncSecretsByPath({ projectId, environmentSlug: environment, secretPath }); + await syncIntegrations({ secretPath, projectId, environment, deDupeQueue, isManual: false }); if (!excludeReplication) { await replicateSecrets({ diff --git a/backend/src/services/smtp/smtp-service.ts b/backend/src/services/smtp/smtp-service.ts index a2ed85749..d997d52f4 100644 --- a/backend/src/services/smtp/smtp-service.ts +++ b/backend/src/services/smtp/smtp-service.ts @@ -35,6 +35,7 @@ export enum SmtpTemplates { ScimUserProvisioned = "scimUserProvisioned.handlebars", PkiExpirationAlert = "pkiExpirationAlert.handlebars", IntegrationSyncFailed = "integrationSyncFailed.handlebars", + SecretSyncFailed = "secretSyncFailed.handlebars", ExternalImportSuccessful = "externalImportSuccessful.handlebars", ExternalImportFailed = "externalImportFailed.handlebars", ExternalImportStarted = "externalImportStarted.handlebars" diff --git a/backend/src/services/smtp/templates/secretSyncFailed.handlebars b/backend/src/services/smtp/templates/secretSyncFailed.handlebars new file mode 100644 index 000000000..3e7ad7831 --- /dev/null +++ b/backend/src/services/smtp/templates/secretSyncFailed.handlebars @@ -0,0 +1,39 @@ + + + + + + {{syncDestination}} Sync "{{syncName}}" Failed + + + +

Infisical

+ +
+

{{content}}

+ + View in Infisical. + +
+ +
+
+

Name: {{syncName}}

+

Destination: {{syncDestination}}

+

Project: {{projectName}}

+ {{#if environment}} +

Environment: {{environment}}

+ {{/if}} + {{#if secretPath}} +

Secret Path: {{secretPath}}

+ {{/if}} +
+ + {{#if failureMessage}} +

Reason: {{failureMessage}}

+ {{/if}} + + {{emailFooter}} + + + \ No newline at end of file diff --git a/docs/api-reference/endpoints/app-connections/aws/available.mdx b/docs/api-reference/endpoints/app-connections/aws/available.mdx new file mode 100644 index 000000000..1386c068d --- /dev/null +++ b/docs/api-reference/endpoints/app-connections/aws/available.mdx @@ -0,0 +1,4 @@ +--- +title: "Available" +openapi: "GET /api/v1/app-connections/aws/available" +--- diff --git a/docs/api-reference/endpoints/app-connections/aws/get-by-name.mdx b/docs/api-reference/endpoints/app-connections/aws/get-by-name.mdx index d18994f7c..d6db40ade 100644 --- a/docs/api-reference/endpoints/app-connections/aws/get-by-name.mdx +++ b/docs/api-reference/endpoints/app-connections/aws/get-by-name.mdx @@ -1,4 +1,4 @@ --- title: "Get by Name" -openapi: "GET /api/v1/app-connections/aws/name/{connectionName}" +openapi: "GET /api/v1/app-connections/aws/connection-name/{connectionName}" --- diff --git a/docs/api-reference/endpoints/app-connections/github/available.mdx b/docs/api-reference/endpoints/app-connections/github/available.mdx new file mode 100644 index 000000000..6d5596629 --- /dev/null +++ b/docs/api-reference/endpoints/app-connections/github/available.mdx @@ -0,0 +1,4 @@ +--- +title: "Available" +openapi: "GET /api/v1/app-connections/github/available" +--- diff --git a/docs/api-reference/endpoints/app-connections/github/get-by-name.mdx b/docs/api-reference/endpoints/app-connections/github/get-by-name.mdx index 95ddbd6e9..cf959827b 100644 --- a/docs/api-reference/endpoints/app-connections/github/get-by-name.mdx +++ b/docs/api-reference/endpoints/app-connections/github/get-by-name.mdx @@ -1,4 +1,4 @@ --- title: "Get by Name" -openapi: "GET /api/v1/app-connections/github/name/{connectionName}" +openapi: "GET /api/v1/app-connections/github/connection-name/{connectionName}" --- diff --git a/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/create.mdx b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/create.mdx new file mode 100644 index 000000000..2e29b8a5a --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/create.mdx @@ -0,0 +1,4 @@ +--- +title: "Create" +openapi: "POST /api/v1/secret-syncs/aws-parameter-store" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/delete.mdx b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/delete.mdx new file mode 100644 index 000000000..2c801aba3 --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/delete.mdx @@ -0,0 +1,4 @@ +--- +title: "Delete" +openapi: "DELETE /api/v1/secret-syncs/aws-parameter-store/{syncId}" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/get-by-id.mdx b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/get-by-id.mdx new file mode 100644 index 000000000..aeecf16e1 --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/get-by-id.mdx @@ -0,0 +1,4 @@ +--- +title: "Get by ID" +openapi: "GET /api/v1/secret-syncs/aws-parameter-store/{syncId}" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/get-by-name.mdx b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/get-by-name.mdx new file mode 100644 index 000000000..67930be3c --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/get-by-name.mdx @@ -0,0 +1,4 @@ +--- +title: "Get by Name" +openapi: "GET /api/v1/secret-syncs/aws-parameter-store/sync-name/{syncName}" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/import-secrets.mdx b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/import-secrets.mdx new file mode 100644 index 000000000..217fd849c --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/import-secrets.mdx @@ -0,0 +1,4 @@ +--- +title: "Import Secrets" +openapi: "POST /api/v1/secret-syncs/aws-parameter-store/{syncId}/import-secrets" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/list.mdx b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/list.mdx new file mode 100644 index 000000000..8a0c2281d --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/list.mdx @@ -0,0 +1,4 @@ +--- +title: "List" +openapi: "GET /api/v1/secret-syncs/aws-parameter-store" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/remove-secrets.mdx b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/remove-secrets.mdx new file mode 100644 index 000000000..bc617b40d --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/remove-secrets.mdx @@ -0,0 +1,4 @@ +--- +title: "Remove Secrets" +openapi: "POST /api/v1/secret-syncs/aws-parameter-store/{syncId}/remove-secrets" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/sync-secrets.mdx b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/sync-secrets.mdx new file mode 100644 index 000000000..12b723054 --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/sync-secrets.mdx @@ -0,0 +1,4 @@ +--- +title: "Sync Secrets" +openapi: "POST /api/v1/secret-syncs/aws-parameter-store/{syncId}/sync-secrets" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/update.mdx b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/update.mdx new file mode 100644 index 000000000..b290ddfa4 --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/aws-parameter-store/update.mdx @@ -0,0 +1,4 @@ +--- +title: "Update" +openapi: "PATCH /api/v1/secret-syncs/aws-parameter-store/{syncId}" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/github/create.mdx b/docs/api-reference/endpoints/secret-syncs/github/create.mdx new file mode 100644 index 000000000..d0260b8ea --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/github/create.mdx @@ -0,0 +1,4 @@ +--- +title: "Create" +openapi: "POST /api/v1/secret-syncs/github" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/github/delete.mdx b/docs/api-reference/endpoints/secret-syncs/github/delete.mdx new file mode 100644 index 000000000..409a65cda --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/github/delete.mdx @@ -0,0 +1,4 @@ +--- +title: "Delete" +openapi: "DELETE /api/v1/secret-syncs/github/{syncId}" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/github/get-by-id.mdx b/docs/api-reference/endpoints/secret-syncs/github/get-by-id.mdx new file mode 100644 index 000000000..d3c6da848 --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/github/get-by-id.mdx @@ -0,0 +1,4 @@ +--- +title: "Get by ID" +openapi: "GET /api/v1/secret-syncs/github/{syncId}" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/github/get-by-name.mdx b/docs/api-reference/endpoints/secret-syncs/github/get-by-name.mdx new file mode 100644 index 000000000..b4c17b4d8 --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/github/get-by-name.mdx @@ -0,0 +1,4 @@ +--- +title: "Get by Name" +openapi: "GET /api/v1/secret-syncs/github/sync-name/{syncName}" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/github/list.mdx b/docs/api-reference/endpoints/secret-syncs/github/list.mdx new file mode 100644 index 000000000..c3c0e10ab --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/github/list.mdx @@ -0,0 +1,4 @@ +--- +title: "List" +openapi: "GET /api/v1/secret-syncs/github" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/github/remove-secrets.mdx b/docs/api-reference/endpoints/secret-syncs/github/remove-secrets.mdx new file mode 100644 index 000000000..1c133da8c --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/github/remove-secrets.mdx @@ -0,0 +1,4 @@ +--- +title: "Remove Secrets" +openapi: "POST /api/v1/secret-syncs/github/{syncId}/remove-secrets" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/github/sync-secrets.mdx b/docs/api-reference/endpoints/secret-syncs/github/sync-secrets.mdx new file mode 100644 index 000000000..e1bcf1045 --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/github/sync-secrets.mdx @@ -0,0 +1,4 @@ +--- +title: "Sync Secrets" +openapi: "POST /api/v1/secret-syncs/github/{syncId}/sync-secrets" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/github/update.mdx b/docs/api-reference/endpoints/secret-syncs/github/update.mdx new file mode 100644 index 000000000..62d30327e --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/github/update.mdx @@ -0,0 +1,4 @@ +--- +title: "Update" +openapi: "PATCH /api/v1/secret-syncs/github/{syncId}" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/list.mdx b/docs/api-reference/endpoints/secret-syncs/list.mdx new file mode 100644 index 000000000..d18b47f9a --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/list.mdx @@ -0,0 +1,4 @@ +--- +title: "List" +openapi: "GET /api/v1/secret-syncs" +--- diff --git a/docs/api-reference/endpoints/secret-syncs/options.mdx b/docs/api-reference/endpoints/secret-syncs/options.mdx new file mode 100644 index 000000000..cc485111b --- /dev/null +++ b/docs/api-reference/endpoints/secret-syncs/options.mdx @@ -0,0 +1,4 @@ +--- +title: "Options" +openapi: "GET /api/v1/secret-syncs/options" +--- diff --git a/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-created.png b/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-created.png new file mode 100644 index 000000000..009331fe6 Binary files /dev/null and b/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-created.png differ diff --git a/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-destination.png b/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-destination.png new file mode 100644 index 000000000..d7136cd9a Binary files /dev/null and b/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-destination.png differ diff --git a/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-details.png b/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-details.png new file mode 100644 index 000000000..2d4b59a3f Binary files /dev/null and b/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-details.png differ diff --git a/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-options.png b/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-options.png new file mode 100644 index 000000000..11923e5a2 Binary files /dev/null and b/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-options.png differ diff --git a/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-review.png b/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-review.png new file mode 100644 index 000000000..0db843ede Binary files /dev/null and b/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-review.png differ diff --git a/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-source.png b/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-source.png new file mode 100644 index 000000000..4a5ec9904 Binary files /dev/null and b/docs/images/secret-syncs/aws-parameter-store/aws-parameter-store-source.png differ diff --git a/docs/images/secret-syncs/aws-parameter-store/select-aws-parameter-store-option.png b/docs/images/secret-syncs/aws-parameter-store/select-aws-parameter-store-option.png new file mode 100644 index 000000000..d43a79715 Binary files /dev/null and b/docs/images/secret-syncs/aws-parameter-store/select-aws-parameter-store-option.png differ diff --git a/docs/images/secret-syncs/general/secret-sync-tab.png b/docs/images/secret-syncs/general/secret-sync-tab.png new file mode 100644 index 000000000..dad8c2426 Binary files /dev/null and b/docs/images/secret-syncs/general/secret-sync-tab.png differ diff --git a/docs/images/secret-syncs/github/github-created.png b/docs/images/secret-syncs/github/github-created.png new file mode 100644 index 000000000..f3ab7241b Binary files /dev/null and b/docs/images/secret-syncs/github/github-created.png differ diff --git a/docs/images/secret-syncs/github/github-destination.png b/docs/images/secret-syncs/github/github-destination.png new file mode 100644 index 000000000..3713932bb Binary files /dev/null and b/docs/images/secret-syncs/github/github-destination.png differ diff --git a/docs/images/secret-syncs/github/github-details.png b/docs/images/secret-syncs/github/github-details.png new file mode 100644 index 000000000..a8cbbbb94 Binary files /dev/null and b/docs/images/secret-syncs/github/github-details.png differ diff --git a/docs/images/secret-syncs/github/github-options.png b/docs/images/secret-syncs/github/github-options.png new file mode 100644 index 000000000..8f2e3a4ba Binary files /dev/null and b/docs/images/secret-syncs/github/github-options.png differ diff --git a/docs/images/secret-syncs/github/github-review.png b/docs/images/secret-syncs/github/github-review.png new file mode 100644 index 000000000..4cbed76b6 Binary files /dev/null and b/docs/images/secret-syncs/github/github-review.png differ diff --git a/docs/images/secret-syncs/github/github-source.png b/docs/images/secret-syncs/github/github-source.png new file mode 100644 index 000000000..dd4bc3ec8 Binary files /dev/null and b/docs/images/secret-syncs/github/github-source.png differ diff --git a/docs/images/secret-syncs/github/select-github-option.png b/docs/images/secret-syncs/github/select-github-option.png new file mode 100644 index 000000000..ebee947b4 Binary files /dev/null and b/docs/images/secret-syncs/github/select-github-option.png differ diff --git a/docs/integrations/app-connections/aws.mdx b/docs/integrations/app-connections/aws.mdx index 65af1bcdc..176847764 100644 --- a/docs/integrations/app-connections/aws.mdx +++ b/docs/integrations/app-connections/aws.mdx @@ -67,7 +67,7 @@ Infisical supports two methods for connecting to AWS. Depending on your use case, add one or more of the following policies to your IAM Role: - + Use the following custom policy to grant the minimum permissions required by Infisical to sync secrets to AWS Secrets Manager: @@ -217,7 +217,7 @@ Infisical supports two methods for connecting to AWS. Depending on your use case, add one or more of the following policies to your IAM Role: - + Use the following custom policy to grant the minimum permissions required by Infisical to sync secrets to AWS Secrets Manager: diff --git a/docs/integrations/app-connections/github.mdx b/docs/integrations/app-connections/github.mdx index 18f702bb6..5d33bad58 100644 --- a/docs/integrations/app-connections/github.mdx +++ b/docs/integrations/app-connections/github.mdx @@ -23,7 +23,7 @@ Infisical supports two methods for connecting to GitHub. ![integrations github app create](/images/integrations/github/app/self-hosted-github-app-create.png) - Give the application a name, a homepage URL (your self-hosted domain i.e. `https://your-domain.com`), and a callback URL (i.e. `https://your-domain.com/app-connections/github/oauth/callback`). + Give the application a name, a homepage URL (your self-hosted domain i.e. `https://your-domain.com`), and a callback URL (i.e. `https://your-domain.com/organization/app-connections/github/oauth/callback`). ![integrations github app basic details](/images/integrations/github/app/self-hosted-github-app-basic-details.png) @@ -116,7 +116,7 @@ Infisical supports two methods for connecting to GitHub. ![integrations github config](../../images/integrations/github/integrations-github-config-new-app.png) Create the OAuth application. As part of the form, set the **Homepage URL** to your self-hosted domain `https://your-domain.com` - and the **Authorization callback URL** to `https://your-domain.com/app-connections/github/oauth/callback`. + and the **Authorization callback URL** to `https://your-domain.com/organization/app-connections/github/oauth/callback`. ![integrations github config](../../images/integrations/github/integrations-github-config-new-app-form.png) diff --git a/docs/integrations/secret-syncs/aws-parameter-store.mdx b/docs/integrations/secret-syncs/aws-parameter-store.mdx new file mode 100644 index 000000000..a9055ff1e --- /dev/null +++ b/docs/integrations/secret-syncs/aws-parameter-store.mdx @@ -0,0 +1,139 @@ +--- +title: "AWS Parameter Store Sync" +description: "Learn how to configure an AWS Parameter Store Sync for Infisical." +--- + +**Prerequisites:** + + - Set up and add secrets to [Infisical Cloud](https://app.infisical.com) + - Create an [AWS Connection](/integrations/app-connections/aws) with the required **Secret Sync** permissions + + + + 1. Navigate to **Project** > **Integrations** and select the **Secret Syncs** tab. Click on the **Add Sync** button. + ![Secret Syncs Tab](/images/secret-syncs/general/secret-sync-tab.png) + + 2. Select the **AWS Parameter Store** option. + ![Select AWS Parameter Store](/images/secret-syncs/aws-parameter-store/select-aws-parameter-store-option.png) + + 3. Configure the **Source** from where secrets should be retrieved, then click **Next**. + ![Configure Source](/images/secret-syncs/aws-parameter-store/aws-parameter-store-source.png) + + - **Environment**: The project environment to retrieve secrets from. + - **Secret Path**: The folder path to retrieve secrets from. + + + If you need to sync secrets from multiple folder locations, check out [secret imports](/documentation/platform/secret-reference#secret-imports). + + + 4. Configure the **Destination** to where secrets should be deployed, then click **Next**. + ![Configure Destination](/images/secret-syncs/aws-parameter-store/aws-parameter-store-destination.png) + + - **AWS Connection**: The AWS Connection to authenticate with. + - **Region**: The AWS region to deploy secrets to. + - **Path**: The AWS Parameter Store path to deploy secrets to. + + 5. Configure the **Sync Options** to specify how secrets should be synced, then click **Next**. + ![Configure Options](/images/secret-syncs/aws-parameter-store/aws-parameter-store-options.png) + + - **Initial Sync Behavior**: Determines how Infisical should resolve the initial sync. + - **Overwrite Destination Secrets**: Removes any secrets at the destination endpoint not present in Infisical. + - **Import Secrets (Prioritize Infisical)**: Imports secrets from the destination endpoint prior to syncing, prioritizing values present in Infisical if secrets conflict. + - **Import Secrets (Prioritize Parameter Store)**: Imports secrets from the destination endpoint prior to syncing, prioritizing values present in Parameter Store if secrets conflict. + - **Auto-Sync Enabled**: If enabled, secrets will automatically be synced from the source location when changes occur. Disable to enforce manual syncing only. + + 6. Configure the **Details** of your Parameter Store Sync, then click **Next**. + ![Configure Details](/images/secret-syncs/aws-parameter-store/aws-parameter-store-details.png) + + - **Name**: The name of your sync. Must be slug-friendly. + - **Description**: An optional description for your sync. + + 7. Review your Parameter Store Sync configuration, then click **Create Sync**. + ![Confirm Configuration](/images/secret-syncs/aws-parameter-store/aws-parameter-store-review.png) + + 8. If enabled, your Parameter Store Sync will begin syncing your secrets to the destination endpoint. + ![Sync Secrets](/images/secret-syncs/aws-parameter-store/aws-parameter-store-created.png) + + + + To create an **AWS Parameter Store Sync**, make an API request to the [Create AWS + Parameter Store Sync](/api-reference/endpoints/secret-syncs/aws-parameter-store/create) API endpoint. + + ### Sample request + + ```bash Request + curl --request POST \ + --url https://app.infisical.com/api/v1/secret-syncs/aws-parameter-store \ + --header 'Content-Type: application/json' \ + --data '{ + "name": "my-parameter-store-sync", + "projectId": "3c90c3cc-0d44-4b50-8888-8dd25736052a", + "description": "an example sync", + "connectionId": "3c90c3cc-0d44-4b50-8888-8dd25736052a", + "environment": "dev", + "secretPath": "/my-secrets", + "isEnabled": true, + "syncOptions": { + "initialSyncBehavior": "overwrite-destination" + }, + "destinationConfig": { + "region": "us-east-1", + "path": "/my-aws/path/" + } + }' + ``` + + ### Sample response + + ```bash Response + { + "secretSync": { + "id": "3c90c3cc-0d44-4b50-8888-8dd25736052a", + "name": "my-parameter-store-sync", + "description": "an example sync", + "isEnabled": true, + "version": 1, + "folderId": "3c90c3cc-0d44-4b50-8888-8dd25736052a", + "connectionId": "3c90c3cc-0d44-4b50-8888-8dd25736052a", + "createdAt": "2023-11-07T05:31:56Z", + "updatedAt": "2023-11-07T05:31:56Z", + "syncStatus": "succeeded", + "lastSyncJobId": "123", + "lastSyncMessage": null, + "lastSyncedAt": "2023-11-07T05:31:56Z", + "importStatus": null, + "lastImportJobId": null, + "lastImportMessage": null, + "lastImportedAt": null, + "removeStatus": null, + "lastRemoveJobId": null, + "lastRemoveMessage": null, + "lastRemovedAt": null, + "syncOptions": { + "initialSyncBehavior": "overwrite-destination" + }, + "projectId": "3c90c3cc-0d44-4b50-8888-8dd25736052a", + "connection": { + "app": "aws", + "name": "my-aws-connection", + "id": "3c90c3cc-0d44-4b50-8888-8dd25736052a" + }, + "environment": { + "slug": "dev", + "name": "Development", + "id": "3c90c3cc-0d44-4b50-8888-8dd25736052a" + }, + "folder": { + "id": "3c90c3cc-0d44-4b50-8888-8dd25736052a", + "path": "/my-secrets" + }, + "destination": "aws-parameter-store", + "destinationConfig": { + "region": "us-east-1", + "path": "/my-aws/path/" + } + } + } + ``` + + diff --git a/docs/integrations/secret-syncs/github.mdx b/docs/integrations/secret-syncs/github.mdx new file mode 100644 index 000000000..4c6d52efb --- /dev/null +++ b/docs/integrations/secret-syncs/github.mdx @@ -0,0 +1,162 @@ +--- +title: "GitHub Sync" +description: "Learn how to configure a GitHub Sync for Infisical." +--- + +**Prerequisites:** + + - Set up and add secrets to [Infisical Cloud](https://app.infisical.com) + - Create a [GitHub Connection](/integrations/app-connections/github) + + + + 1. Navigate to **Project** > **Integrations** and select the **Secret Syncs** tab. Click on the **Add Sync** button. + ![Secret Syncs Tab](/images/secret-syncs/general/secret-sync-tab.png) + + 2. Select the **GitHub Store** option. + ![Select GitHub](/images/secret-syncs/github/select-github-option.png) + + 3. Configure the **Source** from where secrets should be retrieved, then click **Next**. + ![Configure Source](/images/secret-syncs/github/github-source.png) + + - **Environment**: The project environment to retrieve secrets from. + - **Secret Path**: The folder path to retrieve secrets from. + + + If you need to sync secrets from multiple folder locations, check out [secret imports](/documentation/platform/secret-reference#secret-imports). + + + 4. Configure the **Destination** to where secrets should be deployed, then click **Next**. + ![Configure Destination](/images/secret-syncs/github/github-destination.png) + + - **GitHub Connection**: The GitHub Connection to authenticate with. + - **Scope**: The GitHub secret scope to sync secrets to. + - **Organization**: Sync secrets to a specific organization. + - **Repository**: Sync secrets to a specific repository. + - **Repository Environment**: Sync secrets to a specific repository's environment. +

+ The remaining fields are determined by the selected **Scope**: + + + - **Organization**: The organization to deploy secrets to. + - **Visibility**: Determines which organization repositories can access deployed secrets. + - **All Repositories**: All repositories of the organization. (Public repositories if not a Pro/Team account) + - **Private Repositories**: All private repositories of the organization. (Requires Pro/Team account) + - **Selected Repositories**: Only the selected Repositories. + - **Selected Repositories**: The selected repositories if **Visibility** is set to **Selected Repositories**. + + + - **Repository**: The repository to deploy secrets to. + + + - **Repository**: The repository to deploy secrets to. + - **Environment**: The repository's environment to deploy secrets to. + + + + 5. Configure the **Sync Options** to specify how secrets should be synced, then click **Next**. + ![Configure Options](/images/secret-syncs/github/github-options.png) + + - **Initial Sync Behavior**: Determines how Infisical should resolve the initial sync. + - **Overwrite Destination Secrets**: Removes any secrets at the destination endpoint not present in Infisical. + + GitHub does not support importing secrets. + + - **Auto-Sync Enabled**: If enabled, secrets will automatically be synced from the source location when changes occur. Disable to enforce manual syncing only. + + 6. Configure the **Details** of your GitHub Sync, then click **Next**. + ![Configure Details](/images/secret-syncs/github/github-details.png) + + - **Name**: The name of your sync. Must be slug-friendly. + - **Description**: An optional description for your sync. + + 7. Review your GitHub Sync configuration, then click **Create Sync**. + ![Confirm Configuration](/images/secret-syncs/github/github-review.png) + + 8. If enabled, your GitHub Sync will begin syncing your secrets to the destination endpoint. + ![Sync Secrets](/images/secret-syncs/github/github-created.png) + + + + To create an **GitHub Sync**, make an API request to the [Create GitHub Sync](/api-reference/endpoints/secret-syncs/github/create) API endpoint. + + ### Sample request + + ```bash Request + curl --request POST \ + --url https://app.infisical.com/api/v1/secret-syncs/github \ + --header 'Content-Type: application/json' \ + --data '{ + "name": "my-github-sync", + "projectId": "3c90c3cc-0d44-4b50-8888-8dd25736052a", + "description": "an example sync", + "connectionId": "3c90c3cc-0d44-4b50-8888-8dd25736052a", + "environment": "dev", + "secretPath": "/my-secrets", + "isEnabled": true, + "syncOptions": { + "initialSyncBehavior": "overwrite-destination" + }, + "destinationConfig": { + "scope": "repository", + "owner": "my-github", + "repo": "my-repository" + } + }' + ``` + + ### Sample response + + ```bash Response + { + "secretSync": { + "id": "3c90c3cc-0d44-4b50-8888-8dd25736052a", + "name": "my-github-sync", + "description": "an example sync", + "isEnabled": true, + "version": 1, + "folderId": "3c90c3cc-0d44-4b50-8888-8dd25736052a", + "connectionId": "3c90c3cc-0d44-4b50-8888-8dd25736052a", + "createdAt": "2023-11-07T05:31:56Z", + "updatedAt": "2023-11-07T05:31:56Z", + "syncStatus": "succeeded", + "lastSyncJobId": "123", + "lastSyncMessage": null, + "lastSyncedAt": "2023-11-07T05:31:56Z", + "importStatus": null, + "lastImportJobId": null, + "lastImportMessage": null, + "lastImportedAt": null, + "removeStatus": null, + "lastRemoveJobId": null, + "lastRemoveMessage": null, + "lastRemovedAt": null, + "syncOptions": { + "initialSyncBehavior": "overwrite-destination" + }, + "projectId": "3c90c3cc-0d44-4b50-8888-8dd25736052a", + "connection": { + "app": "github", + "name": "my-github-connection", + "id": "3c90c3cc-0d44-4b50-8888-8dd25736052a" + }, + "environment": { + "slug": "dev", + "name": "Development", + "id": "3c90c3cc-0d44-4b50-8888-8dd25736052a" + }, + "folder": { + "id": "3c90c3cc-0d44-4b50-8888-8dd25736052a", + "path": "/my-secrets" + }, + "destination": "github", + "destinationConfig": { + "scope": "repository", + "owner": "my-github", + "repo": "my-repository" + } + } + } + ``` + + diff --git a/docs/integrations/secret-syncs/overview.mdx b/docs/integrations/secret-syncs/overview.mdx new file mode 100644 index 000000000..cc93d8036 --- /dev/null +++ b/docs/integrations/secret-syncs/overview.mdx @@ -0,0 +1,89 @@ +--- +sidebarTitle: "Overview" +description: "Learn how to sync secrets to third-party services with Infisical." +--- + +Secret Syncs enable you to sync secrets from Infisical to third-party services using [App Connections](/integrations/app-connections/overview). + + + Secret Syncs will gradually replace Native Integrations as they become available. Native Integrations will be deprecated in the future, so opt for configuring a Secret Sync when available. + + +## Concept + +Secret Syncs are a project-level resource used to sync secrets, via an [App Connection](/integrations/app-connections/overview), from a particular project environment and folder path (source) +to a third-party service (destination). Changes to the source will automatically be propagated to the destination, ensuring +your secrets are always up-to-date. + +
+ +

+ + ```mermaid + %%{init: {'flowchart': {'curve': 'linear'} } }%% + graph LR + A[App Connection] + B[Secret Sync] + C[Secret 1] + D[Secret 2] + E[Secret 3] + F[Third-Party Service] + G[Secret 1] + H[Secret 2] + I[Secret 3] + J[Project Source] + + B --> A + C --> J + D --> J + E --> J + A --> F + F --> G + F --> H + F --> I + J --> B + + classDef default fill:#ffffff,stroke:#666,stroke-width:2px,rx:10px,color:black + classDef connection fill:#FFF2B2,stroke:#E6C34A,stroke-width:2px,color:black,rx:15px + classDef secret fill:#E6F4FF,stroke:#0096D6,stroke-width:2px,color:black,rx:15px + classDef sync fill:#F4FFE6,stroke:#96D600,stroke-width:2px,color:black,rx:15px + classDef service fill:#E6E6FF,stroke:#6B4E96,stroke-width:2px,color:black,rx:15px + classDef project fill:#FFE6E6,stroke:#D63F3F,stroke-width:2px,color:black,rx:15px + + class A connection + class B sync + class C,D,E,G,H,I secret + class F project + class J service + ``` + +
+ +## Workflow + +Configuring a Secret Sync requires three components: a source location to retrieve secrets from, +a destination endpoint to deploy secrets to, and configuration options to determine how your secrets +should be synced. Follow these steps to start syncing: + + + For step-by-step guides on syncing to a particular third-party service, refer to the Secret Syncs section in the Navigation Bar. + + +1. Create App Connection: If you have not already done so, create an [App Connection](/integrations/app-connections/overview) +via the UI or API for the third-party service you intend to sync secrets to. + +2. Create Secret Sync: Configure a Secret Sync in the desired project by specifying the following parameters via the UI or API: + - Source: The project environment and folder path you wish to retrieve secrets from. + - Destination: The App Connection to utilize and the destination endpoint to deploy secrets to. These can vary between services. + - Options: Customize how secrets should be synced. Examples include adding a suffix or prefix to your secrets, or importing secrets from the destination on the initial sync. + + + Some third-party services do not support importing secrets. + + +3. Utilize Sync: Any changes to the source location will now automatically be propagated to the destination endpoint. + + + Infisical is continuously expanding it's Secret Sync third-party service support. If the service you need isn't available, + you can still use our Native Integrations in the interim, or contact us at team@infisical.com to make a request . + \ No newline at end of file diff --git a/docs/mint.json b/docs/mint.json index 26d199fce..23b10f579 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -343,6 +343,22 @@ "cli/faq" ] }, + { + "group": "App Connections", + "pages": [ + "integrations/app-connections/overview", + "integrations/app-connections/aws", + "integrations/app-connections/github" + ] + }, + { + "group": "Secret Syncs", + "pages": [ + "integrations/secret-syncs/overview", + "integrations/secret-syncs/aws-parameter-store", + "integrations/secret-syncs/github" + ] + }, { "group": "Infrastructure Integrations", "pages": [ @@ -767,6 +783,67 @@ "api-reference/endpoints/identity-specific-privilege/list" ] }, + { + "group": "App Connections", + "pages": [ + "api-reference/endpoints/app-connections/list", + "api-reference/endpoints/app-connections/options", + { "group": "AWS", + "pages": [ + "api-reference/endpoints/app-connections/aws/list", + "api-reference/endpoints/app-connections/aws/available", + "api-reference/endpoints/app-connections/aws/get-by-id", + "api-reference/endpoints/app-connections/aws/get-by-name", + "api-reference/endpoints/app-connections/aws/create", + "api-reference/endpoints/app-connections/aws/update", + "api-reference/endpoints/app-connections/aws/delete" + ] + }, + { "group": "GitHub", + "pages": [ + "api-reference/endpoints/app-connections/github/list", + "api-reference/endpoints/app-connections/github/available", + "api-reference/endpoints/app-connections/github/get-by-id", + "api-reference/endpoints/app-connections/github/get-by-name", + "api-reference/endpoints/app-connections/github/create", + "api-reference/endpoints/app-connections/github/update", + "api-reference/endpoints/app-connections/github/delete" + ] + } + ] + }, + { + "group": "Secret Syncs", + "pages": [ + "api-reference/endpoints/secret-syncs/list", + "api-reference/endpoints/secret-syncs/options", + { "group": "AWS Parameter Store", + "pages": [ + "api-reference/endpoints/secret-syncs/aws-parameter-store/list", + "api-reference/endpoints/secret-syncs/aws-parameter-store/get-by-id", + "api-reference/endpoints/secret-syncs/aws-parameter-store/get-by-name", + "api-reference/endpoints/secret-syncs/aws-parameter-store/create", + "api-reference/endpoints/secret-syncs/aws-parameter-store/update", + "api-reference/endpoints/secret-syncs/aws-parameter-store/delete", + "api-reference/endpoints/secret-syncs/aws-parameter-store/sync-secrets", + "api-reference/endpoints/secret-syncs/aws-parameter-store/import-secrets", + "api-reference/endpoints/secret-syncs/aws-parameter-store/remove-secrets" + ] + }, + { "group": "GitHub", + "pages": [ + "api-reference/endpoints/secret-syncs/github/list", + "api-reference/endpoints/secret-syncs/github/get-by-id", + "api-reference/endpoints/secret-syncs/github/get-by-name", + "api-reference/endpoints/secret-syncs/github/create", + "api-reference/endpoints/secret-syncs/github/update", + "api-reference/endpoints/secret-syncs/github/delete", + "api-reference/endpoints/secret-syncs/github/sync-secrets", + "api-reference/endpoints/secret-syncs/github/remove-secrets" + ] + } + ] + }, { "group": "Integrations", "pages": [ diff --git a/frontend/src/components/secret-syncs/CreateSecretSyncModal.tsx b/frontend/src/components/secret-syncs/CreateSecretSyncModal.tsx new file mode 100644 index 000000000..3e232765c --- /dev/null +++ b/frontend/src/components/secret-syncs/CreateSecretSyncModal.tsx @@ -0,0 +1,70 @@ +import { useState } from "react"; + +import { Modal, ModalContent } from "@app/components/v2"; +import { SecretSync, TSecretSync } from "@app/hooks/api/secretSyncs"; + +import { CreateSecretSyncForm } from "./forms"; +import { SecretSyncModalHeader } from "./SecretSyncModalHeader"; +import { SecretSyncSelect } from "./SecretSyncSelect"; + +type Props = { + isOpen: boolean; + onOpenChange: (isOpen: boolean) => void; +}; + +type ContentProps = { + onComplete: (secretSync: TSecretSync) => void; + selectedSync: SecretSync | null; + setSelectedSync: (selectedSync: SecretSync | null) => void; +}; + +const Content = ({ onComplete, setSelectedSync, selectedSync }: ContentProps) => { + if (selectedSync) { + return ( + setSelectedSync(null)} + destination={selectedSync} + /> + ); + } + + return ; +}; + +export const CreateSecretSyncModal = ({ onOpenChange, ...props }: Props) => { + const [selectedSync, setSelectedSync] = useState(null); + + return ( + { + if (!isOpen) setSelectedSync(null); + onOpenChange(isOpen); + }} + > + + ) : ( + "Add Sync" + ) + } + onPointerDownOutside={(e) => e.preventDefault()} + className="max-w-2xl" + subTitle={selectedSync ? undefined : "Select a third-party service to sync secrets to."} + bodyClassName="overflow-visible" + > + { + setSelectedSync(null); + onOpenChange(false); + }} + selectedSync={selectedSync} + setSelectedSync={setSelectedSync} + /> + + + ); +}; diff --git a/frontend/src/components/secret-syncs/DeleteSecretSyncModal.tsx b/frontend/src/components/secret-syncs/DeleteSecretSyncModal.tsx new file mode 100644 index 000000000..1daed67a3 --- /dev/null +++ b/frontend/src/components/secret-syncs/DeleteSecretSyncModal.tsx @@ -0,0 +1,70 @@ +import { useState } from "react"; + +import { createNotification } from "@app/components/notifications"; +import { DeleteActionModal, Switch } from "@app/components/v2"; +import { SECRET_SYNC_MAP } from "@app/helpers/secretSyncs"; +import { TSecretSync, useDeleteSecretSync } from "@app/hooks/api/secretSyncs"; + +type Props = { + secretSync?: TSecretSync; + isOpen: boolean; + onOpenChange: (isOpen: boolean) => void; + onComplete?: () => void; +}; + +export const DeleteSecretSyncModal = ({ isOpen, onOpenChange, secretSync, onComplete }: Props) => { + const deleteSync = useDeleteSecretSync(); + const [removeSecrets, setRemoveSecrets] = useState(false); + + if (!secretSync) return null; + + const { id: syncId, name, destination } = secretSync; + + const handleDeleteSecretSync = async () => { + const destinationName = SECRET_SYNC_MAP[destination].name; + + try { + await deleteSync.mutateAsync({ + syncId, + destination, + removeSecrets + }); + + createNotification({ + text: `Successfully removed ${destinationName} Sync`, + type: "success" + }); + + if (onComplete) onComplete(); + onOpenChange(false); + } catch (err) { + console.error(err); + + createNotification({ + text: `Failed to remove ${destinationName} Sync`, + type: "error" + }); + } + }; + + return ( + + + Remove Synced Secrets + + + ); +}; diff --git a/frontend/src/components/secret-syncs/EditSecretSyncModal.tsx b/frontend/src/components/secret-syncs/EditSecretSyncModal.tsx new file mode 100644 index 000000000..c7d0661d3 --- /dev/null +++ b/frontend/src/components/secret-syncs/EditSecretSyncModal.tsx @@ -0,0 +1,33 @@ +import { SecretSyncEditFields } from "@app/components/secret-syncs/types"; +import { Modal, ModalContent } from "@app/components/v2"; +import { TSecretSync } from "@app/hooks/api/secretSyncs"; + +import { EditSecretSyncForm } from "./forms"; +import { SecretSyncModalHeader } from "./SecretSyncModalHeader"; + +type Props = { + isOpen: boolean; + onOpenChange: (isOpen: boolean) => void; + secretSync?: TSecretSync; + fields: SecretSyncEditFields; +}; + +export const EditSecretSyncModal = ({ secretSync, onOpenChange, fields, ...props }: Props) => { + if (!secretSync) return null; + + return ( + + } + className="max-w-2xl" + bodyClassName="overflow-visible" + > + onOpenChange(false)} + fields={fields} + secretSync={secretSync} + /> + + + ); +}; diff --git a/frontend/src/components/secret-syncs/SecretSyncImportSecretsModal.tsx b/frontend/src/components/secret-syncs/SecretSyncImportSecretsModal.tsx new file mode 100644 index 000000000..aab8a4243 --- /dev/null +++ b/frontend/src/components/secret-syncs/SecretSyncImportSecretsModal.tsx @@ -0,0 +1,167 @@ +import { Controller, useForm } from "react-hook-form"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { z } from "zod"; + +import { createNotification } from "@app/components/notifications"; +import { + Button, + FormControl, + Modal, + ModalClose, + ModalContent, + Select, + SelectItem +} from "@app/components/v2"; +import { SECRET_SYNC_IMPORT_BEHAVIOR_MAP, SECRET_SYNC_MAP } from "@app/helpers/secretSyncs"; +import { + SecretSyncImportBehavior, + TSecretSync, + useTriggerSecretSyncImportSecrets +} from "@app/hooks/api/secretSyncs"; + +type Props = { + secretSync?: TSecretSync; + isOpen: boolean; + onOpenChange: (isOpen: boolean) => void; +}; + +type ContentProps = { + secretSync: TSecretSync; + onComplete: () => void; +}; + +const FormSchema = z.object({ + importBehavior: z.nativeEnum(SecretSyncImportBehavior) +}); + +type TFormData = z.infer; + +const Content = ({ secretSync, onComplete }: ContentProps) => { + const { id: syncId, destination } = secretSync; + const destinationName = SECRET_SYNC_MAP[destination].name; + + const { + handleSubmit, + control, + formState: { isSubmitting, isDirty } + } = useForm({ resolver: zodResolver(FormSchema) }); + + const triggerImportSecrets = useTriggerSecretSyncImportSecrets(); + + const handleTriggerImportSecrets = async ({ importBehavior }: TFormData) => { + try { + await triggerImportSecrets.mutateAsync({ + syncId, + destination, + importBehavior + }); + + createNotification({ + text: `Successfully triggered secret import for ${destinationName} Sync`, + type: "success" + }); + + onComplete(); + } catch (err) { + console.error(err); + + createNotification({ + text: `Failed to trigger secret import for ${destinationName} Sync`, + type: "error" + }); + } + }; + + return ( +
+

+ Are you sure you want to import secrets from this {destinationName} destination into + Infiscal? +

+ ( + +

+ Specify how Infisical should resolve the initial sync to {destinationName}. The + following options are available: +

+
    + {Object.values(SECRET_SYNC_IMPORT_BEHAVIOR_MAP).map((details) => { + const { name, description } = details(destinationName); + + return ( +
  • +

    + {name}: {description} +

    +
    • + ); + })} +
+ + } + errorText={error?.message} + isError={Boolean(error?.message)} + label="Import Behavior" + > + + + )} + /> +
+ + + + +
+ + ); +}; + +export const SecretSyncImportSecretsModal = ({ isOpen, onOpenChange, secretSync }: Props) => { + if (!secretSync) return null; + + const destinationName = SECRET_SYNC_MAP[secretSync.destination].name; + + return ( + + + onOpenChange(false)} /> + + + ); +}; diff --git a/frontend/src/components/secret-syncs/SecretSyncImportStatusBadge.tsx b/frontend/src/components/secret-syncs/SecretSyncImportStatusBadge.tsx new file mode 100644 index 000000000..5f54d5c61 --- /dev/null +++ b/frontend/src/components/secret-syncs/SecretSyncImportStatusBadge.tsx @@ -0,0 +1,112 @@ +import { ReactNode, useEffect, useMemo, useState } from "react"; +import { + faCheck, + faDownload, + faTriangleExclamation, + faXmark, + IconDefinition +} from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { differenceInSeconds } from "date-fns"; +import { twMerge } from "tailwind-merge"; + +import { Badge, Tooltip } from "@app/components/v2"; +import { BadgeProps } from "@app/components/v2/Badge/Badge"; +import { SECRET_SYNC_MAP } from "@app/helpers/secretSyncs"; +import { SecretSyncStatus, TSecretSync } from "@app/hooks/api/secretSyncs"; + +type Props = { + secretSync: TSecretSync; + className?: string; + mini?: boolean; +}; + +export const SecretSyncImportStatusBadge = ({ secretSync, className, mini }: Props) => { + const { importStatus, lastImportMessage, lastImportedAt, destination } = secretSync; + const [hide, setHide] = useState(importStatus === SecretSyncStatus.Succeeded); + const destinationName = SECRET_SYNC_MAP[destination].name; + + useEffect(() => { + if (importStatus === SecretSyncStatus.Succeeded) { + setTimeout(() => setHide(true), 3000); + } else { + setHide(false); + } + }, [importStatus]); + + const failureMessage = useMemo(() => { + if (importStatus === SecretSyncStatus.Failed) { + if (lastImportMessage) + try { + return JSON.stringify(JSON.parse(lastImportMessage), null, 2); + } catch { + return lastImportMessage; + } + + return "An Unknown Error Occurred."; + } + return null; + }, [importStatus, lastImportMessage]); + + if (!importStatus || hide) return null; + + let variant: BadgeProps["variant"]; + let label: string; + let icon: IconDefinition; + let tooltipContent: ReactNode; + + switch (importStatus) { + case SecretSyncStatus.Pending: + case SecretSyncStatus.Running: + variant = "primary"; + label = "Importing Secrets..."; + tooltipContent = `Importing secrets from ${destinationName}. This may take a moment.`; + icon = faDownload; + + break; + case SecretSyncStatus.Failed: + variant = "danger"; + label = "Failed to Import Secrets"; + icon = faTriangleExclamation; + tooltipContent = ( +
+ {failureMessage && ( +
+
+ +
+ {mini ? "Failed to Import Secrets" : "Failure Reason"} +
+
+
{failureMessage}
+
+ )} +
+ ); + + break; + case SecretSyncStatus.Succeeded: + default: + // only show success for a bit... + if (lastImportedAt && differenceInSeconds(new Date(), lastImportedAt) > 15) return null; + + tooltipContent = "Successfully imported secrets."; + variant = "success"; + label = "Secrets Imported"; + icon = faCheck; + } + + return ( + +
+ + + {!mini && {label}} + +
+ + ); +}; diff --git a/frontend/src/components/secret-syncs/SecretSyncLabel.tsx b/frontend/src/components/secret-syncs/SecretSyncLabel.tsx new file mode 100644 index 000000000..8528c9bc6 --- /dev/null +++ b/frontend/src/components/secret-syncs/SecretSyncLabel.tsx @@ -0,0 +1,22 @@ +import { ReactNode } from "react"; +import { twMerge } from "tailwind-merge"; + +type Props = { + label: string; + children?: ReactNode; + className?: string; + labelClassName?: string; +}; + +export const SecretSyncLabel = ({ label, children, className, labelClassName }: Props) => { + return ( +
+

{label}

+ {children ? ( +

{children}

+ ) : ( +

None

+ )} +
+ ); +}; diff --git a/frontend/src/components/secret-syncs/SecretSyncModalHeader.tsx b/frontend/src/components/secret-syncs/SecretSyncModalHeader.tsx new file mode 100644 index 000000000..d8d07d9c9 --- /dev/null +++ b/frontend/src/components/secret-syncs/SecretSyncModalHeader.tsx @@ -0,0 +1,49 @@ +import { faArrowUpRightFromSquare, faBookOpen } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; + +import { SECRET_SYNC_MAP } from "@app/helpers/secretSyncs"; +import { SecretSync } from "@app/hooks/api/secretSyncs"; + +type Props = { + destination: SecretSync; + isConfigured: boolean; +}; + +export const SecretSyncModalHeader = ({ destination, isConfigured }: Props) => { + const destinationDetails = SECRET_SYNC_MAP[destination]; + + return ( +
+ {`${destinationDetails.name} +
+
+ {destinationDetails.name} Sync + +
+ + Docs + +
+ +
+

+ {isConfigured + ? `Edit ${destinationDetails.name} Sync` + : `Sync secrets to ${destinationDetails.name}`} +

+
+
+ ); +}; diff --git a/frontend/src/components/secret-syncs/SecretSyncRemoveSecretsModal.tsx b/frontend/src/components/secret-syncs/SecretSyncRemoveSecretsModal.tsx new file mode 100644 index 000000000..c8bdeee6d --- /dev/null +++ b/frontend/src/components/secret-syncs/SecretSyncRemoveSecretsModal.tsx @@ -0,0 +1,85 @@ +import { createNotification } from "@app/components/notifications"; +import { Button, Modal, ModalClose, ModalContent } from "@app/components/v2"; +import { SECRET_SYNC_MAP } from "@app/helpers/secretSyncs"; +import { TSecretSync, useTriggerSecretSyncRemoveSecrets } from "@app/hooks/api/secretSyncs"; + +type Props = { + secretSync?: TSecretSync; + isOpen: boolean; + onOpenChange: (isOpen: boolean) => void; +}; + +type ContentProps = { + secretSync: TSecretSync; + onComplete: () => void; +}; + +const Content = ({ secretSync, onComplete }: ContentProps) => { + const { id: syncId, destination } = secretSync; + const destinationName = SECRET_SYNC_MAP[destination].name; + + const triggerSyncImport = useTriggerSecretSyncRemoveSecrets(); + + const handleTriggerRemoveSecrets = async () => { + try { + await triggerSyncImport.mutateAsync({ + syncId, + destination + }); + + createNotification({ + text: `Successfully triggered secret removal for ${destinationName} Sync`, + type: "success" + }); + + onComplete(); + } catch (err) { + console.error(err); + + createNotification({ + text: `Failed to trigger secret removal for ${destinationName} Sync`, + type: "error" + }); + } + }; + + return ( + <> +

+ Are you sure you want to remove synced secrets from this {destinationName} destination? +

+
+ + + + +
+ + ); +}; + +export const SecretSyncRemoveSecretsModal = ({ isOpen, onOpenChange, secretSync }: Props) => { + if (!secretSync) return null; + + const destinationName = SECRET_SYNC_MAP[secretSync.destination].name; + + return ( + + + onOpenChange(false)} /> + + + ); +}; diff --git a/frontend/src/components/secret-syncs/SecretSyncRemoveStatusBadge.tsx b/frontend/src/components/secret-syncs/SecretSyncRemoveStatusBadge.tsx new file mode 100644 index 000000000..81d6eab30 --- /dev/null +++ b/frontend/src/components/secret-syncs/SecretSyncRemoveStatusBadge.tsx @@ -0,0 +1,112 @@ +import { ReactNode, useEffect, useMemo, useState } from "react"; +import { + faCheck, + faEraser, + faTriangleExclamation, + faXmark, + IconDefinition +} from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { differenceInSeconds } from "date-fns"; +import { twMerge } from "tailwind-merge"; + +import { Badge, Tooltip } from "@app/components/v2"; +import { BadgeProps } from "@app/components/v2/Badge/Badge"; +import { SECRET_SYNC_MAP } from "@app/helpers/secretSyncs"; +import { SecretSyncStatus, TSecretSync } from "@app/hooks/api/secretSyncs"; + +type Props = { + secretSync: TSecretSync; + className?: string; + mini?: boolean; +}; + +export const SecretSyncRemoveStatusBadge = ({ secretSync, className, mini }: Props) => { + const { removeStatus, lastRemoveMessage, lastRemovedAt, destination } = secretSync; + const [hide, setHide] = useState(removeStatus === SecretSyncStatus.Succeeded); + const destinationName = SECRET_SYNC_MAP[destination].name; + + useEffect(() => { + if (removeStatus === SecretSyncStatus.Succeeded) { + setTimeout(() => setHide(true), 3000); + } else { + setHide(false); + } + }, [removeStatus]); + + const failureMessage = useMemo(() => { + if (removeStatus === SecretSyncStatus.Failed) { + if (lastRemoveMessage) + try { + return JSON.stringify(JSON.parse(lastRemoveMessage), null, 2); + } catch { + return lastRemoveMessage; + } + + return "An Unknown Error Occurred."; + } + return null; + }, [removeStatus, lastRemoveMessage]); + + if (!removeStatus || hide) return null; + + let variant: BadgeProps["variant"]; + let label: string; + let icon: IconDefinition; + let tooltipContent: ReactNode; + + switch (removeStatus) { + case SecretSyncStatus.Pending: + case SecretSyncStatus.Running: + variant = "primary"; + label = "Removing Secrets..."; + tooltipContent = `Removing secrets from ${destinationName}. This may take a moment.`; + icon = faEraser; + + break; + case SecretSyncStatus.Failed: + variant = "danger"; + label = "Failed to Remove Secrets"; + icon = faTriangleExclamation; + tooltipContent = ( +
+ {failureMessage && ( +
+
+ +
+ {mini ? "Failed to Remove Secrets" : "Failure Reason"} +
+
+
{failureMessage}
+
+ )} +
+ ); + + break; + case SecretSyncStatus.Succeeded: + default: + // only show success for a bit... + if (lastRemovedAt && differenceInSeconds(new Date(), lastRemovedAt) > 15) return null; + + tooltipContent = "Successfully removed secrets."; + variant = "success"; + label = "Secrets Removed"; + icon = faCheck; + } + + return ( + +
+ + + {!mini && {label}} + +
+ + ); +}; diff --git a/frontend/src/components/secret-syncs/SecretSyncSelect.tsx b/frontend/src/components/secret-syncs/SecretSyncSelect.tsx new file mode 100644 index 000000000..6e3030a01 --- /dev/null +++ b/frontend/src/components/secret-syncs/SecretSyncSelect.tsx @@ -0,0 +1,88 @@ +import { faWrench } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; + +import { Spinner, Tooltip } from "@app/components/v2"; +import { SECRET_SYNC_MAP } from "@app/helpers/secretSyncs"; +import { SecretSync, useSecretSyncOptions } from "@app/hooks/api/secretSyncs"; + +type Props = { + onSelect: (destination: SecretSync) => void; +}; + +export const SecretSyncSelect = ({ onSelect }: Props) => { + const { isLoading, data: secretSyncOptions } = useSecretSyncOptions(); + + if (isLoading) { + return ( +
+ +

Loading options...

+
+ ); + } + + return ( +
+ {secretSyncOptions?.map(({ destination }) => { + const { image, name } = SECRET_SYNC_MAP[destination]; + return ( + + ); + })} + +

Infisical is constantly adding support for more services.

+

+ {`If you don't see the third-party + service you're looking for,`}{" "} + + let us know on Slack + {" "} + or{" "} + + make a request on GitHub + + . +

+ + } + > +
+ +
+ Coming Soon +
+
+ +
+ ); +}; diff --git a/frontend/src/components/secret-syncs/SecretSyncStatusBadge.tsx b/frontend/src/components/secret-syncs/SecretSyncStatusBadge.tsx new file mode 100644 index 000000000..dbf543f61 --- /dev/null +++ b/frontend/src/components/secret-syncs/SecretSyncStatusBadge.tsx @@ -0,0 +1,47 @@ +import { + faCheck, + faExclamationTriangle, + faRotate, + IconDefinition +} from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; + +import { Badge, BadgeProps } from "@app/components/v2/Badge/Badge"; +import { SecretSyncStatus } from "@app/hooks/api/secretSyncs"; + +type Props = { + status: SecretSyncStatus; +} & Omit; + +export const SecretSyncStatusBadge = ({ status }: Props) => { + let variant: BadgeProps["variant"]; + let text: string; + let icon: IconDefinition; + + switch (status) { + case SecretSyncStatus.Failed: + variant = "danger"; + text = "Failed to Sync"; + icon = faExclamationTriangle; + break; + case SecretSyncStatus.Succeeded: + variant = "success"; + text = "Synced"; + icon = faCheck; + break; + case SecretSyncStatus.Pending: // no need to differentiate from user perspective + case SecretSyncStatus.Running: + default: + variant = "primary"; + text = "Syncing"; + icon = faRotate; + break; + } + + return ( + + + {text} + + ); +}; diff --git a/frontend/src/components/secret-syncs/forms/CreateSecretSyncForm.tsx b/frontend/src/components/secret-syncs/forms/CreateSecretSyncForm.tsx new file mode 100644 index 000000000..e7721f113 --- /dev/null +++ b/frontend/src/components/secret-syncs/forms/CreateSecretSyncForm.tsx @@ -0,0 +1,235 @@ +import { useState } from "react"; +import { Controller, FormProvider, useForm } from "react-hook-form"; +import { Tab } from "@headlessui/react"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { twMerge } from "tailwind-merge"; + +import { createNotification } from "@app/components/notifications"; +import { Button, Checkbox, FormControl, Switch } from "@app/components/v2"; +import { useWorkspace } from "@app/context"; +import { SECRET_SYNC_MAP } from "@app/helpers/secretSyncs"; +import { + SecretSync, + SecretSyncInitialSyncBehavior, + TSecretSync, + useCreateSecretSync, + useSecretSyncOption +} from "@app/hooks/api/secretSyncs"; + +import { SecretSyncFormSchema, TSecretSyncForm } from "./schemas"; +import { SecretSyncDestinationFields } from "./SecretSyncDestinationFields"; +import { SecretSyncDetailsFields } from "./SecretSyncDetailsFields"; +import { SecretSyncOptionsFields } from "./SecretSyncOptionsFields"; +import { SecretSyncReviewFields } from "./SecretSyncReviewFields"; +import { SecretSyncSourceFields } from "./SecretSyncSourceFields"; + +type Props = { + onComplete: (secretSync: TSecretSync) => void; + destination: SecretSync; + onCancel: () => void; +}; + +const FORM_TABS: { name: string; key: string; fields: (keyof TSecretSyncForm)[] }[] = [ + { name: "Source", key: "source", fields: ["secretPath", "environment"] }, + { name: "Destination", key: "destination", fields: ["connection", "destinationConfig"] }, + { name: "Options", key: "options", fields: ["syncOptions"] }, + { name: "Details", key: "details", fields: ["name", "description"] }, + { name: "Review", key: "review", fields: [] } +]; + +export const CreateSecretSyncForm = ({ destination, onComplete, onCancel }: Props) => { + const createSecretSync = useCreateSecretSync(); + const { currentWorkspace } = useWorkspace(); + const { name: destinationName } = SECRET_SYNC_MAP[destination]; + + const [selectedTabIndex, setSelectedTabIndex] = useState(0); + const [confirmOverwrite, setConfirmOverwrite] = useState(false); + + const { syncOption } = useSecretSyncOption(destination); + + const formMethods = useForm({ + resolver: zodResolver(SecretSyncFormSchema), + defaultValues: { + destination, + isAutoSyncEnabled: true, + syncOptions: { + initialSyncBehavior: syncOption?.canImportSecrets + ? undefined + : SecretSyncInitialSyncBehavior.OverwriteDestination + } + }, + reValidateMode: "onChange" + }); + + const onSubmit = async ({ environment, connection, ...formData }: TSecretSyncForm) => { + try { + const secretSync = await createSecretSync.mutateAsync({ + ...formData, + connectionId: connection.id, + environment: environment.slug, + projectId: currentWorkspace.id + }); + + createNotification({ + text: `Successfully added ${destinationName} Sync`, + type: "success" + }); + onComplete(secretSync); + } catch (err: any) { + console.error(err); + createNotification({ + title: `Failed to add ${destinationName} Sync`, + text: err.message, + type: "error" + }); + } + }; + + const handlePrev = () => { + if (selectedTabIndex === 0) { + onCancel(); + return; + } + + setSelectedTabIndex((prev) => prev - 1); + }; + + const { handleSubmit, trigger, watch, control } = formMethods; + + const isStepValid = async (index: number) => trigger(FORM_TABS[index].fields); + + const isFinalStep = selectedTabIndex === FORM_TABS.length - 1; + + const handleNext = async () => { + if (isFinalStep) { + handleSubmit(onSubmit)(); + return; + } + + const isValid = await isStepValid(selectedTabIndex); + + if (!isValid) return; + + setSelectedTabIndex((prev) => prev + 1); + }; + + const isTabEnabled = async (index: number) => { + let isEnabled = true; + for (let i = index - 1; i >= 0; i -= 1) { + // eslint-disable-next-line no-await-in-loop + isEnabled = isEnabled && (await isStepValid(i)); + } + + return isEnabled; + }; + + const initialSyncBehavior = watch("syncOptions.initialSyncBehavior"); + + return ( +
+ + + + {FORM_TABS.map((tab, index) => ( + { + e.preventDefault(); + const isEnabled = await isTabEnabled(index); + setSelectedTabIndex((prev) => (isEnabled ? index : prev)); + }} + className={({ selected }) => + `w-30 -mb-[0.14rem] ${index > selectedTabIndex ? "opacity-30" : ""} px-4 py-2 text-sm font-medium outline-none disabled:opacity-60 ${ + selected + ? "border-b-2 border-mineshaft-300 text-mineshaft-200" + : "text-bunker-300" + }` + } + key={tab.key} + > + {index + 1}. {tab.name} + + ))} + + + + + + + + + + + { + return ( + + +

Auto-Sync {value ? "Enabled" : "Disabled"}

+ + + ); + }} + /> + + + + + + + + + + + {isFinalStep && + initialSyncBehavior === SecretSyncInitialSyncBehavior.OverwriteDestination && ( + setConfirmOverwrite(Boolean(isChecked))} + > +

+ I understand all secrets present in the configured {destinationName} destination will + be removed if they are not present within Infisical. +

+ + )} +
+ + {selectedTabIndex > 0 && ( + + )} +
+
+ ); +}; diff --git a/frontend/src/components/secret-syncs/forms/EditSecretSyncForm.tsx b/frontend/src/components/secret-syncs/forms/EditSecretSyncForm.tsx new file mode 100644 index 000000000..d9306c671 --- /dev/null +++ b/frontend/src/components/secret-syncs/forms/EditSecretSyncForm.tsx @@ -0,0 +1,105 @@ +import { ReactNode } from "react"; +import { FormProvider, useForm } from "react-hook-form"; +import { zodResolver } from "@hookform/resolvers/zod"; + +import { createNotification } from "@app/components/notifications"; +import { SecretSyncEditFields } from "@app/components/secret-syncs/types"; +import { Button, ModalClose } from "@app/components/v2"; +import { SECRET_SYNC_MAP } from "@app/helpers/secretSyncs"; +import { TSecretSync, useUpdateSecretSync } from "@app/hooks/api/secretSyncs"; + +import { TSecretSyncForm, UpdateSecretSyncFormSchema } from "./schemas"; +import { SecretSyncDestinationFields } from "./SecretSyncDestinationFields"; +import { SecretSyncDetailsFields } from "./SecretSyncDetailsFields"; +import { SecretSyncOptionsFields } from "./SecretSyncOptionsFields"; +import { SecretSyncSourceFields } from "./SecretSyncSourceFields"; + +type Props = { + onComplete: (secretSync: TSecretSync) => void; + secretSync: TSecretSync; + fields: SecretSyncEditFields; +}; + +export const EditSecretSyncForm = ({ secretSync, fields, onComplete }: Props) => { + const updateSecretSync = useUpdateSecretSync(); + const { name: destinationName } = SECRET_SYNC_MAP[secretSync.destination]; + + const formMethods = useForm({ + resolver: zodResolver(UpdateSecretSyncFormSchema), + defaultValues: { + ...secretSync, + environment: secretSync.environment ?? undefined, + secretPath: secretSync.folder?.path, + description: secretSync.description ?? "" + }, + reValidateMode: "onChange" + }); + + const onSubmit = async ({ environment, connection, ...formData }: TSecretSyncForm) => { + try { + const updatedSecretSync = await updateSecretSync.mutateAsync({ + syncId: secretSync.id, + ...formData, + environment: environment?.slug, + connectionId: connection.id + }); + + createNotification({ + text: `Successfully updated ${destinationName} Sync`, + type: "success" + }); + onComplete(updatedSecretSync); + } catch (err: any) { + console.error(err); + createNotification({ + title: `Failed to update ${destinationName} Sync`, + text: err.message, + type: "error" + }); + } + }; + + let Component: ReactNode; + + switch (fields) { + case SecretSyncEditFields.Destination: + Component = ; + break; + case SecretSyncEditFields.Options: + Component = ; + break; + case SecretSyncEditFields.Source: + Component = ; + break; + case SecretSyncEditFields.Details: + default: + Component = ; + break; + } + + const { + handleSubmit, + formState: { isSubmitting, isDirty } + } = formMethods; + + return ( +
+ {Component} +
+ + + + +
+
+ ); +}; diff --git a/frontend/src/components/secret-syncs/forms/SecretSyncConnectionField.tsx b/frontend/src/components/secret-syncs/forms/SecretSyncConnectionField.tsx new file mode 100644 index 000000000..408e4af67 --- /dev/null +++ b/frontend/src/components/secret-syncs/forms/SecretSyncConnectionField.tsx @@ -0,0 +1,90 @@ +import { Controller, useFormContext } from "react-hook-form"; +import { faInfoCircle } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { Link } from "@tanstack/react-router"; + +import { FilterableSelect, FormControl } from "@app/components/v2"; +import { OrgPermissionSubjects, useOrgPermission } from "@app/context"; +import { OrgPermissionAppConnectionActions } from "@app/context/OrgPermissionContext/types"; +import { APP_CONNECTION_MAP } from "@app/helpers/appConnections"; +import { SECRET_SYNC_CONNECTION_MAP } from "@app/helpers/secretSyncs"; +import { useListAvailableAppConnections } from "@app/hooks/api/appConnections"; + +import { TSecretSyncForm } from "./schemas"; + +type Props = { + onChange?: VoidFunction; +}; + +export const SecretSyncConnectionField = ({ onChange: callback }: Props) => { + const { permission } = useOrgPermission(); + const { control, watch } = useFormContext(); + + const destination = watch("destination"); + const app = SECRET_SYNC_CONNECTION_MAP[destination]; + + const { data: options, isLoading } = useListAvailableAppConnections(app); + + const connectionName = APP_CONNECTION_MAP[app].name; + + const canCreateConnection = permission.can( + OrgPermissionAppConnectionActions.Create, + OrgPermissionSubjects.AppConnections + ); + + const appName = APP_CONNECTION_MAP[SECRET_SYNC_CONNECTION_MAP[destination]].name; + + return ( + <> +

+ Specify the {appName} Connection to use to connect to {connectionName} and configure + destination parameters. +

+ ( + + { + onChange(newValue); + if (callback) callback(); + }} + isLoading={isLoading} + options={options} + placeholder="Select connection..." + getOptionLabel={(option) => option.name} + getOptionValue={(option) => option.id} + /> + + )} + control={control} + name="connection" + /> + {options?.length === 0 && ( +

+ + {canCreateConnection ? ( + <> + You do not have access to any {appName} Connections. Create one from the{" "} + + Organization Settings + {" "} + page. + + ) : ( + `You do not have access to any ${appName} Connections. Contact an admin to create one.` + )} +

+ )} + + ); +}; diff --git a/frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/AwsParameterStoreSyncFields.tsx b/frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/AwsParameterStoreSyncFields.tsx new file mode 100644 index 000000000..a77b0cc04 --- /dev/null +++ b/frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/AwsParameterStoreSyncFields.tsx @@ -0,0 +1,67 @@ +import { Controller, useFormContext } from "react-hook-form"; +import { components, OptionProps, SingleValue } from "react-select"; +import { faCheckCircle } from "@fortawesome/free-regular-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; + +import { SecretSyncConnectionField } from "@app/components/secret-syncs/forms/SecretSyncConnectionField"; +import { Badge, FilterableSelect, FormControl, Input } from "@app/components/v2"; +import { AWS_REGIONS } from "@app/helpers/appConnections"; +import { SecretSync } from "@app/hooks/api/secretSyncs"; + +import { TSecretSyncForm } from "../schemas"; + +const Option = ({ isSelected, children, ...props }: OptionProps<(typeof AWS_REGIONS)[number]>) => { + return ( + +
+

{children}

+ + {props.data.slug} + + {isSelected && ( + + )} +
+ + ); +}; + +export const AwsParameterStoreSyncFields = () => { + const { control } = useFormContext< + TSecretSyncForm & { destination: SecretSync.AWSParameterStore } + >(); + + return ( + <> + + ( + + region.slug === value)} + onChange={(option) => + onChange((option as SingleValue<(typeof AWS_REGIONS)[number]>)?.slug) + } + options={AWS_REGIONS} + placeholder="Select region..." + getOptionLabel={(option) => option.name} + getOptionValue={(option) => option.slug} + components={{ Option }} + /> + + )} + control={control} + name="destinationConfig.region" + /> + ( + + + + )} + control={control} + name="destinationConfig.path" + /> + + ); +}; diff --git a/frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/GitHubSyncFields.tsx b/frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/GitHubSyncFields.tsx new file mode 100644 index 000000000..cabda1e66 --- /dev/null +++ b/frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/GitHubSyncFields.tsx @@ -0,0 +1,231 @@ +import { Controller, useFormContext, useWatch } from "react-hook-form"; +import { MultiValue, SingleValue } from "react-select"; + +import { SecretSyncConnectionField } from "@app/components/secret-syncs/forms/SecretSyncConnectionField"; +import { FilterableSelect, FormControl, Select, SelectItem } from "@app/components/v2"; +import { + TGitHubConnectionEnvironment, + TGitHubConnectionOrganization, + TGitHubConnectionRepository, + useGitHubConnectionListEnvironments, + useGitHubConnectionListOrganizations, + useGitHubConnectionListRepositories +} from "@app/hooks/api/appConnections/github"; +import { SecretSync } from "@app/hooks/api/secretSyncs"; +import { + GitHubSyncScope, + GitHubSyncVisibility +} from "@app/hooks/api/secretSyncs/types/github-sync"; + +import { TSecretSyncForm } from "../schemas"; + +export const GitHubSyncFields = () => { + const { control, watch, setValue } = useFormContext< + TSecretSyncForm & { destination: SecretSync.GitHub } + >(); + + const connectionId = useWatch({ name: "connection.id", control }); + const currentScope = watch("destinationConfig.scope"); + const currentVisibility = watch("destinationConfig.visibility"); + const currentOrg = watch("destinationConfig.org"); + const currentRepo = watch("destinationConfig.repo"); + const currentOwner = watch("destinationConfig.owner"); + + const { data: repositories = [], isPending: isRepositoriesPending } = + useGitHubConnectionListRepositories(connectionId, { + enabled: Boolean(connectionId) + }); + + const { data: organizations = [], isPending: isOrganizationsPending } = + useGitHubConnectionListOrganizations(connectionId, { + enabled: Boolean(connectionId && currentScope === GitHubSyncScope.Organization) + }); + + const { data: environments = [], isPending: isEnvironmentsPending } = + useGitHubConnectionListEnvironments( + { + connectionId, + repo: currentRepo, + owner: currentOwner + }, + { + enabled: Boolean( + connectionId && + currentRepo && + currentOwner && + currentScope === GitHubSyncScope.RepositoryEnvironment + ) + } + ); + + return ( + <> + { + setValue("destinationConfig.org", ""); + setValue("destinationConfig.repo", ""); + setValue("destinationConfig.owner", ""); + setValue("destinationConfig.selectedRepositoryIds", undefined); + }} + /> + ( + + + + )} + /> + {currentScope === GitHubSyncScope.Organization && ( + <> + ( + + org.login === value) ?? null} + onChange={(option) => + onChange((option as SingleValue)?.login ?? null) + } + options={organizations} + placeholder="Select an organization..." + getOptionLabel={(option) => option.login} + getOptionValue={(option) => option.login} + /> + + )} + /> + ( + + + + )} + /> + {currentVisibility === GitHubSyncVisibility.Selected && ( + ( + + value?.includes(repo.id))} + onChange={(option) => { + const repos = option as MultiValue; + onChange(repos.map((repo) => repo.id)); + }} + options={repositories.filter((repo) => repo.owner.login === currentOrg)} + placeholder="Select one or more repositories..." + getOptionLabel={(option) => `${option.owner.login}/${option.name}`} + getOptionValue={(option) => option.id.toString()} + /> + + )} + control={control} + name="destinationConfig.selectedRepositoryIds" + /> + )} + + )} + {currentScope !== GitHubSyncScope.Organization && ( + ( + + repo.name === value) ?? null} + onChange={(option) => { + const repo = option as SingleValue; + + onChange(repo?.name); + setValue("destinationConfig.owner", repo?.owner.login ?? ""); + setValue("destinationConfig.env", ""); + }} + options={repositories} + placeholder="Select a repository..." + getOptionLabel={(option) => `${option.owner.login}/${option.name}`} + getOptionValue={(option) => option.id.toString()} + /> + + )} + control={control} + name="destinationConfig.repo" + /> + )} + {currentScope === GitHubSyncScope.RepositoryEnvironment && ( + ( + + env.name === value) ?? null} + onChange={(option) => + onChange((option as SingleValue)?.name ?? null) + } + options={environments} + placeholder="Select an environment..." + getOptionLabel={(option) => option.name} + getOptionValue={(option) => option.id.toString()} + /> + + )} + /> + )} + + ); +}; diff --git a/frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/SecretSyncDestinationFields.tsx b/frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/SecretSyncDestinationFields.tsx new file mode 100644 index 000000000..8edca89fb --- /dev/null +++ b/frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/SecretSyncDestinationFields.tsx @@ -0,0 +1,22 @@ +import { useFormContext } from "react-hook-form"; + +import { SecretSync } from "@app/hooks/api/secretSyncs"; + +import { TSecretSyncForm } from "../schemas"; +import { AwsParameterStoreSyncFields } from "./AwsParameterStoreSyncFields"; +import { GitHubSyncFields } from "./GitHubSyncFields"; + +export const SecretSyncDestinationFields = () => { + const { watch } = useFormContext(); + + const destination = watch("destination"); + + switch (destination) { + case SecretSync.AWSParameterStore: + return ; + case SecretSync.GitHub: + return ; + default: + throw new Error(`Unhandled Destination Config Field: ${destination}`); + } +}; diff --git a/frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/index.ts b/frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/index.ts new file mode 100644 index 000000000..1d68e01c6 --- /dev/null +++ b/frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/index.ts @@ -0,0 +1 @@ +export * from "./SecretSyncDestinationFields"; diff --git a/frontend/src/components/secret-syncs/forms/SecretSyncDetailsFields.tsx b/frontend/src/components/secret-syncs/forms/SecretSyncDetailsFields.tsx new file mode 100644 index 000000000..8af512bfa --- /dev/null +++ b/frontend/src/components/secret-syncs/forms/SecretSyncDetailsFields.tsx @@ -0,0 +1,51 @@ +import { Controller, useFormContext } from "react-hook-form"; + +import { FormControl, Input, TextArea } from "@app/components/v2"; + +import { TSecretSyncForm } from "./schemas"; + +export const SecretSyncDetailsFields = () => { + const { control } = useFormContext(); + + return ( + <> +

+ Provide a name and description for this Secret Sync. +

+ ( + + + + )} + control={control} + name="name" + /> + ( + +