From 69297bc16e8d8542f2ba1211d0a49c373612007f Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Mon, 25 Mar 2024 00:49:48 +0530 Subject: [PATCH] feat(server): added limit to leases creation and support for force delete when external system fails to comply --- backend/src/lib/config/env.ts | 1 + .../routes/v1/dynamic-secret-lease-router.ts | 3 ++- .../server/routes/v1/dynamic-secret-router.ts | 3 ++- .../dynamic-secret-lease-dal.ts | 11 +++++++- .../dynamic-secret-lease-queue.ts | 4 +-- .../dynamic-secret-lease-service.ts | 26 +++++++++++++++++-- .../dynamic-secret-lease-types.ts | 1 + .../dynamic-secret/dynamic-secret-service.ts | 26 +++++++++++++++++-- .../dynamic-secret/dynamic-secret-types.ts | 1 + .../dynamic-secret/providers/models.ts | 1 + .../dynamic-secret/providers/sql-database.ts | 12 +++++++++ 11 files changed, 80 insertions(+), 9 deletions(-) diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 6b7c02f6b..93c39a06f 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -18,6 +18,7 @@ const envSchema = z DB_CONNECTION_URI: zpStr(z.string().describe("Postgres database connection string")).default( `postgresql://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}` ), + MAX_LEASE_LIMIT: z.coerce.number().default(10000), DB_ROOT_CERT: zpStr(z.string().describe("Postgres database base64-encoded CA cert").optional()), DB_HOST: zpStr(z.string().describe("Postgres database host").optional()), DB_PORT: zpStr(z.string().describe("Postgres database port").optional()).default("5432"), diff --git a/backend/src/server/routes/v1/dynamic-secret-lease-router.ts b/backend/src/server/routes/v1/dynamic-secret-lease-router.ts index 3c59742dc..530990b6f 100644 --- a/backend/src/server/routes/v1/dynamic-secret-lease-router.ts +++ b/backend/src/server/routes/v1/dynamic-secret-lease-router.ts @@ -62,7 +62,8 @@ export const registerDynamicSecretLeaseRouter = async (server: FastifyZodProvide body: z.object({ projectSlug: z.string().min(1), path: z.string().min(1).trim().default("/").transform(removeTrailingSlash), - environment: z.string().min(1) + environment: z.string().min(1), + isForced: z.boolean().default(false) }), response: { 200: z.object({ diff --git a/backend/src/server/routes/v1/dynamic-secret-router.ts b/backend/src/server/routes/v1/dynamic-secret-router.ts index 4ab7e428f..c1a9564c6 100644 --- a/backend/src/server/routes/v1/dynamic-secret-router.ts +++ b/backend/src/server/routes/v1/dynamic-secret-router.ts @@ -132,7 +132,8 @@ export const registerDynamicSecretRouter = async (server: FastifyZodProvider) => body: z.object({ projectSlug: z.string().min(1), path: z.string().trim().default("/").transform(removeTrailingSlash), - environment: z.string().min(1) + environment: z.string().min(1), + isForced: z.boolean().default(false) }), response: { 200: z.object({ diff --git a/backend/src/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts b/backend/src/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts index e1e9b7514..70a6de5da 100644 --- a/backend/src/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts +++ b/backend/src/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts @@ -10,6 +10,15 @@ export type TDynamicSecretLeaseDALFactory = ReturnType { const orm = ormify(db, TableName.DynamicSecretLease); + const countLeasesForDynamicSecret = async (dynamicSecretId: string, tx?: Knex) => { + try { + const doc = await (tx || db)(TableName.DynamicSecretLease).count("*").where({ dynamicSecretId }).first(); + return parseInt(doc || "0", 10); + } catch (error) { + throw new DatabaseError({ error, name: "DynamicSecretCountLeases" }); + } + }; + const findById = async (id: string, tx?: Knex) => { try { const doc = await (tx || db)(TableName.DynamicSecretLease) @@ -67,5 +76,5 @@ export const dynamicSecretLeaseDALFactory = (db: TDbClient) => { } }; - return { ...orm, findById }; + return { ...orm, findById, countLeasesForDynamicSecret }; }; diff --git a/backend/src/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts b/backend/src/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts index baabce7ec..9bdb1c24e 100644 --- a/backend/src/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts +++ b/backend/src/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts @@ -112,8 +112,8 @@ export const dynamicSecretLeaseQueueServiceFactory = ({ }) ) as object; - await Promise.allSettled(dynamicSecretLeases.map(({ id }) => unsetLeaseRevocation(id))); - await Promise.allSettled( + await Promise.all(dynamicSecretLeases.map(({ id }) => unsetLeaseRevocation(id))); + await Promise.all( dynamicSecretLeases.map(({ externalEntityId }) => selectedProvider.revoke(decryptedStoredInput, externalEntityId) ) diff --git a/backend/src/services/dynamic-secret-lease/dynamic-secret-lease-service.ts b/backend/src/services/dynamic-secret-lease/dynamic-secret-lease-service.ts index 6c214b9fd..fa599cfc7 100644 --- a/backend/src/services/dynamic-secret-lease/dynamic-secret-lease-service.ts +++ b/backend/src/services/dynamic-secret-lease/dynamic-secret-lease-service.ts @@ -4,6 +4,7 @@ import ms from "ms"; import { SecretKeyEncoding } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { getConfig } from "@app/lib/config/env"; import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; import { BadRequestError } from "@app/lib/errors"; @@ -14,6 +15,7 @@ import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TDynamicSecretLeaseDALFactory } from "./dynamic-secret-lease-dal"; import { TDynamicSecretLeaseQueueServiceFactory } from "./dynamic-secret-lease-queue"; import { + DynamicSecretLeaseStatus, TCreateDynamicSecretLeaseDTO, TDeleteDynamicSecretLeaseDTO, TDetailsDynamicSecretLeaseDTO, @@ -53,6 +55,7 @@ export const dynamicSecretLeaseServiceFactory = ({ actorAuthMethod, ttl }: TCreateDynamicSecretLeaseDTO) => { + const appCfg = getConfig(); const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); if (!project) throw new BadRequestError({ message: "Project not found" }); @@ -75,6 +78,10 @@ export const dynamicSecretLeaseServiceFactory = ({ const dynamicSecretCfg = await dynamicSecretDAL.findOne({ slug, folderId: folder.id }); if (!dynamicSecretCfg) throw new BadRequestError({ message: "Dynamic secret not found" }); + const totalLeasesTaken = await dynamicSecretLeaseDAL.countLeasesForDynamicSecret(dynamicSecretCfg.id); + if (totalLeasesTaken >= appCfg.MAX_LEASE_LIMIT) + throw new BadRequestError({ message: `Max lease limit reached. Limit: ${appCfg.MAX_LEASE_LIMIT}` }); + const selectedProvider = dynamicSecretProviders[dynamicSecretCfg.type as DynamicSecretProviders]; const decryptedStoredInput = JSON.parse( infisicalSymmetricDecrypt({ @@ -179,7 +186,8 @@ export const dynamicSecretLeaseServiceFactory = ({ actor, actorId, actorOrgId, - actorAuthMethod + actorAuthMethod, + isForced }: TDeleteDynamicSecretLeaseDTO) => { const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); if (!project) throw new BadRequestError({ message: "Project not found" }); @@ -214,7 +222,21 @@ export const dynamicSecretLeaseServiceFactory = ({ }) ) as object; - await selectedProvider.revoke(decryptedStoredInput, dynamicSecretLease.externalEntityId); + const revokeResponse = await selectedProvider + .revoke(decryptedStoredInput, dynamicSecretLease.externalEntityId) + .catch(async (err) => { + // only propogate this error if forced is false + if (!isForced) return { error: err as Error }; + }); + + if ((revokeResponse as { error?: Error })?.error) { + const { error } = revokeResponse as { error?: Error }; + const deletedDynamicSecretLease = await dynamicSecretLeaseDAL.updateById(dynamicSecretLease.id, { + status: DynamicSecretLeaseStatus.FailedDeletion, + statusDetails: error?.message?.slice(0, 255) + }); + return deletedDynamicSecretLease; + } await dynamicSecretQueueService.unsetLeaseRevocation(dynamicSecretLease.id); const deletedDynamicSecretLease = await dynamicSecretLeaseDAL.deleteById(dynamicSecretLease.id); diff --git a/backend/src/services/dynamic-secret-lease/dynamic-secret-lease-types.ts b/backend/src/services/dynamic-secret-lease/dynamic-secret-lease-types.ts index 6f1a17d78..81e4ed056 100644 --- a/backend/src/services/dynamic-secret-lease/dynamic-secret-lease-types.ts +++ b/backend/src/services/dynamic-secret-lease/dynamic-secret-lease-types.ts @@ -31,6 +31,7 @@ export type TDeleteDynamicSecretLeaseDTO = { path: string; environment: string; projectSlug: string; + isForced?: boolean; } & Omit; export type TRenewDynamicSecretLeaseDTO = { diff --git a/backend/src/services/dynamic-secret/dynamic-secret-service.ts b/backend/src/services/dynamic-secret/dynamic-secret-service.ts index ffa9925f9..4b053bf3b 100644 --- a/backend/src/services/dynamic-secret/dynamic-secret-service.ts +++ b/backend/src/services/dynamic-secret/dynamic-secret-service.ts @@ -25,7 +25,10 @@ type TDynamicSecretServiceFactoryDep = { dynamicSecretDAL: TDynamicSecretDALFactory; dynamicSecretLeaseDAL: Pick; dynamicSecretProviders: Record; - dynamicSecretQueueService: Pick; + dynamicSecretQueueService: Pick< + TDynamicSecretLeaseQueueServiceFactory, + "pruneDynamicSecret" | "unsetLeaseRevocation" + >; folderDAL: Pick; projectDAL: Pick; permissionService: Pick; @@ -80,6 +83,10 @@ export const dynamicSecretServiceFactory = ({ const selectedProvider = dynamicSecretProviders[provider.type]; const inputs = await selectedProvider.validateProviderInputs(provider.inputs); + + const isConnected = await selectedProvider.validateConnection(provider.inputs); + if (!isConnected) throw new BadRequestError({ message: "Provider connection failed" }); + const encryptedInput = infisicalSymmetricEncypt(JSON.stringify(inputs)); const dynamicSecretCfg = await dynamicSecretDAL.create({ type: provider.type, @@ -151,6 +158,10 @@ export const dynamicSecretServiceFactory = ({ ) as object; const newInput = { ...decryptedStoredInput, ...(inputs || {}) }; const updatedInput = await selectedProvider.validateProviderInputs(newInput); + + const isConnected = await selectedProvider.validateConnection(newInput); + if (!isConnected) throw new BadRequestError({ message: "Provider connection failed" }); + const encryptedInput = infisicalSymmetricEncypt(JSON.stringify(updatedInput)); const updatedDynamicCfg = await dynamicSecretDAL.updateById(dynamicSecretCfg.id, { inputIV: encryptedInput.iv, @@ -176,7 +187,8 @@ export const dynamicSecretServiceFactory = ({ projectSlug, slug, path, - environment + environment, + isForced }: TDeleteDynamicSecretDTO) => { const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId); if (!project) throw new BadRequestError({ message: "Project not found" }); @@ -202,6 +214,16 @@ export const dynamicSecretServiceFactory = ({ if (!dynamicSecretCfg) throw new BadRequestError({ message: "Dynamic secret not found" }); const leases = await dynamicSecretLeaseDAL.find({ dynamicSecretId: dynamicSecretCfg.id }); + // when not forced we check with the external system to first remove the things + // we introduce a forced concept because consider the external lease got deleted by some other external like a human or another system + // this allows user to clean up it from infisical + if (isForced) { + // clear all queues for lease revocations + await Promise.all(leases.map(({ id: leaseId }) => dynamicSecretQueueService.unsetLeaseRevocation(leaseId))); + + const deletedDynamicSecretCfg = await dynamicSecretDAL.deleteById(dynamicSecretCfg.id); + return deletedDynamicSecretCfg; + } // if leases exist we should flag it as deleting and then remove leases in background // then delete the main one if (leases.length) { diff --git a/backend/src/services/dynamic-secret/dynamic-secret-types.ts b/backend/src/services/dynamic-secret/dynamic-secret-types.ts index d5b3baca9..5d5159974 100644 --- a/backend/src/services/dynamic-secret/dynamic-secret-types.ts +++ b/backend/src/services/dynamic-secret/dynamic-secret-types.ts @@ -37,6 +37,7 @@ export type TDeleteDynamicSecretDTO = { path: string; environment: string; projectSlug: string; + isForced?: boolean; } & Omit; export type TDetailsDynamicSecretDTO = { diff --git a/backend/src/services/dynamic-secret/providers/models.ts b/backend/src/services/dynamic-secret/providers/models.ts index 3b9eb7e16..2932fd924 100644 --- a/backend/src/services/dynamic-secret/providers/models.ts +++ b/backend/src/services/dynamic-secret/providers/models.ts @@ -27,6 +27,7 @@ export const DynamicSecretProviderSchema = z.discriminatedUnion("type", [ export type TDynamicProviderFns = { create: (inputs: unknown, expireAt: number) => Promise<{ entityId: string; data: unknown }>; + validateConnection: (inputs: unknown) => Promise; validateProviderInputs: (inputs: object) => Promise; revoke: (inputs: unknown, entityId: string) => Promise<{ entityId: string }>; renew: (inputs: unknown, entityId: string, expireAt: number) => Promise<{ entityId: string }>; diff --git a/backend/src/services/dynamic-secret/providers/sql-database.ts b/backend/src/services/dynamic-secret/providers/sql-database.ts index 4cf84e77a..076f00a9a 100644 --- a/backend/src/services/dynamic-secret/providers/sql-database.ts +++ b/backend/src/services/dynamic-secret/providers/sql-database.ts @@ -46,6 +46,17 @@ export const SqlDatabaseProvider = (): TDynamicProviderFns => { return db; }; + const validateConnection = async (inputs: unknown) => { + const providerInputs = await validateProviderInputs(inputs); + const db = await getClient(providerInputs); + const isConnected = await db + .raw("SELECT NOW()") + .then(() => true) + .catch(() => false); + await db.destroy(); + return isConnected; + }; + const create = async (inputs: unknown, expireAt: number) => { const providerInputs = await validateProviderInputs(inputs); const db = await getClient(providerInputs); @@ -94,6 +105,7 @@ export const SqlDatabaseProvider = (): TDynamicProviderFns => { return { validateProviderInputs, + validateConnection, create, revoke, renew