From e6ed1231cd99d946009273aa72bc52627cf538b8 Mon Sep 17 00:00:00 2001 From: ShubhamPalriwala Date: Wed, 12 Jun 2024 10:24:36 +0530 Subject: [PATCH 01/14] feat: custom rate limit for self hosters --- backend/src/@types/fastify.d.ts | 2 + backend/src/@types/knex.d.ts | 4 + ...327_custom-rate-limits-for-self-hosting.ts | 28 +++ backend/src/db/schemas/index.ts | 1 + backend/src/db/schemas/models.ts | 1 + backend/src/db/schemas/rate-limit.ts | 26 ++ backend/src/server/app.ts | 2 +- backend/src/server/config/rateLimiter.ts | 53 ++-- backend/src/server/routes/index.ts | 7 + backend/src/server/routes/v1/index.ts | 2 + .../src/server/routes/v1/rate-limit-router.ts | 58 +++++ .../src/services/rate-limit/rate-limit-dal.ts | 7 + .../services/rate-limit/rate-limit-service.ts | 44 ++++ .../services/rate-limit/rate-limit-types.ts | 16 ++ frontend/src/hooks/api/rateLimit/index.ts | 2 + frontend/src/hooks/api/rateLimit/mutation.ts | 21 ++ frontend/src/hooks/api/rateLimit/queries.ts | 34 +++ frontend/src/hooks/api/rateLimit/types.ts | 10 + .../admin/DashboardPage/DashboardPage.tsx | 237 +++++++++++++++++- 19 files changed, 535 insertions(+), 20 deletions(-) create mode 100644 backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts create mode 100644 backend/src/db/schemas/rate-limit.ts create mode 100644 backend/src/server/routes/v1/rate-limit-router.ts create mode 100644 backend/src/services/rate-limit/rate-limit-dal.ts create mode 100644 backend/src/services/rate-limit/rate-limit-service.ts create mode 100644 backend/src/services/rate-limit/rate-limit-types.ts create mode 100644 frontend/src/hooks/api/rateLimit/index.ts create mode 100644 frontend/src/hooks/api/rateLimit/mutation.ts create mode 100644 frontend/src/hooks/api/rateLimit/queries.ts create mode 100644 frontend/src/hooks/api/rateLimit/types.ts diff --git a/backend/src/@types/fastify.d.ts b/backend/src/@types/fastify.d.ts index 3f1ca94e9..81fc0c541 100644 --- a/backend/src/@types/fastify.d.ts +++ b/backend/src/@types/fastify.d.ts @@ -48,6 +48,7 @@ import { TProjectEnvServiceFactory } from "@app/services/project-env/project-env import { TProjectKeyServiceFactory } from "@app/services/project-key/project-key-service"; import { TProjectMembershipServiceFactory } from "@app/services/project-membership/project-membership-service"; import { TProjectRoleServiceFactory } from "@app/services/project-role/project-role-service"; +import { TRateLimitServiceFactory } from "@app/services/rate-limit/rate-limit-service"; import { TSecretServiceFactory } from "@app/services/secret/secret-service"; import { TSecretBlindIndexServiceFactory } from "@app/services/secret-blind-index/secret-blind-index-service"; import { TSecretFolderServiceFactory } from "@app/services/secret-folder/secret-folder-service"; @@ -147,6 +148,7 @@ declare module "fastify" { projectUserAdditionalPrivilege: TProjectUserAdditionalPrivilegeServiceFactory; identityProjectAdditionalPrivilege: TIdentityProjectAdditionalPrivilegeServiceFactory; secretSharing: TSecretSharingServiceFactory; + rateLimit: TRateLimitServiceFactory; }; // this is exclusive use for middlewares in which we need to inject data // everywhere else access using service layer diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index 117a74e76..fbcaa3528 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -149,6 +149,9 @@ import { TProjectUserMembershipRoles, TProjectUserMembershipRolesInsert, TProjectUserMembershipRolesUpdate, + TRateLimit, + TRateLimitInsert, + TRateLimitUpdate, TSamlConfigs, TSamlConfigsInsert, TSamlConfigsUpdate, @@ -343,6 +346,7 @@ declare module "knex/types/tables" { TSecretFolderVersionsUpdate >; [TableName.SecretSharing]: Knex.CompositeTableType; + [TableName.RateLimit]: Knex.CompositeTableType; [TableName.SecretTag]: Knex.CompositeTableType; [TableName.SecretImport]: Knex.CompositeTableType; [TableName.Integration]: Knex.CompositeTableType; diff --git a/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts b/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts new file mode 100644 index 000000000..196b8cdb3 --- /dev/null +++ b/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts @@ -0,0 +1,28 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; +import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasTable(TableName.RateLimit))) { + await knex.schema.createTable(TableName.RateLimit, (t) => { + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.integer("readRateLimit").defaultTo(600).notNullable(); + t.integer("writeRateLimit").defaultTo(200).notNullable(); + t.integer("secretsRateLimit").defaultTo(60).notNullable(); + t.integer("authRateLimit").defaultTo(60).notNullable(); + t.integer("inviteUserRateLimit").defaultTo(30).notNullable(); + t.integer("mfaRateLimit").defaultTo(20).notNullable(); + t.integer("creationLimit").defaultTo(30).notNullable(); + t.integer("publicEndpointLimit").defaultTo(30).notNullable(); + t.timestamps(true, true, true); + }); + + await createOnUpdateTrigger(knex, TableName.RateLimit); + } +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists(TableName.RateLimit); + await dropOnUpdateTrigger(knex, TableName.RateLimit); +} diff --git a/backend/src/db/schemas/index.ts b/backend/src/db/schemas/index.ts index 1eaa86c87..5771cb669 100644 --- a/backend/src/db/schemas/index.ts +++ b/backend/src/db/schemas/index.ts @@ -48,6 +48,7 @@ export * from "./project-roles"; export * from "./project-user-additional-privilege"; export * from "./project-user-membership-roles"; export * from "./projects"; +export * from "./rate-limit"; export * from "./saml-configs"; export * from "./scim-tokens"; export * from "./secret-approval-policies"; diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index f9c8436df..5d2213003 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -18,6 +18,7 @@ export enum TableName { IncidentContact = "incident_contacts", UserAction = "user_actions", SuperAdmin = "super_admin", + RateLimit = "rate_limit", ApiKey = "api_keys", Project = "projects", ProjectBot = "project_bots", diff --git a/backend/src/db/schemas/rate-limit.ts b/backend/src/db/schemas/rate-limit.ts new file mode 100644 index 000000000..86b8776cc --- /dev/null +++ b/backend/src/db/schemas/rate-limit.ts @@ -0,0 +1,26 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const RateLimitSchema = z.object({ + id: z.string().uuid(), + readRateLimit: z.number().default(600), + writeRateLimit: z.number().default(200), + secretsRateLimit: z.number().default(60), + authRateLimit: z.number().default(60), + inviteUserRateLimit: z.number().default(30), + mfaRateLimit: z.number().default(20), + creationLimit: z.number().default(30), + publicEndpointLimit: z.number().default(30), + createdAt: z.date(), + updatedAt: z.date() +}); + +export type TRateLimit = z.infer; +export type TRateLimitInsert = Omit, TImmutableDBKeys>; +export type TRateLimitUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index 51cef185a..21721bd5d 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -69,7 +69,7 @@ export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => { // Rate limiters and security headers if (appCfg.isProductionMode) { - await server.register(ratelimiter, globalRateLimiterCfg()); + await server.register(ratelimiter, await globalRateLimiterCfg(db)); } await server.register(helmet, { contentSecurityPolicy: false }); diff --git a/backend/src/server/config/rateLimiter.ts b/backend/src/server/config/rateLimiter.ts index 8c41eb2fa..48d7e41b1 100644 --- a/backend/src/server/config/rateLimiter.ts +++ b/backend/src/server/config/rateLimiter.ts @@ -1,22 +1,10 @@ import type { RateLimitOptions, RateLimitPluginOptions } from "@fastify/rate-limit"; import { Redis } from "ioredis"; +import { Knex } from "knex"; import { getConfig } from "@app/lib/config/env"; - -export const globalRateLimiterCfg = (): RateLimitPluginOptions => { - const appCfg = getConfig(); - const redis = appCfg.isRedisConfigured - ? new Redis(appCfg.REDIS_URL, { connectTimeout: 500, maxRetriesPerRequest: 1 }) - : null; - - return { - timeWindow: 60 * 1000, - max: 600, - redis, - allowList: (req) => req.url === "/healthcheck" || req.url === "/api/status", - keyGenerator: (req) => req.realIp - }; -}; +import { rateLimitDALFactory } from "@app/services/rate-limit/rate-limit-dal"; +import { rateLimitServiceFactory } from "@app/services/rate-limit/rate-limit-service"; // GET endpoints export const readLimit: RateLimitOptions = { @@ -74,3 +62,38 @@ export const publicEndpointLimit: RateLimitOptions = { max: 30, keyGenerator: (req) => req.realIp }; + +async function fetchRateLimitsFromDb(db: Knex) { + try { + const rateLimitDAL = rateLimitDALFactory(db); + const rateLimits = await rateLimitServiceFactory({ rateLimitDAL }).getRateLimits(); + + readLimit.max = rateLimits.readRateLimit; + publicEndpointLimit.max = rateLimits.publicEndpointLimit; + writeLimit.max = rateLimits.writeRateLimit; + secretsLimit.max = rateLimits.secretsRateLimit; + authRateLimit.max = rateLimits.authRateLimit; + inviteUserRateLimit.max = rateLimits.inviteUserRateLimit; + mfaRateLimit.max = rateLimits.mfaRateLimit; + creationLimit.max = rateLimits.creationLimit; + } catch (error) { + console.error("Error fetching rate limits:", error); + } +} + +export const globalRateLimiterCfg = async (db: Knex): Promise => { + const appCfg = getConfig(); + const redis = appCfg.isRedisConfigured + ? new Redis(appCfg.REDIS_URL, { connectTimeout: 500, maxRetriesPerRequest: 1 }) + : null; + + await fetchRateLimitsFromDb(db); + + return { + timeWindow: 60 * 1000, + max: 600, + redis, + allowList: (req) => req.url === "/healthcheck" || req.url === "/api/status", + keyGenerator: (req) => req.realIp + }; +}; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 00590386a..f6fe80efe 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -121,6 +121,8 @@ import { projectMembershipServiceFactory } from "@app/services/project-membershi import { projectUserMembershipRoleDALFactory } from "@app/services/project-membership/project-user-membership-role-dal"; import { projectRoleDALFactory } from "@app/services/project-role/project-role-dal"; import { projectRoleServiceFactory } from "@app/services/project-role/project-role-service"; +import { rateLimitDALFactory } from "@app/services/rate-limit/rate-limit-dal"; +import { rateLimitServiceFactory } from "@app/services/rate-limit/rate-limit-service"; import { dailyResourceCleanUpQueueServiceFactory } from "@app/services/resource-cleanup/resource-cleanup-queue"; import { secretDALFactory } from "@app/services/secret/secret-dal"; import { secretQueueFactory } from "@app/services/secret/secret-queue"; @@ -185,6 +187,7 @@ export const registerRoutes = async ( const incidentContactDAL = incidentContactDALFactory(db); const orgRoleDAL = orgRoleDALFactory(db); const superAdminDAL = superAdminDALFactory(db); + const rateLimitDAL = rateLimitDALFactory(db); const apiKeyDAL = apiKeyDALFactory(db); const projectDAL = projectDALFactory(db); @@ -444,6 +447,9 @@ export const registerRoutes = async ( orgService, keyStore }); + const rateLimitService = rateLimitServiceFactory({ + rateLimitDAL + }); const apiKeyService = apiKeyServiceFactory({ apiKeyDAL, userDAL }); const secretScanningQueue = secretScanningQueueFactory({ @@ -859,6 +865,7 @@ export const registerRoutes = async ( secret: secretService, secretReplication: secretReplicationService, secretTag: secretTagService, + rateLimit: rateLimitService, folder: folderService, secretImport: secretImportService, projectBot: projectBotService, diff --git a/backend/src/server/routes/v1/index.ts b/backend/src/server/routes/v1/index.ts index cbf67ce79..fd8255e63 100644 --- a/backend/src/server/routes/v1/index.ts +++ b/backend/src/server/routes/v1/index.ts @@ -17,6 +17,7 @@ import { registerProjectEnvRouter } from "./project-env-router"; import { registerProjectKeyRouter } from "./project-key-router"; import { registerProjectMembershipRouter } from "./project-membership-router"; import { registerProjectRouter } from "./project-router"; +import { registerRateLimitRouter } from "./rate-limit-router"; import { registerSecretFolderRouter } from "./secret-folder-router"; import { registerSecretImportRouter } from "./secret-import-router"; import { registerSecretSharingRouter } from "./secret-sharing-router"; @@ -43,6 +44,7 @@ export const registerV1Routes = async (server: FastifyZodProvider) => { await server.register(registerPasswordRouter, { prefix: "/password" }); await server.register(registerOrgRouter, { prefix: "/organization" }); await server.register(registerAdminRouter, { prefix: "/admin" }); + await server.register(registerRateLimitRouter, { prefix: "/rate-limit" }); await server.register(registerUserRouter, { prefix: "/user" }); await server.register(registerInviteOrgRouter, { prefix: "/invite-org" }); await server.register(registerUserActionRouter, { prefix: "/user-action" }); diff --git a/backend/src/server/routes/v1/rate-limit-router.ts b/backend/src/server/routes/v1/rate-limit-router.ts new file mode 100644 index 000000000..1ea14d0b4 --- /dev/null +++ b/backend/src/server/routes/v1/rate-limit-router.ts @@ -0,0 +1,58 @@ +import { z } from "zod"; + +import { RateLimitSchema } from "@app/db/schemas"; +import { readLimit } from "@app/server/config/rateLimiter"; +import { verifySuperAdmin } from "@app/server/plugins/auth/superAdmin"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { AuthMode } from "@app/services/auth/auth-type"; + +export const registerRateLimitRouter = async (server: FastifyZodProvider) => { + server.route({ + method: "GET", + url: "/", + config: { + rateLimit: readLimit + }, + schema: { + response: { + 200: z.object({ + rateLimit: RateLimitSchema + }) + } + }, + handler: async () => { + const rateLimit = await server.services.rateLimit.getRateLimits(); + return { rateLimit }; + } + }); + + server.route({ + method: "PATCH", + url: "/", + config: { + rateLimit: readLimit + }, + onRequest: (req, res, done) => { + verifyAuth([AuthMode.JWT, AuthMode.API_KEY])(req, res, () => { + verifySuperAdmin(req, res, done); + }); + }, + + schema: { + body: RateLimitSchema.omit({ + id: true, + createdAt: true, + updatedAt: true + }), + response: { + 200: z.object({ + rateLimit: RateLimitSchema + }) + } + }, + handler: async (req) => { + const rateLimit = await server.services.rateLimit.updateRateLimit(req.body); + return { rateLimit }; + } + }); +}; diff --git a/backend/src/services/rate-limit/rate-limit-dal.ts b/backend/src/services/rate-limit/rate-limit-dal.ts new file mode 100644 index 000000000..7279ff8ea --- /dev/null +++ b/backend/src/services/rate-limit/rate-limit-dal.ts @@ -0,0 +1,7 @@ +import { TDbClient } from "@app/db"; +import { TableName } from "@app/db/schemas"; +import { ormify } from "@app/lib/knex"; + +export type TRateLimitDALFactory = ReturnType; + +export const rateLimitDALFactory = (db: TDbClient) => ormify(db, TableName.RateLimit, {}); diff --git a/backend/src/services/rate-limit/rate-limit-service.ts b/backend/src/services/rate-limit/rate-limit-service.ts new file mode 100644 index 000000000..d510cb056 --- /dev/null +++ b/backend/src/services/rate-limit/rate-limit-service.ts @@ -0,0 +1,44 @@ +import { BadRequestError } from "@app/lib/errors"; + +import { TRateLimitDALFactory } from "./rate-limit-dal"; +import { TRateLimit, TRateLimitUpdateDTO } from "./rate-limit-types"; + +type TRateLimitServiceFactoryDep = { + rateLimitDAL: TRateLimitDALFactory; +}; + +export type TRateLimitServiceFactory = ReturnType; + +export const rateLimitServiceFactory = ({ rateLimitDAL }: TRateLimitServiceFactoryDep) => { + const initRateLimits = async (): Promise => { + const rateLimit = await rateLimitDAL.create({}); + return rateLimit; + }; + + const getRateLimits = async (): Promise => { + let rateLimit = (await rateLimitDAL.find({}))[0]; + if (!rateLimit) { + rateLimit = await initRateLimits(); + } + return rateLimit; + }; + + const updateRateLimit = async (updates: TRateLimitUpdateDTO): Promise => { + const rateLimit = await rateLimitDAL.findOne({}); + if (!rateLimit) throw new BadRequestError({ name: "Rate Limit Update", message: "Rate Limit does not exist yet" }); + + const updateData: Record = {}; + for (const [key, value] of Object.entries(updates)) { + updateData[key] = value; + } + + const updatedRateLimit = await rateLimitDAL.updateById(rateLimit.id, updateData); + return updatedRateLimit; + }; + + return { + initRateLimits, + getRateLimits, + updateRateLimit + }; +}; diff --git a/backend/src/services/rate-limit/rate-limit-types.ts b/backend/src/services/rate-limit/rate-limit-types.ts new file mode 100644 index 000000000..19519aafb --- /dev/null +++ b/backend/src/services/rate-limit/rate-limit-types.ts @@ -0,0 +1,16 @@ +export type TRateLimitUpdateDTO = { + readRateLimit: number; + writeRateLimit: number; + secretsRateLimit: number; + authRateLimit: number; + inviteUserRateLimit: number; + mfaRateLimit: number; + creationLimit: number; + publicEndpointLimit: number; +}; + +export type TRateLimit = { + id: string; + createdAt: Date; + updatedAt: Date; +} & TRateLimitUpdateDTO; diff --git a/frontend/src/hooks/api/rateLimit/index.ts b/frontend/src/hooks/api/rateLimit/index.ts new file mode 100644 index 000000000..f3f81b1c9 --- /dev/null +++ b/frontend/src/hooks/api/rateLimit/index.ts @@ -0,0 +1,2 @@ +export { useUpdateRateLimit } from "./mutation"; +export { useGetRateLimit } from "./queries"; diff --git a/frontend/src/hooks/api/rateLimit/mutation.ts b/frontend/src/hooks/api/rateLimit/mutation.ts new file mode 100644 index 000000000..e2cccf78f --- /dev/null +++ b/frontend/src/hooks/api/rateLimit/mutation.ts @@ -0,0 +1,21 @@ +import { useMutation, useQueryClient } from "@tanstack/react-query"; + +import { apiRequest } from "@app/config/request"; + +import { rateLimitQueryKeys } from "./queries"; +import { TRateLimit } from "./types"; + +export const useUpdateRateLimit = () => { + const queryClient = useQueryClient(); + + return useMutation>({ + mutationFn: async (opt) => { + const { data } = await apiRequest.patch<{ rateLimit: TRateLimit }>("/api/v1/rate-limit", opt); + return data.rateLimit; + }, + onSuccess: (data) => { + queryClient.setQueryData(rateLimitQueryKeys.rateLimit(), data); + queryClient.invalidateQueries(rateLimitQueryKeys.rateLimit()); + } + }); +}; diff --git a/frontend/src/hooks/api/rateLimit/queries.ts b/frontend/src/hooks/api/rateLimit/queries.ts new file mode 100644 index 000000000..5a52ff74b --- /dev/null +++ b/frontend/src/hooks/api/rateLimit/queries.ts @@ -0,0 +1,34 @@ +import { useQuery, UseQueryOptions } from "@tanstack/react-query"; + +import { apiRequest } from "@app/config/request"; + +import { TRateLimit } from "./types"; + +export const rateLimitQueryKeys = { + rateLimit: () => ["rate-limit"] as const +}; + +const fetchRateLimit = async () => { + const { data } = await apiRequest.get<{ rateLimit: TRateLimit }>("/api/v1/rate-limit"); + return data.rateLimit; +}; + +export const useGetRateLimit = ({ + options = {} +}: { + options?: Omit< + UseQueryOptions< + TRateLimit, + unknown, + TRateLimit, + ReturnType + >, + "queryKey" | "queryFn" + >; +} = {}) => + useQuery({ + queryKey: rateLimitQueryKeys.rateLimit(), + queryFn: fetchRateLimit, + ...options, + enabled: options?.enabled ?? true + }); diff --git a/frontend/src/hooks/api/rateLimit/types.ts b/frontend/src/hooks/api/rateLimit/types.ts new file mode 100644 index 000000000..5697fc298 --- /dev/null +++ b/frontend/src/hooks/api/rateLimit/types.ts @@ -0,0 +1,10 @@ +export type TRateLimit = { + readRateLimit: number; + writeRateLimit: number; + secretsRateLimit: number; + authRateLimit: number; + inviteUserRateLimit: number; + mfaRateLimit: number; + creationLimit: number; + publicEndpointLimit: number; +}; diff --git a/frontend/src/views/admin/DashboardPage/DashboardPage.tsx b/frontend/src/views/admin/DashboardPage/DashboardPage.tsx index ce1b117f3..ae243c130 100644 --- a/frontend/src/views/admin/DashboardPage/DashboardPage.tsx +++ b/frontend/src/views/admin/DashboardPage/DashboardPage.tsx @@ -18,12 +18,15 @@ import { Tab, TabList, TabPanel, - Tabs} from "@app/components/v2"; + Tabs +} from "@app/components/v2"; import { useOrganization, useServerConfig, useUser } from "@app/context"; import { useUpdateServerConfig } from "@app/hooks/api"; +import { useGetRateLimit, useUpdateRateLimit } from "@app/hooks/api/rateLimit"; enum TabSections { - Settings = "settings" + Settings = "settings", + RateLimit = "rate-limit" } enum SignUpModes { @@ -35,13 +38,22 @@ const formSchema = z.object({ signUpMode: z.nativeEnum(SignUpModes), allowedSignUpDomain: z.string().optional().nullable(), trustSamlEmails: z.boolean(), - trustLdapEmails: z.boolean() + trustLdapEmails: z.boolean(), + readRateLimit: z.number(), + writeRateLimit: z.number(), + secretsRateLimit: z.number(), + authRateLimit: z.number(), + inviteUserRateLimit: z.number(), + mfaRateLimit: z.number(), + creationLimit: z.number(), + publicEndpointLimit: z.number() }); type TDashboardForm = z.infer; export const AdminDashboardPage = () => { const router = useRouter(); const data = useServerConfig(); + const { data: rateLimit } = useGetRateLimit(); const { config } = data; const { @@ -56,7 +68,15 @@ export const AdminDashboardPage = () => { signUpMode: config.allowSignUp ? SignUpModes.Anyone : SignUpModes.Disabled, allowedSignUpDomain: config.allowedSignUpDomain, trustSamlEmails: config.trustSamlEmails, - trustLdapEmails: config.trustLdapEmails + trustLdapEmails: config.trustLdapEmails, + readRateLimit: rateLimit?.readRateLimit ?? 600, + writeRateLimit: rateLimit?.writeRateLimit ?? 200, + secretsRateLimit: rateLimit?.secretsRateLimit ?? 60, + authRateLimit: rateLimit?.authRateLimit ?? 60, + inviteUserRateLimit: rateLimit?.inviteUserRateLimit ?? 30, + mfaRateLimit: rateLimit?.mfaRateLimit ?? 20, + creationLimit: rateLimit?.creationLimit ?? 30, + publicEndpointLimit: rateLimit?.publicEndpointLimit ?? 30 } }); @@ -65,6 +85,7 @@ export const AdminDashboardPage = () => { const { user, isLoading: isUserLoading } = useUser(); const { orgs } = useOrganization(); const { mutateAsync: updateServerConfig } = useUpdateServerConfig(); + const { mutateAsync: updateRateLimit } = useUpdateRateLimit(); const isNotAllowed = !user?.superAdmin; @@ -101,6 +122,42 @@ export const AdminDashboardPage = () => { } }; + const onRateLimitFormSubmit = async (formData: TDashboardForm) => { + try { + const { + readRateLimit, + writeRateLimit, + secretsRateLimit, + authRateLimit, + inviteUserRateLimit, + mfaRateLimit, + creationLimit, + publicEndpointLimit + } = formData; + + await updateRateLimit({ + readRateLimit, + writeRateLimit, + secretsRateLimit, + authRateLimit, + inviteUserRateLimit, + mfaRateLimit, + creationLimit, + publicEndpointLimit + }); + createNotification({ + text: "Successfully changed rate limits. Please restart your server", + type: "success" + }); + } catch (e) { + console.error(e); + createNotification({ + type: "error", + text: "Failed to update rate limiting setting." + }); + } + }; + return (
@@ -117,6 +174,7 @@ export const AdminDashboardPage = () => {
General + Rate Limit
@@ -233,6 +291,177 @@ export const AdminDashboardPage = () => { + +
+
+
+ Set Rate Limits for your Infisical Instance +
+ ( + + field.onChange(Number(e.target.value))} + /> + + )} + /> + ( + + field.onChange(Number(e.target.value))} + /> + + )} + /> + ( + + field.onChange(Number(e.target.value))} + /> + + )} + /> + ( + + field.onChange(Number(e.target.value))} + /> + + )} + /> + ( + + field.onChange(Number(e.target.value))} + /> + + )} + /> + ( + + field.onChange(Number(e.target.value))} + /> + + )} + /> + ( + + field.onChange(Number(e.target.value))} + /> + + )} + /> + ( + + field.onChange(Number(e.target.value))} + /> + + )} + /> +
+ +
+
)} From 9e24050f1798130b61cbd4c8ab34a2bc3e1a84c0 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Thu, 13 Jun 2024 15:42:15 +0800 Subject: [PATCH 02/14] misc: addressed review comments --- ...327_custom-rate-limits-for-self-hosting.ts | 2 +- backend/src/server/app.ts | 7 +- backend/src/server/config/rateLimiter.ts | 33 +-- .../src/server/routes/v1/rate-limit-router.ts | 13 +- .../services/rate-limit/rate-limit-service.ts | 25 +- frontend/src/hooks/api/index.tsx | 1 + .../admin/DashboardPage/DashboardPage.tsx | 231 +--------------- .../admin/DashboardPage/RateLimitPanel.tsx | 248 ++++++++++++++++++ 8 files changed, 287 insertions(+), 273 deletions(-) create mode 100644 frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx diff --git a/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts b/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts index 196b8cdb3..32bb55a61 100644 --- a/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts +++ b/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts @@ -6,7 +6,7 @@ import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils"; export async function up(knex: Knex): Promise { if (!(await knex.schema.hasTable(TableName.RateLimit))) { await knex.schema.createTable(TableName.RateLimit, (t) => { - t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); + t.uuid("id", { primaryKey: true }).defaultTo("00000000-0000-0000-0000-000000000000"); t.integer("readRateLimit").defaultTo(600).notNullable(); t.integer("writeRateLimit").defaultTo(200).notNullable(); t.integer("secretsRateLimit").defaultTo(60).notNullable(); diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index 21721bd5d..fe94d7ac0 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -17,6 +17,8 @@ import { Logger } from "pino"; import { TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig } from "@app/lib/config/env"; import { TQueueServiceFactory } from "@app/queue"; +import { rateLimitDALFactory } from "@app/services/rate-limit/rate-limit-dal"; +import { rateLimitServiceFactory } from "@app/services/rate-limit/rate-limit-service"; import { TSmtpService } from "@app/services/smtp/smtp-service"; import { globalRateLimiterCfg } from "./config/rateLimiter"; @@ -69,8 +71,11 @@ export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => { // Rate limiters and security headers if (appCfg.isProductionMode) { - await server.register(ratelimiter, await globalRateLimiterCfg(db)); + const rateLimitDAL = rateLimitDALFactory(db); + const rateLimits = await rateLimitServiceFactory({ rateLimitDAL }).getRateLimits(); + await server.register(ratelimiter, globalRateLimiterCfg(rateLimits)); } + await server.register(helmet, { contentSecurityPolicy: false }); await server.register(maintenanceMode); diff --git a/backend/src/server/config/rateLimiter.ts b/backend/src/server/config/rateLimiter.ts index 48d7e41b1..cc62fd08c 100644 --- a/backend/src/server/config/rateLimiter.ts +++ b/backend/src/server/config/rateLimiter.ts @@ -1,10 +1,8 @@ import type { RateLimitOptions, RateLimitPluginOptions } from "@fastify/rate-limit"; import { Redis } from "ioredis"; -import { Knex } from "knex"; +import { TRateLimit } from "@app/db/schemas"; import { getConfig } from "@app/lib/config/env"; -import { rateLimitDALFactory } from "@app/services/rate-limit/rate-limit-dal"; -import { rateLimitServiceFactory } from "@app/services/rate-limit/rate-limit-service"; // GET endpoints export const readLimit: RateLimitOptions = { @@ -63,31 +61,20 @@ export const publicEndpointLimit: RateLimitOptions = { keyGenerator: (req) => req.realIp }; -async function fetchRateLimitsFromDb(db: Knex) { - try { - const rateLimitDAL = rateLimitDALFactory(db); - const rateLimits = await rateLimitServiceFactory({ rateLimitDAL }).getRateLimits(); - - readLimit.max = rateLimits.readRateLimit; - publicEndpointLimit.max = rateLimits.publicEndpointLimit; - writeLimit.max = rateLimits.writeRateLimit; - secretsLimit.max = rateLimits.secretsRateLimit; - authRateLimit.max = rateLimits.authRateLimit; - inviteUserRateLimit.max = rateLimits.inviteUserRateLimit; - mfaRateLimit.max = rateLimits.mfaRateLimit; - creationLimit.max = rateLimits.creationLimit; - } catch (error) { - console.error("Error fetching rate limits:", error); - } -} - -export const globalRateLimiterCfg = async (db: Knex): Promise => { +export const globalRateLimiterCfg = async (rateLimits: TRateLimit): Promise => { const appCfg = getConfig(); const redis = appCfg.isRedisConfigured ? new Redis(appCfg.REDIS_URL, { connectTimeout: 500, maxRetriesPerRequest: 1 }) : null; - await fetchRateLimitsFromDb(db); + readLimit.max = rateLimits.readRateLimit; + publicEndpointLimit.max = rateLimits.publicEndpointLimit; + writeLimit.max = rateLimits.writeRateLimit; + secretsLimit.max = rateLimits.secretsRateLimit; + authRateLimit.max = rateLimits.authRateLimit; + inviteUserRateLimit.max = rateLimits.inviteUserRateLimit; + mfaRateLimit.max = rateLimits.mfaRateLimit; + creationLimit.max = rateLimits.creationLimit; return { timeWindow: 60 * 1000, diff --git a/backend/src/server/routes/v1/rate-limit-router.ts b/backend/src/server/routes/v1/rate-limit-router.ts index 1ea14d0b4..be38afb73 100644 --- a/backend/src/server/routes/v1/rate-limit-router.ts +++ b/backend/src/server/routes/v1/rate-limit-router.ts @@ -39,10 +39,15 @@ export const registerRateLimitRouter = async (server: FastifyZodProvider) => { }, schema: { - body: RateLimitSchema.omit({ - id: true, - createdAt: true, - updatedAt: true + body: z.object({ + readRateLimit: z.number().optional(), + writeRateLimit: z.number().optional(), + secretsRateLimit: z.number().optional(), + authRateLimit: z.number().optional(), + inviteUserRateLimit: z.number().optional(), + mfaRateLimit: z.number().optional(), + creationLimit: z.number().optional(), + publicEndpointLimit: z.number().optional() }), response: { 200: z.object({ diff --git a/backend/src/services/rate-limit/rate-limit-service.ts b/backend/src/services/rate-limit/rate-limit-service.ts index d510cb056..4d4511127 100644 --- a/backend/src/services/rate-limit/rate-limit-service.ts +++ b/backend/src/services/rate-limit/rate-limit-service.ts @@ -10,34 +10,23 @@ type TRateLimitServiceFactoryDep = { export type TRateLimitServiceFactory = ReturnType; export const rateLimitServiceFactory = ({ rateLimitDAL }: TRateLimitServiceFactoryDep) => { - const initRateLimits = async (): Promise => { - const rateLimit = await rateLimitDAL.create({}); - return rateLimit; - }; - const getRateLimits = async (): Promise => { - let rateLimit = (await rateLimitDAL.find({}))[0]; - if (!rateLimit) { - rateLimit = await initRateLimits(); - } - return rateLimit; + return rateLimitDAL.findOne({ id: "00000000-0000-0000-0000-000000000000" }); }; const updateRateLimit = async (updates: TRateLimitUpdateDTO): Promise => { - const rateLimit = await rateLimitDAL.findOne({}); - if (!rateLimit) throw new BadRequestError({ name: "Rate Limit Update", message: "Rate Limit does not exist yet" }); + const rateLimit = await rateLimitDAL.findOne({ + id: "00000000-0000-0000-0000-000000000000" + }); - const updateData: Record = {}; - for (const [key, value] of Object.entries(updates)) { - updateData[key] = value; + if (!rateLimit) { + throw new BadRequestError({ name: "Rate Limit Update", message: "Rate Limit does not exist yet" }); } - const updatedRateLimit = await rateLimitDAL.updateById(rateLimit.id, updateData); - return updatedRateLimit; + return rateLimitDAL.updateById(rateLimit.id, updates); }; return { - initRateLimits, getRateLimits, updateRateLimit }; diff --git a/frontend/src/hooks/api/index.tsx b/frontend/src/hooks/api/index.tsx index 61e8cb666..19fd5f594 100644 --- a/frontend/src/hooks/api/index.tsx +++ b/frontend/src/hooks/api/index.tsx @@ -17,6 +17,7 @@ export * from "./keys"; export * from "./ldapConfig"; export * from "./organization"; export * from "./projectUserAdditionalPrivilege"; +export * from "./rateLimit"; export * from "./roles"; export * from "./scim"; export * from "./secretApproval"; diff --git a/frontend/src/views/admin/DashboardPage/DashboardPage.tsx b/frontend/src/views/admin/DashboardPage/DashboardPage.tsx index ae243c130..52fddb22c 100644 --- a/frontend/src/views/admin/DashboardPage/DashboardPage.tsx +++ b/frontend/src/views/admin/DashboardPage/DashboardPage.tsx @@ -22,7 +22,8 @@ import { } from "@app/components/v2"; import { useOrganization, useServerConfig, useUser } from "@app/context"; import { useUpdateServerConfig } from "@app/hooks/api"; -import { useGetRateLimit, useUpdateRateLimit } from "@app/hooks/api/rateLimit"; + +import { RateLimitPanel } from "./RateLimitPanel"; enum TabSections { Settings = "settings", @@ -38,22 +39,13 @@ const formSchema = z.object({ signUpMode: z.nativeEnum(SignUpModes), allowedSignUpDomain: z.string().optional().nullable(), trustSamlEmails: z.boolean(), - trustLdapEmails: z.boolean(), - readRateLimit: z.number(), - writeRateLimit: z.number(), - secretsRateLimit: z.number(), - authRateLimit: z.number(), - inviteUserRateLimit: z.number(), - mfaRateLimit: z.number(), - creationLimit: z.number(), - publicEndpointLimit: z.number() + trustLdapEmails: z.boolean() }); type TDashboardForm = z.infer; export const AdminDashboardPage = () => { const router = useRouter(); const data = useServerConfig(); - const { data: rateLimit } = useGetRateLimit(); const { config } = data; const { @@ -68,15 +60,7 @@ export const AdminDashboardPage = () => { signUpMode: config.allowSignUp ? SignUpModes.Anyone : SignUpModes.Disabled, allowedSignUpDomain: config.allowedSignUpDomain, trustSamlEmails: config.trustSamlEmails, - trustLdapEmails: config.trustLdapEmails, - readRateLimit: rateLimit?.readRateLimit ?? 600, - writeRateLimit: rateLimit?.writeRateLimit ?? 200, - secretsRateLimit: rateLimit?.secretsRateLimit ?? 60, - authRateLimit: rateLimit?.authRateLimit ?? 60, - inviteUserRateLimit: rateLimit?.inviteUserRateLimit ?? 30, - mfaRateLimit: rateLimit?.mfaRateLimit ?? 20, - creationLimit: rateLimit?.creationLimit ?? 30, - publicEndpointLimit: rateLimit?.publicEndpointLimit ?? 30 + trustLdapEmails: config.trustLdapEmails } }); @@ -85,7 +69,6 @@ export const AdminDashboardPage = () => { const { user, isLoading: isUserLoading } = useUser(); const { orgs } = useOrganization(); const { mutateAsync: updateServerConfig } = useUpdateServerConfig(); - const { mutateAsync: updateRateLimit } = useUpdateRateLimit(); const isNotAllowed = !user?.superAdmin; @@ -122,42 +105,6 @@ export const AdminDashboardPage = () => { } }; - const onRateLimitFormSubmit = async (formData: TDashboardForm) => { - try { - const { - readRateLimit, - writeRateLimit, - secretsRateLimit, - authRateLimit, - inviteUserRateLimit, - mfaRateLimit, - creationLimit, - publicEndpointLimit - } = formData; - - await updateRateLimit({ - readRateLimit, - writeRateLimit, - secretsRateLimit, - authRateLimit, - inviteUserRateLimit, - mfaRateLimit, - creationLimit, - publicEndpointLimit - }); - createNotification({ - text: "Successfully changed rate limits. Please restart your server", - type: "success" - }); - } catch (e) { - console.error(e); - createNotification({ - type: "error", - text: "Failed to update rate limiting setting." - }); - } - }; - return (
@@ -292,175 +239,7 @@ export const AdminDashboardPage = () => { -
-
-
- Set Rate Limits for your Infisical Instance -
- ( - - field.onChange(Number(e.target.value))} - /> - - )} - /> - ( - - field.onChange(Number(e.target.value))} - /> - - )} - /> - ( - - field.onChange(Number(e.target.value))} - /> - - )} - /> - ( - - field.onChange(Number(e.target.value))} - /> - - )} - /> - ( - - field.onChange(Number(e.target.value))} - /> - - )} - /> - ( - - field.onChange(Number(e.target.value))} - /> - - )} - /> - ( - - field.onChange(Number(e.target.value))} - /> - - )} - /> - ( - - field.onChange(Number(e.target.value))} - /> - - )} - /> -
- -
+
diff --git a/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx b/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx new file mode 100644 index 000000000..f29a9d52b --- /dev/null +++ b/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx @@ -0,0 +1,248 @@ +import { Controller, useForm } from "react-hook-form"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { z } from "zod"; + +import { createNotification } from "@app/components/notifications"; +import { Button, FormControl, Input } from "@app/components/v2"; +import { useGetRateLimit, useUpdateRateLimit } from "@app/hooks/api"; + +const formSchema = z.object({ + readRateLimit: z.number(), + writeRateLimit: z.number(), + secretsRateLimit: z.number(), + authRateLimit: z.number(), + inviteUserRateLimit: z.number(), + mfaRateLimit: z.number(), + creationLimit: z.number(), + publicEndpointLimit: z.number() +}); + +type TRateLimitForm = z.infer; + +export const RateLimitPanel = () => { + const { data: rateLimit } = useGetRateLimit(); + const { mutateAsync: updateRateLimit } = useUpdateRateLimit(); + + const { + control, + handleSubmit, + formState: { isSubmitting, isDirty } + } = useForm({ + resolver: zodResolver(formSchema), + values: { + // eslint-disable-next-line + readRateLimit: rateLimit?.readRateLimit ?? 600, + writeRateLimit: rateLimit?.writeRateLimit ?? 200, + secretsRateLimit: rateLimit?.secretsRateLimit ?? 60, + authRateLimit: rateLimit?.authRateLimit ?? 60, + inviteUserRateLimit: rateLimit?.inviteUserRateLimit ?? 30, + mfaRateLimit: rateLimit?.mfaRateLimit ?? 20, + creationLimit: rateLimit?.creationLimit ?? 30, + publicEndpointLimit: rateLimit?.publicEndpointLimit ?? 30 + } + }); + + const onRateLimitFormSubmit = async (formData: TRateLimitForm) => { + try { + const { + readRateLimit, + writeRateLimit, + secretsRateLimit, + authRateLimit, + inviteUserRateLimit, + mfaRateLimit, + creationLimit, + publicEndpointLimit + } = formData; + + await updateRateLimit({ + readRateLimit, + writeRateLimit, + secretsRateLimit, + authRateLimit, + inviteUserRateLimit, + mfaRateLimit, + creationLimit, + publicEndpointLimit + }); + createNotification({ + text: "Successfully changed rate limits. Please restart your server", + type: "success" + }); + } catch (e) { + console.error(e); + createNotification({ + type: "error", + text: "Failed to update rate limiting setting." + }); + } + }; + + return ( +
+
+
+ Set Rate Limits for your Infisical Instance +
+ ( + + field.onChange(Number(e.target.value))} + /> + + )} + /> + ( + + field.onChange(Number(e.target.value))} + /> + + )} + /> + ( + + field.onChange(Number(e.target.value))} + /> + + )} + /> + ( + + field.onChange(Number(e.target.value))} + /> + + )} + /> + ( + + field.onChange(Number(e.target.value))} + /> + + )} + /> + ( + + field.onChange(Number(e.target.value))} + /> + + )} + /> + ( + + field.onChange(Number(e.target.value))} + /> + + )} + /> + ( + + field.onChange(Number(e.target.value))} + /> + + )} + /> +
+ +
+ ); +}; From d5658d374ab1a051f0ef075e1a01ec772c029678 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Thu, 13 Jun 2024 16:27:13 +0800 Subject: [PATCH 03/14] misc: finalized backend flow --- ...327_custom-rate-limits-for-self-hosting.ts | 3 +++ backend/src/server/config/rateLimiter.ts | 20 ++++++++------- .../src/server/routes/v1/rate-limit-router.ts | 25 ++++++++++++------- .../services/rate-limit/rate-limit-service.ts | 20 ++++++--------- frontend/src/hooks/api/rateLimit/mutation.ts | 4 +-- 5 files changed, 39 insertions(+), 33 deletions(-) diff --git a/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts b/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts index 32bb55a61..cb8007d9f 100644 --- a/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts +++ b/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts @@ -19,6 +19,9 @@ export async function up(knex: Knex): Promise { }); await createOnUpdateTrigger(knex, TableName.RateLimit); + + // create rate limit entry + await knex(TableName.RateLimit).insert({}); } } diff --git a/backend/src/server/config/rateLimiter.ts b/backend/src/server/config/rateLimiter.ts index cc62fd08c..e5bec1c6c 100644 --- a/backend/src/server/config/rateLimiter.ts +++ b/backend/src/server/config/rateLimiter.ts @@ -61,20 +61,22 @@ export const publicEndpointLimit: RateLimitOptions = { keyGenerator: (req) => req.realIp }; -export const globalRateLimiterCfg = async (rateLimits: TRateLimit): Promise => { +export const globalRateLimiterCfg = async (rateLimits?: TRateLimit): Promise => { const appCfg = getConfig(); const redis = appCfg.isRedisConfigured ? new Redis(appCfg.REDIS_URL, { connectTimeout: 500, maxRetriesPerRequest: 1 }) : null; - readLimit.max = rateLimits.readRateLimit; - publicEndpointLimit.max = rateLimits.publicEndpointLimit; - writeLimit.max = rateLimits.writeRateLimit; - secretsLimit.max = rateLimits.secretsRateLimit; - authRateLimit.max = rateLimits.authRateLimit; - inviteUserRateLimit.max = rateLimits.inviteUserRateLimit; - mfaRateLimit.max = rateLimits.mfaRateLimit; - creationLimit.max = rateLimits.creationLimit; + if (rateLimits) { + readLimit.max = rateLimits.readRateLimit; + publicEndpointLimit.max = rateLimits.publicEndpointLimit; + writeLimit.max = rateLimits.writeRateLimit; + secretsLimit.max = rateLimits.secretsRateLimit; + authRateLimit.max = rateLimits.authRateLimit; + inviteUserRateLimit.max = rateLimits.inviteUserRateLimit; + mfaRateLimit.max = rateLimits.mfaRateLimit; + creationLimit.max = rateLimits.creationLimit; + } return { timeWindow: 60 * 1000, diff --git a/backend/src/server/routes/v1/rate-limit-router.ts b/backend/src/server/routes/v1/rate-limit-router.ts index be38afb73..f2f200228 100644 --- a/backend/src/server/routes/v1/rate-limit-router.ts +++ b/backend/src/server/routes/v1/rate-limit-router.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { RateLimitSchema } from "@app/db/schemas"; +import { BadRequestError } from "@app/lib/errors"; import { readLimit } from "@app/server/config/rateLimiter"; import { verifySuperAdmin } from "@app/server/plugins/auth/superAdmin"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; @@ -22,12 +23,18 @@ export const registerRateLimitRouter = async (server: FastifyZodProvider) => { }, handler: async () => { const rateLimit = await server.services.rateLimit.getRateLimits(); + if (!rateLimit) { + throw new BadRequestError({ + name: "Get Rate Limit Error", + message: "Rate limit configuration does not exist." + }); + } return { rateLimit }; } }); server.route({ - method: "PATCH", + method: "PUT", url: "/", config: { rateLimit: readLimit @@ -40,14 +47,14 @@ export const registerRateLimitRouter = async (server: FastifyZodProvider) => { schema: { body: z.object({ - readRateLimit: z.number().optional(), - writeRateLimit: z.number().optional(), - secretsRateLimit: z.number().optional(), - authRateLimit: z.number().optional(), - inviteUserRateLimit: z.number().optional(), - mfaRateLimit: z.number().optional(), - creationLimit: z.number().optional(), - publicEndpointLimit: z.number().optional() + readRateLimit: z.number(), + writeRateLimit: z.number(), + secretsRateLimit: z.number(), + authRateLimit: z.number(), + inviteUserRateLimit: z.number(), + mfaRateLimit: z.number(), + creationLimit: z.number(), + publicEndpointLimit: z.number() }), response: { 200: z.object({ diff --git a/backend/src/services/rate-limit/rate-limit-service.ts b/backend/src/services/rate-limit/rate-limit-service.ts index 4d4511127..5f1e75b95 100644 --- a/backend/src/services/rate-limit/rate-limit-service.ts +++ b/backend/src/services/rate-limit/rate-limit-service.ts @@ -1,5 +1,3 @@ -import { BadRequestError } from "@app/lib/errors"; - import { TRateLimitDALFactory } from "./rate-limit-dal"; import { TRateLimit, TRateLimitUpdateDTO } from "./rate-limit-types"; @@ -10,20 +8,16 @@ type TRateLimitServiceFactoryDep = { export type TRateLimitServiceFactory = ReturnType; export const rateLimitServiceFactory = ({ rateLimitDAL }: TRateLimitServiceFactoryDep) => { - const getRateLimits = async (): Promise => { - return rateLimitDAL.findOne({ id: "00000000-0000-0000-0000-000000000000" }); + const getRateLimits = async (): Promise => { + try { + return await rateLimitDAL.findOne({ id: "00000000-0000-0000-0000-000000000000" }); + } catch (error) { + return undefined; + } }; const updateRateLimit = async (updates: TRateLimitUpdateDTO): Promise => { - const rateLimit = await rateLimitDAL.findOne({ - id: "00000000-0000-0000-0000-000000000000" - }); - - if (!rateLimit) { - throw new BadRequestError({ name: "Rate Limit Update", message: "Rate Limit does not exist yet" }); - } - - return rateLimitDAL.updateById(rateLimit.id, updates); + return rateLimitDAL.updateById("00000000-0000-0000-0000-000000000000", updates); }; return { diff --git a/frontend/src/hooks/api/rateLimit/mutation.ts b/frontend/src/hooks/api/rateLimit/mutation.ts index e2cccf78f..22a7f9898 100644 --- a/frontend/src/hooks/api/rateLimit/mutation.ts +++ b/frontend/src/hooks/api/rateLimit/mutation.ts @@ -8,9 +8,9 @@ import { TRateLimit } from "./types"; export const useUpdateRateLimit = () => { const queryClient = useQueryClient(); - return useMutation>({ + return useMutation({ mutationFn: async (opt) => { - const { data } = await apiRequest.patch<{ rateLimit: TRateLimit }>("/api/v1/rate-limit", opt); + const { data } = await apiRequest.put<{ rateLimit: TRateLimit }>("/api/v1/rate-limit", opt); return data.rateLimit; }, onSuccess: (data) => { From d0ffb94bc78447ed7ebea63ebc3adbbe7d40ad44 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Thu, 13 Jun 2024 18:19:46 +0800 Subject: [PATCH 04/14] misc: added handling of automatic config sync --- backend/package-lock.json | 15 ++++++ backend/package.json | 1 + backend/src/main.ts | 4 +- backend/src/server/app.ts | 12 ++++- backend/src/server/config/rateLimiter.ts | 47 ++++++++++++------- .../services/rate-limit/rate-limit-service.ts | 34 +++++++++++++- 6 files changed, 91 insertions(+), 22 deletions(-) diff --git a/backend/package-lock.json b/backend/package-lock.json index b6f9a37c1..a2a4d0a6e 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -36,6 +36,7 @@ "bcrypt": "^5.1.1", "bullmq": "^5.4.2", "cassandra-driver": "^4.7.2", + "cron": "^3.1.7", "dotenv": "^16.4.1", "fastify": "^4.26.0", "fastify-plugin": "^4.5.1", @@ -4806,6 +4807,11 @@ "long": "*" } }, + "node_modules/@types/luxon": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.4.2.tgz", + "integrity": "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA==" + }, "node_modules/@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -6689,6 +6695,15 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "node_modules/cron": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/cron/-/cron-3.1.7.tgz", + "integrity": "sha512-tlBg7ARsAMQLzgwqVxy8AZl/qlTc5nibqYwtNGoCrd+cV+ugI+tvZC1oT/8dFH8W455YrywGykx/KMmAqOr7Jw==", + "dependencies": { + "@types/luxon": "~3.4.0", + "luxon": "~3.4.0" + } + }, "node_modules/cron-parser": { "version": "4.9.0", "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", diff --git a/backend/package.json b/backend/package.json index 3d1937964..66e720dcf 100644 --- a/backend/package.json +++ b/backend/package.json @@ -97,6 +97,7 @@ "bcrypt": "^5.1.1", "bullmq": "^5.4.2", "cassandra-driver": "^4.7.2", + "cron": "^3.1.7", "dotenv": "^16.4.1", "fastify": "^4.26.0", "fastify-plugin": "^4.5.1", diff --git a/backend/src/main.ts b/backend/src/main.ts index 86681ef33..5de0cccc4 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -22,12 +22,13 @@ const run = async () => { const queue = queueServiceFactory(appCfg.REDIS_URL); const keyStore = keyStoreFactory(appCfg.REDIS_URL); - const server = await main({ db, smtp, logger, queue, keyStore }); + const { server, jobs } = await main({ db, smtp, logger, queue, keyStore }); const bootstrap = await bootstrapCheck({ db }); // eslint-disable-next-line process.on("SIGINT", async () => { await server.close(); await db.destroy(); + jobs.forEach((job) => job.stop()); process.exit(0); }); @@ -35,6 +36,7 @@ const run = async () => { process.on("SIGTERM", async () => { await server.close(); await db.destroy(); + jobs.forEach((job) => job.stop()); process.exit(0); }); diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index fe94d7ac0..5d94de94c 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -10,6 +10,7 @@ import fastifyFormBody from "@fastify/formbody"; import helmet from "@fastify/helmet"; import type { FastifyRateLimitOptions } from "@fastify/rate-limit"; import ratelimiter from "@fastify/rate-limit"; +import { CronJob } from "cron"; import fasitfy from "fastify"; import { Knex } from "knex"; import { Logger } from "pino"; @@ -41,6 +42,7 @@ type TMain = { // Run the server! export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => { const appCfg = getConfig(); + const cronJobs: CronJob[] = []; const server = fasitfy({ logger: appCfg.NODE_ENV === "test" ? false : logger, trustProxy: true, @@ -72,7 +74,13 @@ export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => { // Rate limiters and security headers if (appCfg.isProductionMode) { const rateLimitDAL = rateLimitDALFactory(db); - const rateLimits = await rateLimitServiceFactory({ rateLimitDAL }).getRateLimits(); + const rateLimitService = rateLimitServiceFactory({ rateLimitDAL }); + const rateLimits = await rateLimitService.getRateLimits(); + + if (rateLimits) { + cronJobs.push(rateLimitService.initializeBackgroundSync()); + } + await server.register(ratelimiter, globalRateLimiterCfg(rateLimits)); } @@ -92,7 +100,7 @@ export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => { await server.ready(); server.swagger(); - return server; + return { server, jobs: cronJobs }; } catch (err) { server.log.error(err); await queue.shutdown(); diff --git a/backend/src/server/config/rateLimiter.ts b/backend/src/server/config/rateLimiter.ts index e5bec1c6c..8e0e17bfe 100644 --- a/backend/src/server/config/rateLimiter.ts +++ b/backend/src/server/config/rateLimiter.ts @@ -4,17 +4,28 @@ import { Redis } from "ioredis"; import { TRateLimit } from "@app/db/schemas"; import { getConfig } from "@app/lib/config/env"; +export const rateLimitMaxConfiguration = { + readLimit: 60, + publicEndpointLimit: 30, + writeLimit: 200, + secretsLimit: 60, + authRateLimit: 60, + inviteUserRateLimit: 30, + mfaRateLimit: 20, + creationLimit: 30 +}; + // GET endpoints export const readLimit: RateLimitOptions = { timeWindow: 60 * 1000, - max: 600, + max: () => rateLimitMaxConfiguration.readLimit, keyGenerator: (req) => req.realIp }; // POST, PATCH, PUT, DELETE endpoints export const writeLimit: RateLimitOptions = { timeWindow: 60 * 1000, - max: 200, // (too low, FA having issues so increasing it - maidul) + max: () => rateLimitMaxConfiguration.writeLimit, // (too low, FA having issues so increasing it - maidul) keyGenerator: (req) => req.realIp }; @@ -22,25 +33,25 @@ export const writeLimit: RateLimitOptions = { export const secretsLimit: RateLimitOptions = { // secrets, folders, secret imports timeWindow: 60 * 1000, - max: 60, + max: () => rateLimitMaxConfiguration.secretsLimit, keyGenerator: (req) => req.realIp }; export const authRateLimit: RateLimitOptions = { timeWindow: 60 * 1000, - max: 60, + max: () => rateLimitMaxConfiguration.authRateLimit, keyGenerator: (req) => req.realIp }; export const inviteUserRateLimit: RateLimitOptions = { timeWindow: 60 * 1000, - max: 30, + max: () => rateLimitMaxConfiguration.inviteUserRateLimit, keyGenerator: (req) => req.realIp }; export const mfaRateLimit: RateLimitOptions = { timeWindow: 60 * 1000, - max: 20, + max: () => rateLimitMaxConfiguration.mfaRateLimit, keyGenerator: (req) => { return req.headers.authorization?.split(" ")[1] || req.realIp; } @@ -49,7 +60,7 @@ export const mfaRateLimit: RateLimitOptions = { export const creationLimit: RateLimitOptions = { // identity, project, org timeWindow: 60 * 1000, - max: 30, + max: () => rateLimitMaxConfiguration.creationLimit, keyGenerator: (req) => req.realIp }; @@ -57,25 +68,25 @@ export const creationLimit: RateLimitOptions = { export const publicEndpointLimit: RateLimitOptions = { // Shared Secrets timeWindow: 60 * 1000, - max: 30, + max: () => rateLimitMaxConfiguration.publicEndpointLimit, keyGenerator: (req) => req.realIp }; -export const globalRateLimiterCfg = async (rateLimits?: TRateLimit): Promise => { +export const globalRateLimiterCfg = async (customRateLimits?: TRateLimit): Promise => { const appCfg = getConfig(); const redis = appCfg.isRedisConfigured ? new Redis(appCfg.REDIS_URL, { connectTimeout: 500, maxRetriesPerRequest: 1 }) : null; - if (rateLimits) { - readLimit.max = rateLimits.readRateLimit; - publicEndpointLimit.max = rateLimits.publicEndpointLimit; - writeLimit.max = rateLimits.writeRateLimit; - secretsLimit.max = rateLimits.secretsRateLimit; - authRateLimit.max = rateLimits.authRateLimit; - inviteUserRateLimit.max = rateLimits.inviteUserRateLimit; - mfaRateLimit.max = rateLimits.mfaRateLimit; - creationLimit.max = rateLimits.creationLimit; + if (customRateLimits) { + rateLimitMaxConfiguration.readLimit = customRateLimits.readRateLimit; + rateLimitMaxConfiguration.publicEndpointLimit = customRateLimits.publicEndpointLimit; + rateLimitMaxConfiguration.writeLimit = customRateLimits.writeRateLimit; + rateLimitMaxConfiguration.secretsLimit = customRateLimits.secretsRateLimit; + rateLimitMaxConfiguration.authRateLimit = customRateLimits.authRateLimit; + rateLimitMaxConfiguration.inviteUserRateLimit = customRateLimits.inviteUserRateLimit; + rateLimitMaxConfiguration.mfaRateLimit = customRateLimits.mfaRateLimit; + rateLimitMaxConfiguration.creationLimit = customRateLimits.creationLimit; } return { diff --git a/backend/src/services/rate-limit/rate-limit-service.ts b/backend/src/services/rate-limit/rate-limit-service.ts index 5f1e75b95..ec69b1e95 100644 --- a/backend/src/services/rate-limit/rate-limit-service.ts +++ b/backend/src/services/rate-limit/rate-limit-service.ts @@ -1,3 +1,8 @@ +import { CronJob } from "cron"; + +import { logger } from "@app/lib/logger"; +import { rateLimitMaxConfiguration } from "@app/server/config/rateLimiter"; + import { TRateLimitDALFactory } from "./rate-limit-dal"; import { TRateLimit, TRateLimitUpdateDTO } from "./rate-limit-types"; @@ -20,8 +25,35 @@ export const rateLimitServiceFactory = ({ rateLimitDAL }: TRateLimitServiceFacto return rateLimitDAL.updateById("00000000-0000-0000-0000-000000000000", updates); }; + const initializeBackgroundSync = () => { + const rateLimitSync = async () => { + try { + const rateLimit = await getRateLimits(); + if (rateLimit) { + rateLimitMaxConfiguration.readLimit = rateLimit.readRateLimit; + rateLimitMaxConfiguration.publicEndpointLimit = rateLimit.publicEndpointLimit; + rateLimitMaxConfiguration.writeLimit = rateLimit.writeRateLimit; + rateLimitMaxConfiguration.secretsLimit = rateLimit.secretsRateLimit; + rateLimitMaxConfiguration.authRateLimit = rateLimit.authRateLimit; + rateLimitMaxConfiguration.inviteUserRateLimit = rateLimit.inviteUserRateLimit; + rateLimitMaxConfiguration.mfaRateLimit = rateLimit.mfaRateLimit; + rateLimitMaxConfiguration.creationLimit = rateLimit.creationLimit; + } + } catch (error) { + logger.error(`Error syncing rate limit configurations: %o`, error); + } + }; + + // sync rate limits configuration every 10 minutes + const job = new CronJob("*/10 * * * *", rateLimitSync); + job.start(); + + return job; + }; + return { getRateLimits, - updateRateLimit + updateRateLimit, + initializeBackgroundSync }; }; From 20fea1e25f6f232abb3dabdfbb7269513ce039f8 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Thu, 13 Jun 2024 18:37:29 +0800 Subject: [PATCH 05/14] misc: added flag to disable rate limit updates via API --- .env.example | 2 ++ backend/src/lib/config/env.ts | 3 ++- backend/src/services/rate-limit/rate-limit-service.ts | 9 +++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 67110d69a..6670a0a3a 100644 --- a/.env.example +++ b/.env.example @@ -67,3 +67,5 @@ CLIENT_SECRET_GITLAB_LOGIN= CAPTCHA_SECRET= NEXT_PUBLIC_CAPTCHA_SITE_KEY= + +ALLOW_RATELIMIT_UPDATES= diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 80a2111fc..7c02ebdbf 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -123,7 +123,8 @@ const envSchema = z .optional(), INFISICAL_CLOUD: zodStrBool.default("false"), MAINTENANCE_MODE: zodStrBool.default("false"), - CAPTCHA_SECRET: zpStr(z.string().optional()) + CAPTCHA_SECRET: zpStr(z.string().optional()), + ALLOW_RATELIMIT_UPDATES: zodStrBool.default("false") }) .transform((data) => ({ ...data, diff --git a/backend/src/services/rate-limit/rate-limit-service.ts b/backend/src/services/rate-limit/rate-limit-service.ts index ec69b1e95..a697f1afd 100644 --- a/backend/src/services/rate-limit/rate-limit-service.ts +++ b/backend/src/services/rate-limit/rate-limit-service.ts @@ -1,5 +1,7 @@ import { CronJob } from "cron"; +import { getConfig } from "@app/lib/config/env"; +import { ForbiddenRequestError } from "@app/lib/errors"; import { logger } from "@app/lib/logger"; import { rateLimitMaxConfiguration } from "@app/server/config/rateLimiter"; @@ -22,6 +24,13 @@ export const rateLimitServiceFactory = ({ rateLimitDAL }: TRateLimitServiceFacto }; const updateRateLimit = async (updates: TRateLimitUpdateDTO): Promise => { + const appCfg = getConfig(); + if (!appCfg.ALLOW_RATELIMIT_UPDATES) { + throw new ForbiddenRequestError({ + name: "Rate limit Updates Disabled", + message: "Changes to rate limits are disabled" + }); + } return rateLimitDAL.updateById("00000000-0000-0000-0000-000000000000", updates); }; From 5cee228f5f3531049500f9f951ab7f3b54f8c783 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Thu, 13 Jun 2024 18:45:15 +0800 Subject: [PATCH 06/14] misc: updated rate limit update message --- frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx b/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx index f29a9d52b..0868367dd 100644 --- a/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx +++ b/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx @@ -66,7 +66,7 @@ export const RateLimitPanel = () => { publicEndpointLimit }); createNotification({ - text: "Successfully changed rate limits. Please restart your server", + text: "Rate limits have been successfully updated. Please allow at least 10 minutes for the changes to take effect.", type: "success" }); } catch (e) { From 4de63b61400b60fe622d298cbd9cb398dcb34ec8 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Thu, 13 Jun 2024 19:00:33 +0800 Subject: [PATCH 07/14] fix: updated test --- backend/e2e-test/vitest-environment-knex.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/e2e-test/vitest-environment-knex.ts b/backend/e2e-test/vitest-environment-knex.ts index 09ab05443..c287172f6 100644 --- a/backend/e2e-test/vitest-environment-knex.ts +++ b/backend/e2e-test/vitest-environment-knex.ts @@ -43,7 +43,7 @@ export default { const smtp = mockSmtpServer(); const queue = mockQueue(); const keyStore = mockKeyStore(); - const server = await main({ db, smtp, logger, queue, keyStore }); + const { server } = await main({ db, smtp, logger, queue, keyStore }); // @ts-expect-error type globalThis.testServer = server; // @ts-expect-error type From 44956c6a374e03a1a3cc66a868cbcfbf7a38ae33 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 14 Jun 2024 00:58:54 +0800 Subject: [PATCH 08/14] misc: reorganized cron structure and removed unnecessary checks --- .env.example | 2 -- backend/e2e-test/vitest-environment-knex.ts | 2 +- ...327_custom-rate-limits-for-self-hosting.ts | 2 +- backend/src/lib/config/env.ts | 3 +-- backend/src/main.ts | 4 +-- backend/src/server/app.ts | 9 +------ backend/src/server/routes/index.ts | 7 +++++ .../services/rate-limit/rate-limit-service.ts | 27 ++++++++++--------- 8 files changed, 27 insertions(+), 29 deletions(-) diff --git a/.env.example b/.env.example index 6670a0a3a..67110d69a 100644 --- a/.env.example +++ b/.env.example @@ -67,5 +67,3 @@ CLIENT_SECRET_GITLAB_LOGIN= CAPTCHA_SECRET= NEXT_PUBLIC_CAPTCHA_SITE_KEY= - -ALLOW_RATELIMIT_UPDATES= diff --git a/backend/e2e-test/vitest-environment-knex.ts b/backend/e2e-test/vitest-environment-knex.ts index c287172f6..09ab05443 100644 --- a/backend/e2e-test/vitest-environment-knex.ts +++ b/backend/e2e-test/vitest-environment-knex.ts @@ -43,7 +43,7 @@ export default { const smtp = mockSmtpServer(); const queue = mockQueue(); const keyStore = mockKeyStore(); - const { server } = await main({ db, smtp, logger, queue, keyStore }); + const server = await main({ db, smtp, logger, queue, keyStore }); // @ts-expect-error type globalThis.testServer = server; // @ts-expect-error type diff --git a/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts b/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts index cb8007d9f..84ad9b560 100644 --- a/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts +++ b/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts @@ -6,7 +6,7 @@ import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils"; export async function up(knex: Knex): Promise { if (!(await knex.schema.hasTable(TableName.RateLimit))) { await knex.schema.createTable(TableName.RateLimit, (t) => { - t.uuid("id", { primaryKey: true }).defaultTo("00000000-0000-0000-0000-000000000000"); + t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid()); t.integer("readRateLimit").defaultTo(600).notNullable(); t.integer("writeRateLimit").defaultTo(200).notNullable(); t.integer("secretsRateLimit").defaultTo(60).notNullable(); diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 7c02ebdbf..80a2111fc 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -123,8 +123,7 @@ const envSchema = z .optional(), INFISICAL_CLOUD: zodStrBool.default("false"), MAINTENANCE_MODE: zodStrBool.default("false"), - CAPTCHA_SECRET: zpStr(z.string().optional()), - ALLOW_RATELIMIT_UPDATES: zodStrBool.default("false") + CAPTCHA_SECRET: zpStr(z.string().optional()) }) .transform((data) => ({ ...data, diff --git a/backend/src/main.ts b/backend/src/main.ts index 5de0cccc4..86681ef33 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -22,13 +22,12 @@ const run = async () => { const queue = queueServiceFactory(appCfg.REDIS_URL); const keyStore = keyStoreFactory(appCfg.REDIS_URL); - const { server, jobs } = await main({ db, smtp, logger, queue, keyStore }); + const server = await main({ db, smtp, logger, queue, keyStore }); const bootstrap = await bootstrapCheck({ db }); // eslint-disable-next-line process.on("SIGINT", async () => { await server.close(); await db.destroy(); - jobs.forEach((job) => job.stop()); process.exit(0); }); @@ -36,7 +35,6 @@ const run = async () => { process.on("SIGTERM", async () => { await server.close(); await db.destroy(); - jobs.forEach((job) => job.stop()); process.exit(0); }); diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index 5d94de94c..da1bca3e2 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -10,7 +10,6 @@ import fastifyFormBody from "@fastify/formbody"; import helmet from "@fastify/helmet"; import type { FastifyRateLimitOptions } from "@fastify/rate-limit"; import ratelimiter from "@fastify/rate-limit"; -import { CronJob } from "cron"; import fasitfy from "fastify"; import { Knex } from "knex"; import { Logger } from "pino"; @@ -42,7 +41,6 @@ type TMain = { // Run the server! export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => { const appCfg = getConfig(); - const cronJobs: CronJob[] = []; const server = fasitfy({ logger: appCfg.NODE_ENV === "test" ? false : logger, trustProxy: true, @@ -76,11 +74,6 @@ export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => { const rateLimitDAL = rateLimitDALFactory(db); const rateLimitService = rateLimitServiceFactory({ rateLimitDAL }); const rateLimits = await rateLimitService.getRateLimits(); - - if (rateLimits) { - cronJobs.push(rateLimitService.initializeBackgroundSync()); - } - await server.register(ratelimiter, globalRateLimiterCfg(rateLimits)); } @@ -100,7 +93,7 @@ export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => { await server.ready(); server.swagger(); - return { server, jobs: cronJobs }; + return server; } catch (err) { server.log.error(err); await queue.shutdown(); diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 0b79085cc..2f2de829f 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -1,3 +1,4 @@ +import { CronJob } from "cron"; import { Knex } from "knex"; import { z } from "zod"; @@ -907,6 +908,11 @@ export const registerRoutes = async ( secretSharing: secretSharingService }); + const cronJobs: CronJob[] = []; + if (appCfg.isProductionMode) { + cronJobs.push(rateLimitService.initializeBackgroundSync()); + } + server.decorate("store", { user: userDAL }); @@ -961,6 +967,7 @@ export const registerRoutes = async ( await server.register(registerV3Routes, { prefix: "/api/v3" }); server.addHook("onClose", async () => { + cronJobs.forEach((job) => job.stop()); await telemetryService.flushAll(); }); }; diff --git a/backend/src/services/rate-limit/rate-limit-service.ts b/backend/src/services/rate-limit/rate-limit-service.ts index a697f1afd..e458a9310 100644 --- a/backend/src/services/rate-limit/rate-limit-service.ts +++ b/backend/src/services/rate-limit/rate-limit-service.ts @@ -1,7 +1,5 @@ import { CronJob } from "cron"; -import { getConfig } from "@app/lib/config/env"; -import { ForbiddenRequestError } from "@app/lib/errors"; import { logger } from "@app/lib/logger"; import { rateLimitMaxConfiguration } from "@app/server/config/rateLimiter"; @@ -15,23 +13,28 @@ type TRateLimitServiceFactoryDep = { export type TRateLimitServiceFactory = ReturnType; export const rateLimitServiceFactory = ({ rateLimitDAL }: TRateLimitServiceFactoryDep) => { + const DEFAULT_RATE_LIMIT_CONFIG_ID = "00000000-0000-0000-0000-000000000000"; + const getRateLimits = async (): Promise => { + let rateLimit: TRateLimit; + try { - return await rateLimitDAL.findOne({ id: "00000000-0000-0000-0000-000000000000" }); - } catch (error) { + rateLimit = await rateLimitDAL.findOne({ id: DEFAULT_RATE_LIMIT_CONFIG_ID }); + if (!rateLimit) { + // rate limit might not exist + rateLimit = await rateLimitDAL.create({ + // @ts-expect-error id is kept as fixed because there should only be one rate limit config per instance + id: DEFAULT_RATE_LIMIT_CONFIG_ID + }); + } + return rateLimit; + } catch (err) { return undefined; } }; const updateRateLimit = async (updates: TRateLimitUpdateDTO): Promise => { - const appCfg = getConfig(); - if (!appCfg.ALLOW_RATELIMIT_UPDATES) { - throw new ForbiddenRequestError({ - name: "Rate limit Updates Disabled", - message: "Changes to rate limits are disabled" - }); - } - return rateLimitDAL.updateById("00000000-0000-0000-0000-000000000000", updates); + return rateLimitDAL.updateById(DEFAULT_RATE_LIMIT_CONFIG_ID, updates); }; const initializeBackgroundSync = () => { From 46b48cea633269ae5e6c7ddc603977df66b31170 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 14 Jun 2024 01:14:26 +0800 Subject: [PATCH 09/14] misc: added loader --- frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx b/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx index 0868367dd..a3c39e7e4 100644 --- a/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx +++ b/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx @@ -3,7 +3,7 @@ import { zodResolver } from "@hookform/resolvers/zod"; import { z } from "zod"; import { createNotification } from "@app/components/notifications"; -import { Button, FormControl, Input } from "@app/components/v2"; +import { Button, ContentLoader, FormControl, Input } from "@app/components/v2"; import { useGetRateLimit, useUpdateRateLimit } from "@app/hooks/api"; const formSchema = z.object({ @@ -20,7 +20,7 @@ const formSchema = z.object({ type TRateLimitForm = z.infer; export const RateLimitPanel = () => { - const { data: rateLimit } = useGetRateLimit(); + const { data: rateLimit, isLoading } = useGetRateLimit(); const { mutateAsync: updateRateLimit } = useUpdateRateLimit(); const { @@ -78,7 +78,9 @@ export const RateLimitPanel = () => { } }; - return ( + return isLoading ? ( + + ) : (
Date: Fri, 14 Jun 2024 02:32:21 +0800 Subject: [PATCH 10/14] misc: migrated to structured singleton pattern --- backend/src/server/app.ts | 4 +- backend/src/server/config/rateLimiter.ts | 42 +++--------- .../services/rate-limit/rate-limit-service.ts | 65 +++++++++++++------ 3 files changed, 56 insertions(+), 55 deletions(-) diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index da1bca3e2..19f5986fb 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -73,8 +73,8 @@ export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => { if (appCfg.isProductionMode) { const rateLimitDAL = rateLimitDALFactory(db); const rateLimitService = rateLimitServiceFactory({ rateLimitDAL }); - const rateLimits = await rateLimitService.getRateLimits(); - await server.register(ratelimiter, globalRateLimiterCfg(rateLimits)); + await rateLimitService.syncRateLimitConfiguration(); + await server.register(ratelimiter, globalRateLimiterCfg()); } await server.register(helmet, { contentSecurityPolicy: false }); diff --git a/backend/src/server/config/rateLimiter.ts b/backend/src/server/config/rateLimiter.ts index 8e0e17bfe..0e406da88 100644 --- a/backend/src/server/config/rateLimiter.ts +++ b/backend/src/server/config/rateLimiter.ts @@ -1,31 +1,20 @@ import type { RateLimitOptions, RateLimitPluginOptions } from "@fastify/rate-limit"; import { Redis } from "ioredis"; -import { TRateLimit } from "@app/db/schemas"; import { getConfig } from "@app/lib/config/env"; - -export const rateLimitMaxConfiguration = { - readLimit: 60, - publicEndpointLimit: 30, - writeLimit: 200, - secretsLimit: 60, - authRateLimit: 60, - inviteUserRateLimit: 30, - mfaRateLimit: 20, - creationLimit: 30 -}; +import { getRateLimiterConfig } from "@app/services/rate-limit/rate-limit-service"; // GET endpoints export const readLimit: RateLimitOptions = { timeWindow: 60 * 1000, - max: () => rateLimitMaxConfiguration.readLimit, + max: () => getRateLimiterConfig().readLimit, keyGenerator: (req) => req.realIp }; // POST, PATCH, PUT, DELETE endpoints export const writeLimit: RateLimitOptions = { timeWindow: 60 * 1000, - max: () => rateLimitMaxConfiguration.writeLimit, // (too low, FA having issues so increasing it - maidul) + max: () => getRateLimiterConfig().writeLimit, keyGenerator: (req) => req.realIp }; @@ -33,25 +22,25 @@ export const writeLimit: RateLimitOptions = { export const secretsLimit: RateLimitOptions = { // secrets, folders, secret imports timeWindow: 60 * 1000, - max: () => rateLimitMaxConfiguration.secretsLimit, + max: () => getRateLimiterConfig().secretsLimit, keyGenerator: (req) => req.realIp }; export const authRateLimit: RateLimitOptions = { timeWindow: 60 * 1000, - max: () => rateLimitMaxConfiguration.authRateLimit, + max: () => getRateLimiterConfig().authRateLimit, keyGenerator: (req) => req.realIp }; export const inviteUserRateLimit: RateLimitOptions = { timeWindow: 60 * 1000, - max: () => rateLimitMaxConfiguration.inviteUserRateLimit, + max: () => getRateLimiterConfig().inviteUserRateLimit, keyGenerator: (req) => req.realIp }; export const mfaRateLimit: RateLimitOptions = { timeWindow: 60 * 1000, - max: () => rateLimitMaxConfiguration.mfaRateLimit, + max: () => getRateLimiterConfig().mfaRateLimit, keyGenerator: (req) => { return req.headers.authorization?.split(" ")[1] || req.realIp; } @@ -60,7 +49,7 @@ export const mfaRateLimit: RateLimitOptions = { export const creationLimit: RateLimitOptions = { // identity, project, org timeWindow: 60 * 1000, - max: () => rateLimitMaxConfiguration.creationLimit, + max: () => getRateLimiterConfig().creationLimit, keyGenerator: (req) => req.realIp }; @@ -68,27 +57,16 @@ export const creationLimit: RateLimitOptions = { export const publicEndpointLimit: RateLimitOptions = { // Shared Secrets timeWindow: 60 * 1000, - max: () => rateLimitMaxConfiguration.publicEndpointLimit, + max: () => getRateLimiterConfig().publicEndpointLimit, keyGenerator: (req) => req.realIp }; -export const globalRateLimiterCfg = async (customRateLimits?: TRateLimit): Promise => { +export const globalRateLimiterCfg = (): RateLimitPluginOptions => { const appCfg = getConfig(); const redis = appCfg.isRedisConfigured ? new Redis(appCfg.REDIS_URL, { connectTimeout: 500, maxRetriesPerRequest: 1 }) : null; - if (customRateLimits) { - rateLimitMaxConfiguration.readLimit = customRateLimits.readRateLimit; - rateLimitMaxConfiguration.publicEndpointLimit = customRateLimits.publicEndpointLimit; - rateLimitMaxConfiguration.writeLimit = customRateLimits.writeRateLimit; - rateLimitMaxConfiguration.secretsLimit = customRateLimits.secretsRateLimit; - rateLimitMaxConfiguration.authRateLimit = customRateLimits.authRateLimit; - rateLimitMaxConfiguration.inviteUserRateLimit = customRateLimits.inviteUserRateLimit; - rateLimitMaxConfiguration.mfaRateLimit = customRateLimits.mfaRateLimit; - rateLimitMaxConfiguration.creationLimit = customRateLimits.creationLimit; - } - return { timeWindow: 60 * 1000, max: 600, diff --git a/backend/src/services/rate-limit/rate-limit-service.ts b/backend/src/services/rate-limit/rate-limit-service.ts index e458a9310..a76102ac2 100644 --- a/backend/src/services/rate-limit/rate-limit-service.ts +++ b/backend/src/services/rate-limit/rate-limit-service.ts @@ -1,11 +1,27 @@ import { CronJob } from "cron"; import { logger } from "@app/lib/logger"; -import { rateLimitMaxConfiguration } from "@app/server/config/rateLimiter"; import { TRateLimitDALFactory } from "./rate-limit-dal"; import { TRateLimit, TRateLimitUpdateDTO } from "./rate-limit-types"; +let rateLimitMaxConfiguration = { + readLimit: 60, + publicEndpointLimit: 30, + writeLimit: 200, + secretsLimit: 60, + authRateLimit: 60, + inviteUserRateLimit: 30, + mfaRateLimit: 20, + creationLimit: 30 +}; + +Object.freeze(rateLimitMaxConfiguration); + +export const getRateLimiterConfig = () => { + return rateLimitMaxConfiguration; +}; + type TRateLimitServiceFactoryDep = { rateLimitDAL: TRateLimitDALFactory; }; @@ -37,27 +53,33 @@ export const rateLimitServiceFactory = ({ rateLimitDAL }: TRateLimitServiceFacto return rateLimitDAL.updateById(DEFAULT_RATE_LIMIT_CONFIG_ID, updates); }; - const initializeBackgroundSync = () => { - const rateLimitSync = async () => { - try { - const rateLimit = await getRateLimits(); - if (rateLimit) { - rateLimitMaxConfiguration.readLimit = rateLimit.readRateLimit; - rateLimitMaxConfiguration.publicEndpointLimit = rateLimit.publicEndpointLimit; - rateLimitMaxConfiguration.writeLimit = rateLimit.writeRateLimit; - rateLimitMaxConfiguration.secretsLimit = rateLimit.secretsRateLimit; - rateLimitMaxConfiguration.authRateLimit = rateLimit.authRateLimit; - rateLimitMaxConfiguration.inviteUserRateLimit = rateLimit.inviteUserRateLimit; - rateLimitMaxConfiguration.mfaRateLimit = rateLimit.mfaRateLimit; - rateLimitMaxConfiguration.creationLimit = rateLimit.creationLimit; - } - } catch (error) { - logger.error(`Error syncing rate limit configurations: %o`, error); - } - }; + const syncRateLimitConfiguration = async () => { + try { + const rateLimit = await getRateLimits(); + if (rateLimit) { + const newRateLimitMaxConfiguration: typeof rateLimitMaxConfiguration = { + readLimit: rateLimit.readRateLimit, + publicEndpointLimit: rateLimit.publicEndpointLimit, + writeLimit: rateLimit.writeRateLimit, + secretsLimit: rateLimit.secretsRateLimit, + authRateLimit: rateLimit.authRateLimit, + inviteUserRateLimit: rateLimit.inviteUserRateLimit, + mfaRateLimit: rateLimit.mfaRateLimit, + creationLimit: rateLimit.creationLimit + }; + logger.info(`Rate limit configuration: %o`, newRateLimitMaxConfiguration); + Object.freeze(newRateLimitMaxConfiguration); + rateLimitMaxConfiguration = newRateLimitMaxConfiguration; + } + } catch (error) { + logger.error(`Error syncing rate limit configurations: %o`, error); + } + }; + + const initializeBackgroundSync = () => { // sync rate limits configuration every 10 minutes - const job = new CronJob("*/10 * * * *", rateLimitSync); + const job = new CronJob("*/10 * * * *", syncRateLimitConfiguration); job.start(); return job; @@ -66,6 +88,7 @@ export const rateLimitServiceFactory = ({ rateLimitDAL }: TRateLimitServiceFacto return { getRateLimits, updateRateLimit, - initializeBackgroundSync + initializeBackgroundSync, + syncRateLimitConfiguration }; }; From 08a77f6ddbf3972cef9ec168b149da69cf81bfb9 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 14 Jun 2024 02:37:01 +0800 Subject: [PATCH 11/14] misc: added missing auth check for rate-limit endpoint --- backend/src/server/routes/v1/rate-limit-router.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/src/server/routes/v1/rate-limit-router.ts b/backend/src/server/routes/v1/rate-limit-router.ts index f2f200228..2b08a0c32 100644 --- a/backend/src/server/routes/v1/rate-limit-router.ts +++ b/backend/src/server/routes/v1/rate-limit-router.ts @@ -21,6 +21,11 @@ export const registerRateLimitRouter = async (server: FastifyZodProvider) => { }) } }, + onRequest: (req, res, done) => { + verifyAuth([AuthMode.JWT])(req, res, () => { + verifySuperAdmin(req, res, done); + }); + }, handler: async () => { const rateLimit = await server.services.rateLimit.getRateLimits(); if (!rateLimit) { @@ -40,7 +45,7 @@ export const registerRateLimitRouter = async (server: FastifyZodProvider) => { rateLimit: readLimit }, onRequest: (req, res, done) => { - verifyAuth([AuthMode.JWT, AuthMode.API_KEY])(req, res, () => { + verifyAuth([AuthMode.JWT])(req, res, () => { verifySuperAdmin(req, res, done); }); }, From c9e56e4e9f833e1229d5711c028bdbcd072d3be9 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 14 Jun 2024 02:46:51 +0800 Subject: [PATCH 12/14] misc: updated rate limit labels --- .../views/admin/DashboardPage/RateLimitPanel.tsx | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx b/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx index a3c39e7e4..85bef017f 100644 --- a/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx +++ b/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx @@ -95,7 +95,7 @@ export const RateLimitPanel = () => { defaultValue={300} render={({ field, fieldState: { error } }) => ( { name="writeRateLimit" render={({ field, fieldState: { error } }) => ( { name="secretsRateLimit" render={({ field, fieldState: { error } }) => ( { name="authRateLimit" render={({ field, fieldState: { error } }) => ( { name="inviteUserRateLimit" render={({ field, fieldState: { error } }) => ( { name="mfaRateLimit" render={({ field, fieldState: { error } }) => ( { name="creationLimit" render={({ field, fieldState: { error } }) => ( { name="publicEndpointLimit" render={({ field, fieldState: { error } }) => ( Date: Fri, 14 Jun 2024 02:55:25 +0800 Subject: [PATCH 13/14] misc: added error log --- backend/src/services/rate-limit/rate-limit-service.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/services/rate-limit/rate-limit-service.ts b/backend/src/services/rate-limit/rate-limit-service.ts index a76102ac2..aac10db08 100644 --- a/backend/src/services/rate-limit/rate-limit-service.ts +++ b/backend/src/services/rate-limit/rate-limit-service.ts @@ -45,6 +45,7 @@ export const rateLimitServiceFactory = ({ rateLimitDAL }: TRateLimitServiceFacto } return rateLimit; } catch (err) { + logger.error("Error fetching rate limits %o", err); return undefined; } }; From fa77dc01df115265eefe1959ee5328c521af2dca Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Thu, 13 Jun 2024 21:01:18 -0400 Subject: [PATCH 14/14] apply nits for rate limits --- ...327_custom-rate-limits-for-self-hosting.ts | 2 +- backend/src/server/config/rateLimiter.ts | 30 +++++++++---------- .../services/rate-limit/rate-limit-service.ts | 2 +- .../admin/DashboardPage/RateLimitPanel.tsx | 18 +++++------ 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts b/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts index 84ad9b560..c34b2d196 100644 --- a/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts +++ b/backend/src/db/migrations/20240611151327_custom-rate-limits-for-self-hosting.ts @@ -20,7 +20,7 @@ export async function up(knex: Knex): Promise { await createOnUpdateTrigger(knex, TableName.RateLimit); - // create rate limit entry + // create init rate limit entry with defaults await knex(TableName.RateLimit).insert({}); } } diff --git a/backend/src/server/config/rateLimiter.ts b/backend/src/server/config/rateLimiter.ts index 0e406da88..819aa617f 100644 --- a/backend/src/server/config/rateLimiter.ts +++ b/backend/src/server/config/rateLimiter.ts @@ -4,6 +4,21 @@ import { Redis } from "ioredis"; import { getConfig } from "@app/lib/config/env"; import { getRateLimiterConfig } from "@app/services/rate-limit/rate-limit-service"; +export const globalRateLimiterCfg = (): RateLimitPluginOptions => { + const appCfg = getConfig(); + const redis = appCfg.isRedisConfigured + ? new Redis(appCfg.REDIS_URL, { connectTimeout: 500, maxRetriesPerRequest: 1 }) + : null; + + return { + timeWindow: 60 * 1000, + max: 600, + redis, + allowList: (req) => req.url === "/healthcheck" || req.url === "/api/status", + keyGenerator: (req) => req.realIp + }; +}; + // GET endpoints export const readLimit: RateLimitOptions = { timeWindow: 60 * 1000, @@ -60,18 +75,3 @@ export const publicEndpointLimit: RateLimitOptions = { max: () => getRateLimiterConfig().publicEndpointLimit, keyGenerator: (req) => req.realIp }; - -export const globalRateLimiterCfg = (): RateLimitPluginOptions => { - const appCfg = getConfig(); - const redis = appCfg.isRedisConfigured - ? new Redis(appCfg.REDIS_URL, { connectTimeout: 500, maxRetriesPerRequest: 1 }) - : null; - - return { - timeWindow: 60 * 1000, - max: 600, - redis, - allowList: (req) => req.url === "/healthcheck" || req.url === "/api/status", - keyGenerator: (req) => req.realIp - }; -}; diff --git a/backend/src/services/rate-limit/rate-limit-service.ts b/backend/src/services/rate-limit/rate-limit-service.ts index aac10db08..742628061 100644 --- a/backend/src/services/rate-limit/rate-limit-service.ts +++ b/backend/src/services/rate-limit/rate-limit-service.ts @@ -69,7 +69,7 @@ export const rateLimitServiceFactory = ({ rateLimitDAL }: TRateLimitServiceFacto creationLimit: rateLimit.creationLimit }; - logger.info(`Rate limit configuration: %o`, newRateLimitMaxConfiguration); + logger.info(`syncRateLimitConfiguration: rate limit configuration: %o`, newRateLimitMaxConfiguration); Object.freeze(newRateLimitMaxConfiguration); rateLimitMaxConfiguration = newRateLimitMaxConfiguration; } diff --git a/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx b/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx index 85bef017f..eacb26203 100644 --- a/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx +++ b/frontend/src/views/admin/DashboardPage/RateLimitPanel.tsx @@ -87,7 +87,7 @@ export const RateLimitPanel = () => { >
- Set Rate Limits for your Infisical Instance + Configure rate limits
{ defaultValue={300} render={({ field, fieldState: { error } }) => ( { name="writeRateLimit" render={({ field, fieldState: { error } }) => ( { name="secretsRateLimit" render={({ field, fieldState: { error } }) => ( { name="authRateLimit" render={({ field, fieldState: { error } }) => ( { name="inviteUserRateLimit" render={({ field, fieldState: { error } }) => ( { name="mfaRateLimit" render={({ field, fieldState: { error } }) => ( { name="creationLimit" render={({ field, fieldState: { error } }) => ( { name="publicEndpointLimit" render={({ field, fieldState: { error } }) => (