From de9cb265e0261d2682b18ca62a10c126b170f8ed Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Wed, 21 Aug 2024 23:56:19 +0400 Subject: [PATCH 01/19] Feat: Redis support for dynamic secrets --- .../dynamic-secret/providers/index.ts | 4 +- .../dynamic-secret/providers/models.ts | 17 +- .../dynamic-secret/providers/redis.ts | 183 ++++++++++ frontend/src/hooks/api/dynamicSecret/types.ts | 92 ++--- .../CreateDynamicSecretForm.tsx | 24 ++ .../RedisInputForm.tsx | 328 ++++++++++++++++++ .../CreateDynamicSecretLease.tsx | 19 + 7 files changed, 625 insertions(+), 42 deletions(-) create mode 100644 backend/src/ee/services/dynamic-secret/providers/redis.ts create mode 100644 frontend/src/views/SecretMainPage/components/ActionBar/CreateDynamicSecretForm/RedisInputForm.tsx diff --git a/backend/src/ee/services/dynamic-secret/providers/index.ts b/backend/src/ee/services/dynamic-secret/providers/index.ts index beb6c428e..c69bd9609 100644 --- a/backend/src/ee/services/dynamic-secret/providers/index.ts +++ b/backend/src/ee/services/dynamic-secret/providers/index.ts @@ -1,10 +1,12 @@ import { AwsIamProvider } from "./aws-iam"; import { CassandraProvider } from "./cassandra"; import { DynamicSecretProviders } from "./models"; +import { RedisDatabaseProvider } from "./redis"; import { SqlDatabaseProvider } from "./sql-database"; export const buildDynamicSecretProviders = () => ({ [DynamicSecretProviders.SqlDatabase]: SqlDatabaseProvider(), [DynamicSecretProviders.Cassandra]: CassandraProvider(), - [DynamicSecretProviders.AwsIam]: AwsIamProvider() + [DynamicSecretProviders.AwsIam]: AwsIamProvider(), + [DynamicSecretProviders.Redis]: RedisDatabaseProvider() }); diff --git a/backend/src/ee/services/dynamic-secret/providers/models.ts b/backend/src/ee/services/dynamic-secret/providers/models.ts index 14b79eeea..a28dfceb9 100644 --- a/backend/src/ee/services/dynamic-secret/providers/models.ts +++ b/backend/src/ee/services/dynamic-secret/providers/models.ts @@ -7,6 +7,17 @@ export enum SqlProviders { MsSQL = "mssql" } +export const DynamicSecretRedisDBSchema = z.object({ + host: z.string().trim().toLowerCase(), + port: z.number(), + username: z.string().trim(), // this is often "default". + password: z.string().trim().optional(), // only required if requirepass is set. + creationStatement: z.string().trim(), + revocationStatement: z.string().trim(), + renewStatement: z.string().trim().optional(), + ca: z.string().optional() +}); + export const DynamicSecretSqlDBSchema = z.object({ client: z.nativeEnum(SqlProviders), host: z.string().trim().toLowerCase(), @@ -47,13 +58,15 @@ export const DynamicSecretAwsIamSchema = z.object({ export enum DynamicSecretProviders { SqlDatabase = "sql-database", Cassandra = "cassandra", - AwsIam = "aws-iam" + AwsIam = "aws-iam", + Redis = "redis" } export const DynamicSecretProviderSchema = z.discriminatedUnion("type", [ z.object({ type: z.literal(DynamicSecretProviders.SqlDatabase), inputs: DynamicSecretSqlDBSchema }), z.object({ type: z.literal(DynamicSecretProviders.Cassandra), inputs: DynamicSecretCassandraSchema }), - z.object({ type: z.literal(DynamicSecretProviders.AwsIam), inputs: DynamicSecretAwsIamSchema }) + z.object({ type: z.literal(DynamicSecretProviders.AwsIam), inputs: DynamicSecretAwsIamSchema }), + z.object({ type: z.literal(DynamicSecretProviders.Redis), inputs: DynamicSecretRedisDBSchema }) ]); export type TDynamicProviderFns = { diff --git a/backend/src/ee/services/dynamic-secret/providers/redis.ts b/backend/src/ee/services/dynamic-secret/providers/redis.ts new file mode 100644 index 000000000..23eb454c5 --- /dev/null +++ b/backend/src/ee/services/dynamic-secret/providers/redis.ts @@ -0,0 +1,183 @@ +/* eslint-disable no-console */ +import handlebars from "handlebars"; +import { Redis } from "ioredis"; +import { customAlphabet } from "nanoid"; +import { z } from "zod"; + +import { getConfig } from "@app/lib/config/env"; +import { BadRequestError } from "@app/lib/errors"; +import { getDbConnectionHost } from "@app/lib/knex"; +import { alphaNumericNanoId } from "@app/lib/nanoid"; + +import { DynamicSecretRedisDBSchema, TDynamicProviderFns } from "./models"; + +const generatePassword = () => { + const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.~!*$#"; + return customAlphabet(charset, 64)(); +}; + +const generateUsername = () => { + return alphaNumericNanoId(32); +}; + +const executeTransactions = async (connection: Redis, commands: string[]): Promise<(string | null)[] | null> => { + // Initiate a transaction + const pipeline = connection.multi(); + + // Add all commands to the pipeline + for (const command of commands) { + const args = command + .split(" ") + .map((arg) => arg.trim()) + .filter((arg) => arg.length > 0); + pipeline.call(args[0], ...args.slice(1)); + } + + // Execute the transaction + const results = await pipeline.exec(); + + if (!results) { + throw new BadRequestError({ message: "Redis transaction failed: No results returned" }); + } + + // Check for errors in the results + const errors = results.filter(([err]) => err !== null); + if (errors.length > 0) { + throw new BadRequestError({ message: "Redis transaction failed with errors" }); + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + return results.map(([_, result]) => result as string | null); +}; + +export const RedisDatabaseProvider = (): TDynamicProviderFns => { + const validateProviderInputs = async (inputs: unknown) => { + const appCfg = getConfig(); + const isCloud = Boolean(appCfg.LICENSE_SERVER_KEY); // quick and dirty way to check if its cloud or not + const dbHost = appCfg.DB_HOST || getDbConnectionHost(appCfg.DB_CONNECTION_URI); + + const providerInputs = await DynamicSecretRedisDBSchema.parseAsync(inputs); + if ( + isCloud && + // localhost + // internal ips + (providerInputs.host === "host.docker.internal" || + providerInputs.host.match(/^10\.\d+\.\d+\.\d+/) || + providerInputs.host.match(/^192\.168\.\d+\.\d+/)) + ) + throw new BadRequestError({ message: "Invalid db host" }); + if (providerInputs.host === "localhost" || providerInputs.host === "127.0.0.1" || dbHost === providerInputs.host) + throw new BadRequestError({ message: "Invalid db host" }); + return providerInputs; + }; + + const getClient = async (providerInputs: z.infer) => { + let connection: Redis | null = null; + try { + connection = new Redis({ + username: providerInputs.username, + host: providerInputs.host, + port: providerInputs.port, + password: providerInputs.password, + ...(providerInputs.ca && { + tls: { + rejectUnauthorized: false, + ca: providerInputs.ca + } + }) + }); + + let result: string; + if (providerInputs.password) { + result = await connection.auth(providerInputs.username, providerInputs.password, () => {}); + } else { + result = await connection.auth(providerInputs.username, () => {}); + } + + if (result !== "OK") { + throw new BadRequestError({ message: `Invalid credentials, Redis returned ${result} status` }); + } + + return connection; + } catch (err) { + if (connection) await connection.quit(); + + throw err; + } + }; + + const validateConnection = async (inputs: unknown) => { + const providerInputs = await validateProviderInputs(inputs); + const connection = await getClient(providerInputs); + + const pingResponse = await connection + .ping() + .then(() => true) + .catch(() => false); + + return pingResponse; + }; + + const create = async (inputs: unknown, expireAt: number) => { + const providerInputs = await validateProviderInputs(inputs); + const connection = await getClient(providerInputs); + + const username = generateUsername(); + const password = generatePassword(); + const expiration = new Date(expireAt).toISOString(); + + const creationStatement = handlebars.compile(providerInputs.creationStatement, { noEscape: true })({ + username, + password, + expiration + }); + + const queries = creationStatement.toString().split(";").filter(Boolean); + + await executeTransactions(connection, queries); + + await connection.quit(); + return { entityId: username, data: { DB_USERNAME: username, DB_PASSWORD: password } }; + }; + + const revoke = async (inputs: unknown, entityId: string) => { + const providerInputs = await validateProviderInputs(inputs); + const connection = await getClient(providerInputs); + + const username = entityId; + + const revokeStatement = handlebars.compile(providerInputs.revocationStatement)({ username }); + const queries = revokeStatement.toString().split(";").filter(Boolean); + + await executeTransactions(connection, queries); + + await connection.quit(); + return { entityId: username }; + }; + + const renew = async (inputs: unknown, entityId: string, expireAt: number) => { + const providerInputs = await validateProviderInputs(inputs); + const connection = await getClient(providerInputs); + + const username = entityId; + const expiration = new Date(expireAt).toISOString(); + + const renewStatement = handlebars.compile(providerInputs.renewStatement)({ username, expiration }); + + if (renewStatement) { + const queries = renewStatement.toString().split(";").filter(Boolean); + await executeTransactions(connection, queries); + } + + await connection.quit(); + return { entityId: username }; + }; + + return { + validateProviderInputs, + validateConnection, + create, + revoke, + renew + }; +}; diff --git a/frontend/src/hooks/api/dynamicSecret/types.ts b/frontend/src/hooks/api/dynamicSecret/types.ts index b987e94cc..82aac574a 100644 --- a/frontend/src/hooks/api/dynamicSecret/types.ts +++ b/frontend/src/hooks/api/dynamicSecret/types.ts @@ -18,7 +18,8 @@ export type TDynamicSecret = { export enum DynamicSecretProviders { SqlDatabase = "sql-database", Cassandra = "cassandra", - AwsIam = "aws-iam" + AwsIam = "aws-iam", + Redis = "redis" } export enum SqlProviders { @@ -30,47 +31,60 @@ export enum SqlProviders { export type TDynamicSecretProvider = | { - type: DynamicSecretProviders.SqlDatabase; - inputs: { - client: SqlProviders; - host: string; - port: number; - database: string; - username: string; - password: string; - creationStatement: string; - revocationStatement: string; - renewStatement?: string; - ca?: string | undefined; - }; - } + type: DynamicSecretProviders.SqlDatabase; + inputs: { + client: SqlProviders; + host: string; + port: number; + database: string; + username: string; + password: string; + creationStatement: string; + revocationStatement: string; + renewStatement?: string; + ca?: string | undefined; + }; + } | { - type: DynamicSecretProviders.Cassandra; - inputs: { - host: string; - port: number; - keyspace?: string; - localDataCenter: string; - username: string; - password: string; - creationStatement: string; - revocationStatement: string; - renewStatement?: string; - ca?: string | undefined; - }; - } + type: DynamicSecretProviders.Cassandra; + inputs: { + host: string; + port: number; + keyspace?: string; + localDataCenter: string; + username: string; + password: string; + creationStatement: string; + revocationStatement: string; + renewStatement?: string; + ca?: string | undefined; + }; + } | { - type: DynamicSecretProviders.AwsIam; - inputs: { - accessKey: string; - secretAccessKey: string; - region: string; - awsPath?: string; - policyDocument?: string; - userGroups?: string; - policyArns?: string; + type: DynamicSecretProviders.AwsIam; + inputs: { + accessKey: string; + secretAccessKey: string; + region: string; + awsPath?: string; + policyDocument?: string; + userGroups?: string; + policyArns?: string; + }; + } + | { + type: DynamicSecretProviders.Redis; + inputs: { + host: string; + port: number; + username: string; + password?: string; + creationStatement: string; + renewStatement?: string; + revocationStatement: string; + ca?: string | undefined; + }; }; - }; export type TCreateDynamicSecretDTO = { projectSlug: string; diff --git a/frontend/src/views/SecretMainPage/components/ActionBar/CreateDynamicSecretForm/CreateDynamicSecretForm.tsx b/frontend/src/views/SecretMainPage/components/ActionBar/CreateDynamicSecretForm/CreateDynamicSecretForm.tsx index 760e845ea..e2a40ea9a 100644 --- a/frontend/src/views/SecretMainPage/components/ActionBar/CreateDynamicSecretForm/CreateDynamicSecretForm.tsx +++ b/frontend/src/views/SecretMainPage/components/ActionBar/CreateDynamicSecretForm/CreateDynamicSecretForm.tsx @@ -9,6 +9,7 @@ import { DynamicSecretProviders } from "@app/hooks/api/dynamicSecret/types"; import { AwsIamInputForm } from "./AwsIamInputForm"; import { CassandraInputForm } from "./CassandraInputForm"; +import { RedisInputForm } from "./RedisInputForm"; import { SqlDatabaseInputForm } from "./SqlDatabaseInputForm"; type Props = { @@ -35,6 +36,11 @@ const DYNAMIC_SECRET_LIST = [ provider: DynamicSecretProviders.Cassandra, title: "Cassandra" }, + { + icon: faDatabase, + provider: DynamicSecretProviders.Redis, + title: "Redis" + }, { icon: faAws, provider: DynamicSecretProviders.AwsIam, @@ -118,6 +124,24 @@ export const CreateDynamicSecretForm = ({ /> )} + {wizardStep === WizardSteps.ProviderInputs && + selectedProvider === DynamicSecretProviders.Redis && ( + + + + )} {wizardStep === WizardSteps.ProviderInputs && selectedProvider === DynamicSecretProviders.Cassandra && ( { + const valMs = ms(val); + if (valMs < 60 * 1000) + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be a greater than 1min" }); + // a day + if (valMs > 24 * 60 * 60 * 1000) + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than a day" }); + }), + maxTTL: z + .string() + .optional() + .superRefine((val, ctx) => { + if (!val) return; + const valMs = ms(val); + if (valMs < 60 * 1000) + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be a greater than 1min" }); + // a day + if (valMs > 24 * 60 * 60 * 1000) + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than a day" }); + }), + name: z.string().refine((val) => val.toLowerCase() === val, "Must be lowercase") +}); +type TForm = z.infer; + +type Props = { + onCompleted: () => void; + onCancel: () => void; + secretPath: string; + projectSlug: string; + environment: string; +}; + +export const RedisInputForm = ({ + onCompleted, + onCancel, + environment, + secretPath, + projectSlug +}: Props) => { + const { + control, + formState: { isSubmitting }, + handleSubmit + } = useForm({ + resolver: zodResolver(formSchema), + defaultValues: { + provider: { + username: "default", + creationStatement: "ACL SETUSER {{username}} on >{{password}} ~* &* +@all", + revocationStatement: "ACL DELUSER {{username}}" + } + } + }); + + const createDynamicSecret = useCreateDynamicSecret(); + + const handleCreateDynamicSecret = async ({ name, maxTTL, provider, defaultTTL }: TForm) => { + // wait till previous request is finished + if (createDynamicSecret.isLoading) return; + try { + await createDynamicSecret.mutateAsync({ + provider: { type: DynamicSecretProviders.Redis, inputs: provider }, + maxTTL, + name, + path: secretPath, + defaultTTL, + projectSlug, + environmentSlug: environment + }); + onCompleted(); + } catch (err) { + createNotification({ + type: "error", + text: "Failed to create dynamic secret" + }); + } + }; + + return ( +
+
+
+
+
+ ( + + + + )} + /> +
+
+ ( + } + isError={Boolean(error?.message)} + errorText={error?.message} + > + + + )} + /> +
+
+ ( + } + isError={Boolean(error?.message)} + errorText={error?.message} + > + + + )} + /> +
+
+
+
+ Configuration +
+
+
+ ( + + + + )} + /> + ( + + + + )} + /> +
+
+ ( + + + + )} + /> + ( + + + + )} + /> +
+
+ ( + + + + )} + /> + + + Modify Redis Statements + + ( + +