From 8f79d3210ad3d52ed85d775ca1e04b3653cb3c63 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Sat, 24 Aug 2024 01:48:39 +0800 Subject: [PATCH 01/37] feat: added raw template for agent --- cli/agent-config.yaml | 15 +- cli/packages/cmd/agent.go | 27 ++++ .../platforms/infisical-agent.mdx | 146 ++++++++++-------- 3 files changed, 121 insertions(+), 67 deletions(-) diff --git a/cli/agent-config.yaml b/cli/agent-config.yaml index e767fdca5..210c21413 100644 --- a/cli/agent-config.yaml +++ b/cli/agent-config.yaml @@ -11,12 +11,25 @@ sinks: config: path: "access-token" templates: - - source-path: my-dot-ev-secret-template + - template-content: | + {{- with secret "202f04d7-e4cb-43d4-a292-e893712d61fc" "dev" "/" }} + {{- range . }} + {{ .Key }}={{ .Value }} + {{- end }} + {{- end }} + destination-path: my-dot-env-0.env + config: + polling-interval: 60s + execute: + command: docker-compose -f docker-compose.prod.yml down && docker-compose -f docker-compose.prod.yml up -d + + - base64-template-content: e3stIHdpdGggc2VjcmV0ICIyMDJmMDRkNy1lNGNiLTQzZDQtYTI5Mi1lODkzNzEyZDYxZmMiICJkZXYiICIvIiB9fQp7ey0gcmFuZ2UgLiB9fQp7eyAuS2V5IH19PXt7IC5WYWx1ZSB9fQp7ey0gZW5kIH19Cnt7LSBlbmQgfX0= destination-path: my-dot-env.env config: polling-interval: 60s execute: command: docker-compose -f docker-compose.prod.yml down && docker-compose -f docker-compose.prod.yml up -d + - source-path: my-dot-ev-secret-template1 destination-path: my-dot-env-1.env config: diff --git a/cli/packages/cmd/agent.go b/cli/packages/cmd/agent.go index 0c99377ae..118aca119 100644 --- a/cli/packages/cmd/agent.go +++ b/cli/packages/cmd/agent.go @@ -95,6 +95,7 @@ type Template struct { SourcePath string `yaml:"source-path"` Base64TemplateContent string `yaml:"base64-template-content"` DestinationPath string `yaml:"destination-path"` + TemplateContent string `yaml:"template-content"` Config struct { // Configurations for the template PollingInterval string `yaml:"polling-interval"` // How often to poll for changes in the secret @@ -432,6 +433,30 @@ func ProcessBase64Template(templateId int, encodedTemplate string, data interfac return &buf, nil } +func ProcessLiteralTemplate(templateId int, templateString string, data interface{}, accessToken string, existingEtag string, currentEtag *string, dynamicSecretLeaser *DynamicSecretLeaseManager) (*bytes.Buffer, error) { + secretFunction := secretTemplateFunction(accessToken, existingEtag, currentEtag) // TODO: Fix this + dynamicSecretFunction := dynamicSecretTemplateFunction(accessToken, dynamicSecretLeaser, templateId) + funcs := template.FuncMap{ + "secret": secretFunction, + "dynamic_secret": dynamicSecretFunction, + } + + templateName := "literalTemplate" + + tmpl, err := template.New(templateName).Funcs(funcs).Parse(templateString) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return nil, err + } + + return &buf, nil +} + + type AgentManager struct { accessToken string accessTokenTTL time.Duration @@ -820,6 +845,8 @@ func (tm *AgentManager) MonitorSecretChanges(secretTemplate Template, templateId if secretTemplate.SourcePath != "" { processedTemplate, err = ProcessTemplate(templateId, secretTemplate.SourcePath, nil, token, existingEtag, ¤tEtag, tm.dynamicSecretLeases) + } else if secretTemplate.TemplateContent != "" { + processedTemplate, err = ProcessLiteralTemplate(templateId, secretTemplate.TemplateContent, nil, token, existingEtag, ¤tEtag, tm.dynamicSecretLeases) } else { processedTemplate, err = ProcessBase64Template(templateId, secretTemplate.Base64TemplateContent, nil, token, existingEtag, ¤tEtag, tm.dynamicSecretLeases) } diff --git a/docs/integrations/platforms/infisical-agent.mdx b/docs/integrations/platforms/infisical-agent.mdx index 93c82f8fe..4a97bb320 100644 --- a/docs/integrations/platforms/infisical-agent.mdx +++ b/docs/integrations/platforms/infisical-agent.mdx @@ -9,56 +9,60 @@ It eliminates the need to modify application logic by enabling clients to decide ![agent diagram](/images/agent/infisical-agent-diagram.png) ### Key features: + - Token renewal: Automatically authenticates with Infisical and deposits renewed access tokens at specified path for applications to consume - Templating: Renders secrets via user provided templates to desired formats for applications to consume ### Token renewal + The Infisical agent can help manage the life cycle of access tokens. The token renewal process is split into two main components: a `Method`, which is the authentication process suitable for your current setup, and `Sinks`, which are the places where the agent deposits the new access token whenever it receives updates. -When the Infisical Agent is started, it will attempt to obtain a valid access token using the authentication method you have configured. If the agent is unable to fetch a valid token, the agent will keep trying, increasing the time between each attempt. +When the Infisical Agent is started, it will attempt to obtain a valid access token using the authentication method you have configured. If the agent is unable to fetch a valid token, the agent will keep trying, increasing the time between each attempt. Once a access token is successfully fetched, the agent will make sure the access token stays valid, continuing to renew it before it expires. Every time the agent successfully retrieves a new access token, it writes the new token to the Sinks you've configured. - Access tokens can be utilized with Infisical SDKs or directly in API requests to retrieve secrets from Infisical + Access tokens can be utilized with Infisical SDKs or directly in API requests + to retrieve secrets from Infisical ### Templating -The Infisical agent can help deliver formatted secrets to your application in a variety of environments. To achieve this, the agent will retrieve secrets from Infisical, format them using a specified template, and then save these formatted secrets to a designated file path. + +The Infisical agent can help deliver formatted secrets to your application in a variety of environments. To achieve this, the agent will retrieve secrets from Infisical, format them using a specified template, and then save these formatted secrets to a designated file path. Templating process is done through the use of Go language's [text/template feature](https://pkg.go.dev/text/template). Multiple template definitions can be set in the agent configuration file to generate a variety of formatted secret files. -When the agent is started and templates are defined in the agent configuration file, the agent will attempt to acquire a valid access token using the set authentication method outlined in the agent's configuration. +When the agent is started and templates are defined in the agent configuration file, the agent will attempt to acquire a valid access token using the set authentication method outlined in the agent's configuration. If this initial attempt is unsuccessful, the agent will momentarily pauses before continuing to make more attempts. -Once the agent successfully obtains a valid access token, the agent proceeds to fetch the secrets from Infisical using it. +Once the agent successfully obtains a valid access token, the agent proceeds to fetch the secrets from Infisical using it. It then formats these secrets using the user provided templates and writes the formatted data to configured file paths. -## Agent configuration file +## Agent configuration file -To set up the authentication method for token renewal and to define secret templates, the Infisical agent requires a YAML configuration file containing properties defined below. +To set up the authentication method for token renewal and to define secret templates, the Infisical agent requires a YAML configuration file containing properties defined below. While specifying an authentication method is mandatory to start the agent, configuring sinks and secret templates are optional. -| Field | Description | -| ------------------------------------------------| ----------------------------- | -| `infisical.address` | The URL of the Infisical service. Default: `"https://app.infisical.com"`. | -| `auth.type` | The type of authentication method used. Available options: `universal-auth`, `kubernetes`, `azure`, `gcp-id-token`, `gcp-iam`, `aws-iam`| -| `auth.config.identity-id` | The file path where the machine identity id is stored

This field is required when using any of the following auth types: `kubernetes`, `azure`, `gcp-id-token`, `gcp-iam`, or `aws-iam`. | -| `auth.config.service-account-token` | Path to the Kubernetes service account token to use (optional)

Default: `/var/run/secrets/kubernetes.io/serviceaccount/token` | -| `auth.config.service-account-key` | Path to your GCP service account key file. This field is required when using `gcp-iam` auth type.

Please note that the file should be in JSON format. | -| `auth.config.client-id` | The file path where the universal-auth client id is stored. | -| `auth.config.client-secret` | The file path where the universal-auth client secret is stored. | -| `auth.config.remove_client_secret_on_read` | This will instruct the agent to remove the client secret from disk. | -| `sinks[].type` | The type of sink in a list of sinks. Each item specifies a sink type. Currently, only `"file"` type is available. | -| `sinks[].config.path` | The file path where the access token should be stored for each sink in the list. | -| `templates[].source-path` | The path to the template file that should be used to render secrets. | -| `templates[].destination-path` | The path where the rendered secrets from the source template will be saved to. | -| `templates[].config.polling-interval` | How frequently to check for secret changes. Default: `5 minutes` (optional) | -| `templates[].config.execute.command` | The command to execute when secret change is detected (optional) | -| `templates[].config.execute.timeout` | How long in seconds to wait for command to execute before timing out (optional) | - +| Field | Description | +| ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `infisical.address` | The URL of the Infisical service. Default: `"https://app.infisical.com"`. | +| `auth.type` | The type of authentication method used. Available options: `universal-auth`, `kubernetes`, `azure`, `gcp-id-token`, `gcp-iam`, `aws-iam` | +| `auth.config.identity-id` | The file path where the machine identity id is stored

This field is required when using any of the following auth types: `kubernetes`, `azure`, `gcp-id-token`, `gcp-iam`, or `aws-iam`. | +| `auth.config.service-account-token` | Path to the Kubernetes service account token to use (optional)

Default: `/var/run/secrets/kubernetes.io/serviceaccount/token` | +| `auth.config.service-account-key` | Path to your GCP service account key file. This field is required when using `gcp-iam` auth type.

Please note that the file should be in JSON format. | +| `auth.config.client-id` | The file path where the universal-auth client id is stored. | +| `auth.config.client-secret` | The file path where the universal-auth client secret is stored. | +| `auth.config.remove_client_secret_on_read` | This will instruct the agent to remove the client secret from disk. | +| `sinks[].type` | The type of sink in a list of sinks. Each item specifies a sink type. Currently, only `"file"` type is available. | +| `sinks[].config.path` | The file path where the access token should be stored for each sink in the list. | +| `templates[].source-path` | The path to the template file that should be used to render secrets. | +| `templates[].template-content` | The format to use for rendering the secrets. | +| `templates[].destination-path` | The path where the rendered secrets from the source template will be saved to. | +| `templates[].config.polling-interval` | How frequently to check for secret changes. Default: `5 minutes` (optional) | +| `templates[].config.execute.command` | The command to execute when secret change is detected (optional) | +| `templates[].config.execute.timeout` | How long in seconds to wait for command to execute before timing out (optional) | ## Authentication @@ -68,19 +72,20 @@ The Infisical agent supports multiple authentication methods. Below are the avai The Universal Auth method is a simple and secure way to authenticate with Infisical. It requires a client ID and a client secret to authenticate with Infisical. - - - - Path to the file containing the universal auth client ID. - - - Path to the file containing the universal auth client secret. - - - Instructs the agent to remove the client secret from disk after reading it. - - - + + + + Path to the file containing the universal auth client ID. + + + Path to the file containing the universal auth client secret. + + + Instructs the agent to remove the client secret from disk after reading + it. + + + @@ -98,21 +103,25 @@ The Infisical agent supports multiple authentication methods. Below are the avai remove_client_secret_on_read: false # Optional field, instructs the agent to remove the client secret from disk after reading it ``` + The Native Kubernetes method is used to authenticate with Infisical when running in a Kubernetes environment. It requires a service account token to authenticate with Infisical. - - - - Path to the file containing the machine identity ID. - - - Path to the Kubernetes service account token to use. Default: `/var/run/secrets/kubernetes.io/serviceaccount/token`. - - - +{" "} + + + + + Path to the file containing the machine identity ID. + + + Path to the Kubernetes service account token to use. Default: + `/var/run/secrets/kubernetes.io/serviceaccount/token`. + + + @@ -129,6 +138,7 @@ The Infisical agent supports multiple authentication methods. Below are the avai service-account-token: "/var/run/secrets/kubernetes.io/serviceaccount/token" # Optional field, custom path to the Kubernetes service account token to use ``` + @@ -186,6 +196,7 @@ The Infisical agent supports multiple authentication methods. Below are the avai ``` + The GCP IAM method is used to authenticate with Infisical with a GCP service account key. @@ -217,6 +228,7 @@ The Infisical agent supports multiple authentication methods. Below are the avai ``` + The AWS IAM method is used to authenticate with Infisical with an AWS IAM role while running in an AWS environment like EC2, Lambda, etc. @@ -244,10 +256,12 @@ The Infisical agent supports multiple authentication methods. Below are the avai ``` + ## Quick start Infisical Agent + To install the Infisical agent, you must first install the [Infisical CLI](../cli/overview) in the desired environment where you'd like the agent to run. This is because the Infisical agent is a sub-command of the Infisical CLI. Once you have the CLI installed, you will need to provision programmatic access for the agent via [Universal Auth](/documentation/platform/identities/universal-auth). To obtain a **Client ID** and a **Client Secret**, follow the step by step guide outlined [here](/documentation/platform/identities/universal-auth). @@ -277,8 +291,8 @@ templates: command: ./reload-app.sh ``` -The secret template below will be used to render the secrets with the key and the value separated by `=` sign. You'll notice that a custom function named `secret` is used to fetch the secrets. -This function takes the following arguments: `secret "" "" ""`. +The secret template below will be used to render the secrets with the key and the value separated by `=` sign. You'll notice that a custom function named `secret` is used to fetch the secrets. +This function takes the following arguments: `secret "" "" ""`. ```text my-dot-ev-secret-template {{- with secret "6553ccb2b7da580d7f6e7260" "dev" "/" }} @@ -290,13 +304,12 @@ This function takes the following arguments: `secret "" " ```bash listSecrets "" "environment-slug" "" @@ -309,11 +322,12 @@ infisical agent --config example-agent-config-file.yaml {{- end }} ``` - **Function name**: listSecrets +**Function name**: listSecrets - **Description**: This function can be used to render the full list of secrets within a given project, environment and secret path. +**Description**: This function can be used to render the full list of secrets within a given project, environment and secret path. + +**Returns**: A single secret object with the following keys `Key, WorkspaceId, Value, Type, ID, and Comment` - **Returns**: A single secret object with the following keys `Key, WorkspaceId, Value, Type, ID, and Comment` @@ -321,18 +335,18 @@ infisical agent --config example-agent-config-file.yaml getSecretByName "" "" "" "" ``` - ```bash example-template-usage - {{ with getSecretByName "d821f21d-aa90-453b-8448-8c78c1160a0e" "dev" "/" "POSTHOG_HOST"}} - {{ if .Value }} - password = "{{ .Value }}" - {{ end }} - {{ end }} - ``` +```bash example-template-usage +{{ with getSecretByName "d821f21d-aa90-453b-8448-8c78c1160a0e" "dev" "/" "POSTHOG_HOST"}} +{{ if .Value }} +password = "{{ .Value }}" +{{ end }} +{{ end }} +``` - **Function name**: getSecretByName +**Function name**: getSecretByName - **Description**: This function can be used to render a single secret by it's name. +**Description**: This function can be used to render a single secret by it's name. - **Returns**: A list of secret objects with the following keys `Key, WorkspaceId, Value, Type, ID, and Comment` +**Returns**: A list of secret objects with the following keys `Key, WorkspaceId, Value, Type, ID, and Comment` From b53607f8e444815e5b93b5f4fc284d5893c0a4e9 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Sat, 24 Aug 2024 01:51:39 +0800 Subject: [PATCH 02/37] doc: updated ecs with agent doc to use aws auth --- .../integrations/platforms/ecs-with-agent.mdx | 175 +++++++++--------- 1 file changed, 84 insertions(+), 91 deletions(-) diff --git a/docs/integrations/platforms/ecs-with-agent.mdx b/docs/integrations/platforms/ecs-with-agent.mdx index 43760d2bb..2a510f658 100644 --- a/docs/integrations/platforms/ecs-with-agent.mdx +++ b/docs/integrations/platforms/ecs-with-agent.mdx @@ -1,27 +1,30 @@ --- -title: 'Amazon ECS' +title: "Amazon ECS" description: "Learn how to deliver secrets to Amazon Elastic Container Service." --- ![ecs diagram](/images/guides/agent-with-ecs/ecs-diagram.png) -This guide will go over the steps needed to access secrets stored in Infisical from Amazon Elastic Container Service (ECS). +This guide will go over the steps needed to access secrets stored in Infisical from Amazon Elastic Container Service (ECS). -At a high level, the steps involve setting up an ECS task with a [Infisical Agent](/infisical-agent/overview) as a sidecar container. This sidecar container uses [Universal Auth](/documentation/platform/identities/universal-auth) to authenticate with Infisical to fetch secrets/access tokens. +At a high level, the steps involve setting up an ECS task with an [Infisical Agent](/infisical-agent/overview) as a sidecar container. This sidecar container uses [AWS Auth](/documentation/platform/identities/aws-auth) to authenticate with Infisical to fetch secrets/access tokens. Once the secrets/access tokens are retrieved, they are then stored in a shared [Amazon Elastic File System](https://aws.amazon.com/efs/) (EFS) volume. This volume is then made accessible to your application and all of its replicas. This guide primarily focuses on integrating Infisical Cloud with Amazon ECS on AWS Fargate and Amazon EFS. However, the principles and steps can be adapted for use with any instance of Infisical (on premise or cloud) and different ECS launch configurations. ## Prerequisites + This guide requires the following prerequisites: -- Infisical account + +- Infisical account - Git installed - Terraform v1.0 or later installed - Access to AWS credentials - Understanding of [Infisical Agent](/infisical-agent/overview) ## What we will deploy + For this demonstration, we'll deploy the [File Browser](https://github.com/filebrowser/filebrowser) application on our ECS cluster. Although this guide focuses on File Browser, the principles outlined here can be applied to any application of your choice. @@ -29,20 +32,20 @@ File Browser plays a key role in this context because it enables us to view all This feature is important for our demonstration, as it allows us to verify whether the Infisical agent is depositing the expected files into the designated file volume and if those files are accessible to the application. -Volumes that contain sensitive secrets should not be publicly accessible. The use of File Browser here is solely for demonstration and verification purposes. + Volumes that contain sensitive secrets should not be publicly accessible. The + use of File Browser here is solely for demonstration and verification + purposes. - ## Configure Authentication with Infisical -In order for the Infisical agent to fetch credentials from Infisical, we'll first need to authenticate with Infisical. -While Infisical supports various authentication methods, this guide focuses on using Universal Auth to authenticate the agent with Infisical. -Follow the documentation to configure and generate a client id and client secret with Universal auth [here](/documentation/platform/identities/universal-auth). -Make sure to save these credentials somewhere handy because you'll need them soon. +In order for the Infisical agent to fetch credentials from Infisical, we'll first need to authenticate with Infisical. Follow the documentation to configure a machine identity with AWS Auth [here](/documentation/platform/identities/aws-auth). +Take note of the Machine Identity ID as you will be needing this in the preceding steps. ## Clone guide assets repository + To help you quickly deploy the example application, please clone the guide assets from this [Github repository](https://github.com/Infisical/infisical-guides.git). -This repository contains assets for all Infisical guides. The content for this guide can be found within a sub directory called `aws-ecs-with-agent`. +This repository contains assets for all Infisical guides. The content for this guide can be found within a sub directory called `aws-ecs-with-agent`. The guide will assume that `aws-ecs-with-agent` is your working directory going forward. ## Deploy example application @@ -50,95 +53,83 @@ The guide will assume that `aws-ecs-with-agent` is your working directory going Before we can deploy our full application and its related infrastructure with Terraform, we'll need to first configure our Infisical agent. ### Agent configuration overview + The agent config file defines what authentication method will be used when connecting with Infisical along with where the fetched secrets/access tokens should be saved to. -Since the Infisical agent will be deployed as a sidecar, the agent configuration file and any secret template files will need to be encoded in base64. -This encoding step is necessary as it allows these files to be added into our Terraform configuration file without needing to upload them first. - -#### Secret template file -The Infisical agent accepts one or more optional template files. If provided, the agent will fetch secrets using the set authentication method and format the fetched secrets according to the given template file. - -For demonstration purposes, we will create the following secret template file. -This template will transform our secrets from Infisical project with the ID `62fd92aa8b63973fee23dec7`, in the `dev` environment, and secrets located in the path `/`, into a `KEY=VALUE` format. - - - Remember to update the project id, environment slug and secret path to one that exists within your Infisical project - - -```secrets.template secrets.template -{{- with secret "62fd92aa8b63973fee23dec7" "dev" "/" }} -{{- range . }} -{{ .Key }}={{ .Value }} -{{- end }} -{{- end }} -``` - -Next, we need encode this template file in `base64` so it can be set in the agent configuration file. - -```bash -cat secrets.template | base64 -Cnt7LSB3aXRoIHNlY3JldCAiMWVkMjk2MWQtNDM5NS00MmNlLTlkNzQtYjk2ZGQwYmYzMDg0IiAiZGV2IiAiLyIgfX0Ke3stIHJhbmdlIC4gfX0Ke3sgLktleSB9fT17eyAuVmFsdWUgfX0Ke3stIGVuZCB9fQp7ey0gZW5kIH19 -``` +Since the Infisical agent will be deployed as a sidecar, the agent configuration file will need to be encoded in base64. +This encoding step is necessary as it allows the agent configuration file to be added into our Terraform configuration without needing to upload it first. #### Full agent configuration file -This agent config file will connect with Infisical Cloud using Universal Auth and deposit access tokens at path `/infisical-agent/access-token` and render secrets to file `/infisical-agent/secrets`. -You'll notice that instead of passing the path to the secret template file as we normally would, we set the base64 encoded template from the previous step under `base64-template-content` property. +Inside the `aws-ecs-with-agent` directory, you will find a sample `agent-config.yaml` file. This agent config file will connect with Infisical Cloud using AWS Auth and deposit access tokens at path `/infisical-agent/access-token` and render secrets to file `/infisical-agent/secrets`. ```yaml agent-config.yaml infisical: address: https://app.infisical.com exit-after-auth: true auth: - type: universal-auth - config: - remove_client_secret_on_read: false + type: aws-iam sinks: - type: file config: path: /infisical-agent/access-token templates: - - base64-template-content: Cnt7LSB3aXRoIHNlY3JldCAiMWVkMjk2MWQtNDM5NS00MmNlLTlkNzQtYjk2ZGQwYmYzMDg0IiAiZGV2IiAiLyIgfX0Ke3stIHJhbmdlIC4gfX0Ke3sgLktleSB9fT17eyAuVmFsdWUgfX0Ke3stIGVuZCB9fQp7ey0gZW5kIH19 + - template-content: | + {{- with secret "202f04d7-e4cb-43d4-a292-e893712d61fc" "dev" "/" }} + {{- range . }} + {{ .Key }}={{ .Value }} + {{- end }} + {{- end }} destination-path: /infisical-agent/secrets ``` -Again, we'll need to encode the full configuration file in `base64` so it can be easily delivered via Terraform. +#### Secret template -```bash -cat agent-config.yaml | base64 -aW5maXNpY2FsOgogIGFkZHJlc3M6IGh0dHBzOi8vYXBwLmluZmlzaWNhbC5jb20KICBleGl0LWFmdGVyLWF1dGg6IHRydWUKYXV0aDoKICB0eXBlOiB1bml2ZXJzYWwtYXV0aAogIGNvbmZpZzoKICAgIHJlbW92ZV9jbGllbnRfc2VjcmV0X29uX3JlYWQ6IGZhbHNlCnNpbmtzOgogIC0gdHlwZTogZmlsZQogICAgY29uZmlnOgogICAgICBwYXRoOiAvaW5maXNpY2FsLWFnZW50L2FjY2Vzcy10b2tlbgp0ZW1wbGF0ZXM6CiAgLSBiYXNlNjQtdGVtcGxhdGUtY29udGVudDogQ250N0xTQjNhWFJvSUhObFkzSmxkQ0FpTVdWa01qazJNV1F0TkRNNU5TMDBNbU5sTFRsa056UXRZamsyWkdRd1ltWXpNRGcwSWlBaVpHVjJJaUFpTHlJZ2ZYMEtlM3N0SUhKaGJtZGxJQzRnZlgwS2Uzc2dMa3RsZVNCOWZUMTdleUF1Vm1Gc2RXVWdmWDBLZTNzdElHVnVaQ0I5ZlFwN2V5MGdaVzVrSUgxOQogICAgZGVzdGluYXRpb24tcGF0aDogL2luZmlzaWNhbC1hZ2VudC9zZWNyZXRzCg== -``` +The Infisical agent accepts one or more optional templates. If provided, the agent will fetch secrets using the set authentication method and format the fetched secrets according to the given template. -## Add auth credentials & agent config -With the base64 encoded agent config file and Universal Auth credentials in hand, it's time to assign them as values in our Terraform config file. +In the agent configuration above, the template defined will transform the secrets from Infisical project with the ID `202f04d7-e4cb-43d4-a292-e893712d61fc`, in the `dev` environment, and secrets located in the path `/`, into a `KEY=VALUE` format. -To configure these values, navigate to the `ecs.tf` file in your preferred code editor and assign values to `auth_client_id`, `auth_client_secret`, and `agent_config`. + + Remember to update the project id, environment slug and secret path to one + that exists within your Infisical project + + +## Configure app on terraform + +Navigate to the `ecs.tf` file in your preferred code editor. In the container_definitions section, assign the values to the `machine_identity_id` and `agent_config` properties. +The `agent_config` property expects the base64-encoded agent configuration file. In order to get this, we use the `base64encode` and `file` functions of HCL. ```hcl ecs.tf ...snip... -data "template_file" "cb_app" { - template = file("./templates/ecs/cb_app.json.tpl") - - vars = { - app_image = var.app_image - sidecar_image = var.sidecar_image - app_port = var.app_port - fargate_cpu = var.fargate_cpu - fargate_memory = var.fargate_memory - aws_region = var.aws_region - auth_client_id = "" - auth_client_secret = "" - agent_config = "" +resource "aws_ecs_task_definition" "app" { + family = "cb-app-task" + execution_role_arn = aws_iam_role.ecs_task_execution_role.arn + task_role_arn = aws_iam_role.ecs_task_role.arn + network_mode = "awsvpc" + requires_compatibilities = ["FARGATE"] + cpu = 4096 + memory = 8192 + container_definitions = templatefile("./templates/ecs/cb_app.json.tpl", { + app_image = var.app_image + sidecar_image = var.sidecar_image + app_port = var.app_port + fargate_cpu = var.fargate_cpu + fargate_memory = var.fargate_memory + aws_region = var.aws_region + machine_identity_id = "5655f4f5-332b-45f9-af06-8f493edff36f" + agent_config = base64encode(file("../agent-config.yaml")) + }) + volume { + name = "infisical-efs" + efs_volume_configuration { + file_system_id = aws_efs_file_system.infisical_efs.id + root_directory = "/" + } } } ...snip... ``` - - To keep this guide simple, `auth_client_id`, `auth_client_secret` have been added directly into the ECS container definition. - However, in production, you should securely fetch these values from AWS Secrets Manager or AWS Parameter store and feed them directly to agent sidecar. - - After these values have been set, they will be passed to the Infisical agent during startup through environment variables, as configured in the `infisical-sidecar` container below. ```terraform templates/ecs/cb_app.json.tpl @@ -169,12 +160,8 @@ After these values have been set, they will be passed to the Infisical agent dur }, "environment": [ { - "name": "INFISICAL_UNIVERSAL_AUTH_CLIENT_ID", - "value": "${auth_client_id}" - }, - { - "name": "INFISICAL_UNIVERSAL_CLIENT_SECRET", - "value": "${auth_client_secret}" + "name": "INFISICAL_MACHINE_IDENTITY_ID", + "value": "${machine_identity_id}" }, { "name": "INFISICAL_AGENT_CONFIG_BASE64", @@ -191,9 +178,9 @@ After these values have been set, they will be passed to the Infisical agent dur ] ``` -In the above container definition, you'll notice that that the Infisical agent has a `mountPoints` defined. -This mount point is referencing to the already configured EFS volume as shown below. -`containerPath` is set to `/infisical-agent` because that is that the folder we have instructed the agent to deposit the credentials to. +In the above container definition, you'll notice that that the Infisical agent has a `mountPoints` defined. +This mount point is referencing to the already configured EFS volume as shown below. +`containerPath` is set to `/infisical-agent` because that is that the folder we have instructed the agent to deposit the credentials to. ```hcl terraform/efs.tf resource "aws_efs_file_system" "infisical_efs" { @@ -211,8 +198,9 @@ resource "aws_efs_mount_target" "mount" { ``` ## Configure AWS credentials + Because we'll be deploying the example file browser application to AWS via Terraform, you will need to obtain a set of `AWS Access Key` and `Secret Key`. -Once you have generated these credentials, export them to your terminal. +Once you have generated these credentials, export them to your terminal. 1. Export the AWS Access Key ID: @@ -227,26 +215,31 @@ Once you have generated these credentials, export them to your terminal. ``` ## Deploy terraform configuration + With the agent's sidecar configuration complete, we can now deploy our changes to AWS via Terraform. 1. Change your directory to `terraform` -```sh + +```sh cd terraform ``` 2. Initialize Terraform + ``` -$ terraform init +$ terraform init ``` -3. Preview resources that will be created +3. Preview resources that will be created + ``` $ terraform plan ``` 4. Trigger resource creation + ```bash -$ terraform apply +$ terraform apply Do you want to perform these actions? Terraform will perform the actions described above. @@ -264,24 +257,24 @@ Outputs: alb_hostname = "cb-load-balancer-1675475779.us-east-1.elb.amazonaws.com:8080" ``` -Once the resources have been successfully deployed, Terrafrom will output the host address where the file browser application will be accessible. -It may take a few minutes for the application to become fully ready. - +Once the resources have been successfully deployed, Terraform will output the host address where the file browser application will be accessible. +It may take a few minutes for the application to become fully ready. ## Verify secrets/tokens in EFS volume + To verify that the agent is depositing access tokens and rendering secrets to the paths specified in the agent config, navigate to the web address from the previous step. Once you visit the address, you'll be prompted to login. Enter the credentials shown below. ![file browser main login page](/images/guides/agent-with-ecs/file_browser_main.png) -Since our EFS volume is mounted to the path of the file browser application, we should see the access token and rendered secret file we defined via the agent config file. +Since our EFS volume is mounted to the path of the file browser application, we should see the access token and rendered secret file we defined via the agent config file. ![file browswer dashbaord](/images/guides/agent-with-ecs/filebrowser_afterlogin.png) -As expected, two files are present: `access-token` and `secrets`. -The `access-token` file should hold a valid `Bearer` token, which can be used to make HTTP requests to Infisical. +As expected, two files are present: `access-token` and `secrets`. +The `access-token` file should hold a valid `Bearer` token, which can be used to make HTTP requests to Infisical. The `secrets` file should contain secrets, formatted according to the specifications in our secret template file (presented in key=value format). ![file browser access token deposit](/images/guides/agent-with-ecs/access-token-deposit.png) -![file browser secrets render](/images/guides/agent-with-ecs/secrets-deposit.png) \ No newline at end of file +![file browser secrets render](/images/guides/agent-with-ecs/secrets-deposit.png) From e73d3f87f3461b62b078db47259664fe96108dd2 Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Fri, 23 Aug 2024 14:29:29 -0400 Subject: [PATCH 03/37] small nit --- docs/integrations/platforms/infisical-agent.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/platforms/infisical-agent.mdx b/docs/integrations/platforms/infisical-agent.mdx index 4a97bb320..aab39bd29 100644 --- a/docs/integrations/platforms/infisical-agent.mdx +++ b/docs/integrations/platforms/infisical-agent.mdx @@ -58,7 +58,7 @@ While specifying an authentication method is mandatory to start the agent, confi | `sinks[].type` | The type of sink in a list of sinks. Each item specifies a sink type. Currently, only `"file"` type is available. | | `sinks[].config.path` | The file path where the access token should be stored for each sink in the list. | | `templates[].source-path` | The path to the template file that should be used to render secrets. | -| `templates[].template-content` | The format to use for rendering the secrets. | +| `templates[].template-content` | The inline secret template to be used for rendering the secrets. | | `templates[].destination-path` | The path where the rendered secrets from the source template will be saved to. | | `templates[].config.polling-interval` | How frequently to check for secret changes. Default: `5 minutes` (optional) | | `templates[].config.execute.command` | The command to execute when secret change is detected (optional) | From 403e0d2d9d3edd74bdff694cab2d35bdd1283a0f Mon Sep 17 00:00:00 2001 From: Vlad Matsiiako <78047717+vmatsiiako@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:26:21 -0700 Subject: [PATCH 04/37] Update onboarding.mdx --- company/handbook/onboarding.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/company/handbook/onboarding.mdx b/company/handbook/onboarding.mdx index e85be9f04..963e8b4e9 100644 --- a/company/handbook/onboarding.mdx +++ b/company/handbook/onboarding.mdx @@ -24,5 +24,6 @@ Every new joiner has an onboarding buddy who should ideally be in the the same t 6. Over the first few weeks, feel free to schedule 1:1s with folks on the team to get to know them a bit better. 7. Change your Slack username in the users channel to `[NAME] (Infisical)`. 8. Go through the [technical overview](https://infisical.com/docs/internals/overview) of Infisical. +9. Request a company credit card (Maidul will be able to help with that). From 0401f55bc3b414dadeb91532644a98e841784dc9 Mon Sep 17 00:00:00 2001 From: Vlad Matsiiako <78047717+vmatsiiako@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:28:37 -0700 Subject: [PATCH 05/37] Update onboarding.mdx --- company/handbook/onboarding.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/company/handbook/onboarding.mdx b/company/handbook/onboarding.mdx index 963e8b4e9..bcf47a339 100644 --- a/company/handbook/onboarding.mdx +++ b/company/handbook/onboarding.mdx @@ -19,7 +19,7 @@ Every new joiner has an onboarding buddy who should ideally be in the the same t 1. Join the weekly all-hands meeting. It typically happens on Monday's at 8:30am PT. 2. Ship something together on day one – even if tiny! It feels great to hit the ground running, with a development environment all ready to go. 3. Check out the [Areas of Responsibility (AoR) Table](https://docs.google.com/spreadsheets/d/1RnXlGFg83Sgu0dh7ycuydsSobmFfI3A0XkGw7vrVxEI/edit?usp=sharing). This is helpful to know who you can ask about particular areas of Infisical. Feel free to add yourself to the areas you'd be most interesting to dive into. -4. Read the [Infisical Strategy Doc](https://docs.google.com/document/d/1oy_NP1Q_Zt1oqxLpyNkLIGmhAI3N28AmZq6dDIOONSQ/edit?usp=sharing). +4. Read the [Infisical Strategy Doc](https://docs.google.com/document/d/1RaJd3RoS2QpWLFHlgfHaXnHqCCwRt6mCGZkbJ75J_D0/edit?usp=sharing). 5. Update your LinkedIn profile with one of [Infisical's official banners](https://drive.google.com/drive/u/0/folders/1oSNWjbpRl9oNYwxM_98IqzKs9fAskrb2) (if you want to). You can also coordinate your social posts in the #marketing Slack channel, so that we can boost it from Infisical's official social media accounts. 6. Over the first few weeks, feel free to schedule 1:1s with folks on the team to get to know them a bit better. 7. Change your Slack username in the users channel to `[NAME] (Infisical)`. From 58838c541fcf5291a94556ae3cf0457129b83466 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 27 Aug 2024 23:26:56 +0800 Subject: [PATCH 06/37] misc: added timeout for est connection --- backend/src/server/routes/est/certificate-est-router.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/src/server/routes/est/certificate-est-router.ts b/backend/src/server/routes/est/certificate-est-router.ts index 1eec348ce..7f401216e 100644 --- a/backend/src/server/routes/est/certificate-est-router.ts +++ b/backend/src/server/routes/est/certificate-est-router.ts @@ -42,6 +42,11 @@ export const registerCertificateEstRouter = async (server: FastifyZodProvider) = await res.hijack(); + // definitive connection timeout to clean-up open connections and prevent memory leak + res.raw.setTimeout(10 * 1000, () => { + res.raw.end(); + }); + res.raw.setHeader(wwwAuthenticateHeader, `Basic realm="infisical"`); res.raw.setHeader("Content-Length", 0); res.raw.statusCode = 401; From 3ab6eb62c896ba4bdc80f0c19ad6676814e6aae2 Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Tue, 27 Aug 2024 20:03:36 -0400 Subject: [PATCH 07/37] update health check --- backend/src/server/routes/index.ts | 33 +++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 5d20f1cff..349a41754 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -1,4 +1,5 @@ import { CronJob } from "cron"; +import { Redis } from "ioredis"; import { Knex } from "knex"; import { z } from "zod"; @@ -71,6 +72,7 @@ import { trustedIpDALFactory } from "@app/ee/services/trusted-ip/trusted-ip-dal" import { trustedIpServiceFactory } from "@app/ee/services/trusted-ip/trusted-ip-service"; import { TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig } from "@app/lib/config/env"; +import { logger } from "@app/lib/logger"; import { TQueueServiceFactory } from "@app/queue"; import { readLimit } from "@app/server/config/rateLimiter"; import { accessTokenQueueServiceFactory } from "@app/services/access-token-queue/access-token-queue"; @@ -1257,7 +1259,7 @@ export const registerRoutes = async ( response: { 200: z.object({ date: z.date(), - message: z.literal("Ok"), + message: z.string().optional(), emailConfigured: z.boolean().optional(), inviteOnlySignup: z.boolean().optional(), redisConfigured: z.boolean().optional(), @@ -1266,12 +1268,37 @@ export const registerRoutes = async ( }) } }, - handler: async () => { + handler: async (request, reply) => { const cfg = getConfig(); const serverCfg = await getServerCfg(); + + try { + await db.raw("SELECT NOW()"); + } catch (err) { + logger.error("Health check: database connection failed", err); + return reply.code(503).send({ + date: new Date(), + message: "Service unavailable" + }); + } + + if (cfg.isRedisConfigured) { + const redis = new Redis(cfg.REDIS_URL); + try { + await redis.ping(); + redis.disconnect(); + } catch (err) { + logger.error("Health check: redis connection failed", err); + return reply.code(503).send({ + date: new Date(), + message: "Service unavailable" + }); + } + } + return { date: new Date(), - message: "Ok" as const, + message: "Ok", emailConfigured: cfg.isSmtpConfigured, inviteOnlySignup: Boolean(serverCfg.allowSignUp), redisConfigured: cfg.isRedisConfigured, From 0ab7a4e7134593efc076a59711b7616b0b2ea5ef Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 28 Aug 2024 13:45:25 +0800 Subject: [PATCH 08/37] misc: added transaction for cert template create and update --- .../certificate-template-dal.ts | 7 +- .../certificate-template-service.ts | 75 +++++++++++-------- 2 files changed, 47 insertions(+), 35 deletions(-) diff --git a/backend/src/services/certificate-template/certificate-template-dal.ts b/backend/src/services/certificate-template/certificate-template-dal.ts index a30c4890d..4db383951 100644 --- a/backend/src/services/certificate-template/certificate-template-dal.ts +++ b/backend/src/services/certificate-template/certificate-template-dal.ts @@ -1,3 +1,5 @@ +import { Knex } from "knex"; + import { TDbClient } from "@app/db"; import { TableName } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; @@ -30,10 +32,9 @@ export const certificateTemplateDALFactory = (db: TDbClient) => { } }; - const getById = async (id: string) => { + const getById = async (id: string, tx?: Knex) => { try { - const certTemplate = await db - .replicaNode()(TableName.CertificateTemplate) + const certTemplate = await (tx || db.replicaNode())(TableName.CertificateTemplate) .join( TableName.CertificateAuthority, `${TableName.CertificateAuthority}.id`, diff --git a/backend/src/services/certificate-template/certificate-template-service.ts b/backend/src/services/certificate-template/certificate-template-service.ts index 31745e752..04d5e98eb 100644 --- a/backend/src/services/certificate-template/certificate-template-service.ts +++ b/backend/src/services/certificate-template/certificate-template-service.ts @@ -75,23 +75,28 @@ export const certificateTemplateServiceFactory = ({ ProjectPermissionSub.CertificateTemplates ); - const { id } = await certificateTemplateDAL.create({ - caId, - pkiCollectionId, - name, - commonName, - subjectAlternativeName, - ttl + return certificateTemplateDAL.transaction(async (tx) => { + const { id } = await certificateTemplateDAL.create( + { + caId, + pkiCollectionId, + name, + commonName, + subjectAlternativeName, + ttl + }, + tx + ); + + const certificateTemplate = await certificateTemplateDAL.getById(id, tx); + if (!certificateTemplate) { + throw new NotFoundError({ + message: "Certificate template not found" + }); + } + + return certificateTemplate; }); - - const certificateTemplate = await certificateTemplateDAL.getById(id); - if (!certificateTemplate) { - throw new NotFoundError({ - message: "Certificate template not found" - }); - } - - return certificateTemplate; }; const updateCertTemplate = async ({ @@ -136,23 +141,29 @@ export const certificateTemplateServiceFactory = ({ } } - await certificateTemplateDAL.updateById(certTemplate.id, { - caId, - pkiCollectionId, - commonName, - subjectAlternativeName, - name, - ttl + return certificateTemplateDAL.transaction(async (tx) => { + await certificateTemplateDAL.updateById( + certTemplate.id, + { + caId, + pkiCollectionId, + commonName, + subjectAlternativeName, + name, + ttl + }, + tx + ); + + const updatedTemplate = await certificateTemplateDAL.getById(id, tx); + if (!updatedTemplate) { + throw new NotFoundError({ + message: "Certificate template not found" + }); + } + + return updatedTemplate; }); - - const updatedTemplate = await certificateTemplateDAL.getById(id); - if (!updatedTemplate) { - throw new NotFoundError({ - message: "Certificate template not found" - }); - } - - return updatedTemplate; }; const deleteCertTemplate = async ({ id, actorId, actorAuthMethod, actor, actorOrgId }: TDeleteCertTemplateDTO) => { From e47cfa262aeb6b6d0410989115b6f4c326505adf Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 28 Aug 2024 13:59:47 +0800 Subject: [PATCH 09/37] misc: added cloud est user guide --- docs/documentation/platform/pki/est.mdx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/documentation/platform/pki/est.mdx b/docs/documentation/platform/pki/est.mdx index aab822dd5..ee66c32b6 100644 --- a/docs/documentation/platform/pki/est.mdx +++ b/docs/documentation/platform/pki/est.mdx @@ -21,7 +21,8 @@ These endpoints are exposed on port 8443 under the .well-known/est path e.g. - You need to have an existing [CA hierarchy](/documentation/platform/pki/private-ca). - The client devices need to have a bootstrap/pre-installed certificate. -- The client devices must trust the server certificates used by Infisical's EST server. If the devices are new or lack existing trust configurations, you need to manually establish trust for the appropriate certificates. When using Infisical Cloud, this means establishing trust for certificates issued by AWS. +- The client devices must trust the server certificates used by Infisical's EST server. If the devices are new or lack existing trust configurations, you need to manually establish trust for the appropriate certificates. + - For Infisical Cloud users, the devices must be configured to trust the [Amazon root CA certificates](https://www.amazontrust.com/repository). ## Guide to configuring EST @@ -42,7 +43,7 @@ These endpoints are exposed on port 8443 under the .well-known/est path e.g. 4. Once the configuration of enrollment options is completed, a new **EST Label** field appears in the enrollment settings. This is the value to use as label in the URL when configuring the connection of EST clients to Infisical. ![est enrollment modal create](/images/platform/pki/est/template-enrollment-est-label.png) - For demonstration, the complete URL of the supported EST endpoints will look like the following: + The complete URL of the supported EST endpoints will look like the following: - https://app.infisical.com:8443/.well-known/est/f110f308-9888-40ab-b228-237b12de8b96/cacerts - https://app.infisical.com:8443/.well-known/est/f110f308-9888-40ab-b228-237b12de8b96/simpleenroll From 53075d503a138aab770b5e19267adae6dee2432c Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 28 Aug 2024 15:48:25 +0800 Subject: [PATCH 10/37] misc: added note to secret template docs --- docs/integrations/platforms/ecs-with-agent.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/platforms/ecs-with-agent.mdx b/docs/integrations/platforms/ecs-with-agent.mdx index 2a510f658..31c5a982b 100644 --- a/docs/integrations/platforms/ecs-with-agent.mdx +++ b/docs/integrations/platforms/ecs-with-agent.mdx @@ -86,6 +86,7 @@ templates: #### Secret template The Infisical agent accepts one or more optional templates. If provided, the agent will fetch secrets using the set authentication method and format the fetched secrets according to the given template. +Typically, these templates are passed in to the agent configuration file via file reference using the `source-path` property but for simplicity we define them inline. In the agent configuration above, the template defined will transform the secrets from Infisical project with the ID `202f04d7-e4cb-43d4-a292-e893712d61fc`, in the `dev` environment, and secrets located in the path `/`, into a `KEY=VALUE` format. From 5611b9aba16b15e3455640923877bf3007771007 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 28 Aug 2024 15:53:36 +0800 Subject: [PATCH 11/37] misc: added reference to secret template functions --- docs/integrations/platforms/infisical-agent.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/platforms/infisical-agent.mdx b/docs/integrations/platforms/infisical-agent.mdx index 4a97bb320..cba70e602 100644 --- a/docs/integrations/platforms/infisical-agent.mdx +++ b/docs/integrations/platforms/infisical-agent.mdx @@ -32,7 +32,7 @@ Every time the agent successfully retrieves a new access token, it writes the ne The Infisical agent can help deliver formatted secrets to your application in a variety of environments. To achieve this, the agent will retrieve secrets from Infisical, format them using a specified template, and then save these formatted secrets to a designated file path. -Templating process is done through the use of Go language's [text/template feature](https://pkg.go.dev/text/template). Multiple template definitions can be set in the agent configuration file to generate a variety of formatted secret files. +Templating process is done through the use of Go language's [text/template feature](https://pkg.go.dev/text/template).You can refer to the available secret template functions [here](#available-secret-template-functions). Multiple template definitions can be set in the agent configuration file to generate a variety of formatted secret files. When the agent is started and templates are defined in the agent configuration file, the agent will attempt to acquire a valid access token using the set authentication method outlined in the agent's configuration. If this initial attempt is unsuccessful, the agent will momentarily pauses before continuing to make more attempts. From de9cb265e0261d2682b18ca62a10c126b170f8ed Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Wed, 21 Aug 2024 23:56:19 +0400 Subject: [PATCH 12/37] Feat: Redis support for dynamic secrets --- .../dynamic-secret/providers/index.ts | 4 +- .../dynamic-secret/providers/models.ts | 17 +- .../dynamic-secret/providers/redis.ts | 183 ++++++++++ frontend/src/hooks/api/dynamicSecret/types.ts | 92 ++--- .../CreateDynamicSecretForm.tsx | 24 ++ .../RedisInputForm.tsx | 328 ++++++++++++++++++ .../CreateDynamicSecretLease.tsx | 19 + 7 files changed, 625 insertions(+), 42 deletions(-) create mode 100644 backend/src/ee/services/dynamic-secret/providers/redis.ts create mode 100644 frontend/src/views/SecretMainPage/components/ActionBar/CreateDynamicSecretForm/RedisInputForm.tsx diff --git a/backend/src/ee/services/dynamic-secret/providers/index.ts b/backend/src/ee/services/dynamic-secret/providers/index.ts index beb6c428e..c69bd9609 100644 --- a/backend/src/ee/services/dynamic-secret/providers/index.ts +++ b/backend/src/ee/services/dynamic-secret/providers/index.ts @@ -1,10 +1,12 @@ import { AwsIamProvider } from "./aws-iam"; import { CassandraProvider } from "./cassandra"; import { DynamicSecretProviders } from "./models"; +import { RedisDatabaseProvider } from "./redis"; import { SqlDatabaseProvider } from "./sql-database"; export const buildDynamicSecretProviders = () => ({ [DynamicSecretProviders.SqlDatabase]: SqlDatabaseProvider(), [DynamicSecretProviders.Cassandra]: CassandraProvider(), - [DynamicSecretProviders.AwsIam]: AwsIamProvider() + [DynamicSecretProviders.AwsIam]: AwsIamProvider(), + [DynamicSecretProviders.Redis]: RedisDatabaseProvider() }); diff --git a/backend/src/ee/services/dynamic-secret/providers/models.ts b/backend/src/ee/services/dynamic-secret/providers/models.ts index 14b79eeea..a28dfceb9 100644 --- a/backend/src/ee/services/dynamic-secret/providers/models.ts +++ b/backend/src/ee/services/dynamic-secret/providers/models.ts @@ -7,6 +7,17 @@ export enum SqlProviders { MsSQL = "mssql" } +export const DynamicSecretRedisDBSchema = z.object({ + host: z.string().trim().toLowerCase(), + port: z.number(), + username: z.string().trim(), // this is often "default". + password: z.string().trim().optional(), // only required if requirepass is set. + creationStatement: z.string().trim(), + revocationStatement: z.string().trim(), + renewStatement: z.string().trim().optional(), + ca: z.string().optional() +}); + export const DynamicSecretSqlDBSchema = z.object({ client: z.nativeEnum(SqlProviders), host: z.string().trim().toLowerCase(), @@ -47,13 +58,15 @@ export const DynamicSecretAwsIamSchema = z.object({ export enum DynamicSecretProviders { SqlDatabase = "sql-database", Cassandra = "cassandra", - AwsIam = "aws-iam" + AwsIam = "aws-iam", + Redis = "redis" } export const DynamicSecretProviderSchema = z.discriminatedUnion("type", [ z.object({ type: z.literal(DynamicSecretProviders.SqlDatabase), inputs: DynamicSecretSqlDBSchema }), z.object({ type: z.literal(DynamicSecretProviders.Cassandra), inputs: DynamicSecretCassandraSchema }), - z.object({ type: z.literal(DynamicSecretProviders.AwsIam), inputs: DynamicSecretAwsIamSchema }) + z.object({ type: z.literal(DynamicSecretProviders.AwsIam), inputs: DynamicSecretAwsIamSchema }), + z.object({ type: z.literal(DynamicSecretProviders.Redis), inputs: DynamicSecretRedisDBSchema }) ]); export type TDynamicProviderFns = { diff --git a/backend/src/ee/services/dynamic-secret/providers/redis.ts b/backend/src/ee/services/dynamic-secret/providers/redis.ts new file mode 100644 index 000000000..23eb454c5 --- /dev/null +++ b/backend/src/ee/services/dynamic-secret/providers/redis.ts @@ -0,0 +1,183 @@ +/* eslint-disable no-console */ +import handlebars from "handlebars"; +import { Redis } from "ioredis"; +import { customAlphabet } from "nanoid"; +import { z } from "zod"; + +import { getConfig } from "@app/lib/config/env"; +import { BadRequestError } from "@app/lib/errors"; +import { getDbConnectionHost } from "@app/lib/knex"; +import { alphaNumericNanoId } from "@app/lib/nanoid"; + +import { DynamicSecretRedisDBSchema, TDynamicProviderFns } from "./models"; + +const generatePassword = () => { + const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.~!*$#"; + return customAlphabet(charset, 64)(); +}; + +const generateUsername = () => { + return alphaNumericNanoId(32); +}; + +const executeTransactions = async (connection: Redis, commands: string[]): Promise<(string | null)[] | null> => { + // Initiate a transaction + const pipeline = connection.multi(); + + // Add all commands to the pipeline + for (const command of commands) { + const args = command + .split(" ") + .map((arg) => arg.trim()) + .filter((arg) => arg.length > 0); + pipeline.call(args[0], ...args.slice(1)); + } + + // Execute the transaction + const results = await pipeline.exec(); + + if (!results) { + throw new BadRequestError({ message: "Redis transaction failed: No results returned" }); + } + + // Check for errors in the results + const errors = results.filter(([err]) => err !== null); + if (errors.length > 0) { + throw new BadRequestError({ message: "Redis transaction failed with errors" }); + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + return results.map(([_, result]) => result as string | null); +}; + +export const RedisDatabaseProvider = (): TDynamicProviderFns => { + const validateProviderInputs = async (inputs: unknown) => { + const appCfg = getConfig(); + const isCloud = Boolean(appCfg.LICENSE_SERVER_KEY); // quick and dirty way to check if its cloud or not + const dbHost = appCfg.DB_HOST || getDbConnectionHost(appCfg.DB_CONNECTION_URI); + + const providerInputs = await DynamicSecretRedisDBSchema.parseAsync(inputs); + if ( + isCloud && + // localhost + // internal ips + (providerInputs.host === "host.docker.internal" || + providerInputs.host.match(/^10\.\d+\.\d+\.\d+/) || + providerInputs.host.match(/^192\.168\.\d+\.\d+/)) + ) + throw new BadRequestError({ message: "Invalid db host" }); + if (providerInputs.host === "localhost" || providerInputs.host === "127.0.0.1" || dbHost === providerInputs.host) + throw new BadRequestError({ message: "Invalid db host" }); + return providerInputs; + }; + + const getClient = async (providerInputs: z.infer) => { + let connection: Redis | null = null; + try { + connection = new Redis({ + username: providerInputs.username, + host: providerInputs.host, + port: providerInputs.port, + password: providerInputs.password, + ...(providerInputs.ca && { + tls: { + rejectUnauthorized: false, + ca: providerInputs.ca + } + }) + }); + + let result: string; + if (providerInputs.password) { + result = await connection.auth(providerInputs.username, providerInputs.password, () => {}); + } else { + result = await connection.auth(providerInputs.username, () => {}); + } + + if (result !== "OK") { + throw new BadRequestError({ message: `Invalid credentials, Redis returned ${result} status` }); + } + + return connection; + } catch (err) { + if (connection) await connection.quit(); + + throw err; + } + }; + + const validateConnection = async (inputs: unknown) => { + const providerInputs = await validateProviderInputs(inputs); + const connection = await getClient(providerInputs); + + const pingResponse = await connection + .ping() + .then(() => true) + .catch(() => false); + + return pingResponse; + }; + + const create = async (inputs: unknown, expireAt: number) => { + const providerInputs = await validateProviderInputs(inputs); + const connection = await getClient(providerInputs); + + const username = generateUsername(); + const password = generatePassword(); + const expiration = new Date(expireAt).toISOString(); + + const creationStatement = handlebars.compile(providerInputs.creationStatement, { noEscape: true })({ + username, + password, + expiration + }); + + const queries = creationStatement.toString().split(";").filter(Boolean); + + await executeTransactions(connection, queries); + + await connection.quit(); + return { entityId: username, data: { DB_USERNAME: username, DB_PASSWORD: password } }; + }; + + const revoke = async (inputs: unknown, entityId: string) => { + const providerInputs = await validateProviderInputs(inputs); + const connection = await getClient(providerInputs); + + const username = entityId; + + const revokeStatement = handlebars.compile(providerInputs.revocationStatement)({ username }); + const queries = revokeStatement.toString().split(";").filter(Boolean); + + await executeTransactions(connection, queries); + + await connection.quit(); + return { entityId: username }; + }; + + const renew = async (inputs: unknown, entityId: string, expireAt: number) => { + const providerInputs = await validateProviderInputs(inputs); + const connection = await getClient(providerInputs); + + const username = entityId; + const expiration = new Date(expireAt).toISOString(); + + const renewStatement = handlebars.compile(providerInputs.renewStatement)({ username, expiration }); + + if (renewStatement) { + const queries = renewStatement.toString().split(";").filter(Boolean); + await executeTransactions(connection, queries); + } + + await connection.quit(); + return { entityId: username }; + }; + + return { + validateProviderInputs, + validateConnection, + create, + revoke, + renew + }; +}; diff --git a/frontend/src/hooks/api/dynamicSecret/types.ts b/frontend/src/hooks/api/dynamicSecret/types.ts index b987e94cc..82aac574a 100644 --- a/frontend/src/hooks/api/dynamicSecret/types.ts +++ b/frontend/src/hooks/api/dynamicSecret/types.ts @@ -18,7 +18,8 @@ export type TDynamicSecret = { export enum DynamicSecretProviders { SqlDatabase = "sql-database", Cassandra = "cassandra", - AwsIam = "aws-iam" + AwsIam = "aws-iam", + Redis = "redis" } export enum SqlProviders { @@ -30,47 +31,60 @@ export enum SqlProviders { export type TDynamicSecretProvider = | { - type: DynamicSecretProviders.SqlDatabase; - inputs: { - client: SqlProviders; - host: string; - port: number; - database: string; - username: string; - password: string; - creationStatement: string; - revocationStatement: string; - renewStatement?: string; - ca?: string | undefined; - }; - } + type: DynamicSecretProviders.SqlDatabase; + inputs: { + client: SqlProviders; + host: string; + port: number; + database: string; + username: string; + password: string; + creationStatement: string; + revocationStatement: string; + renewStatement?: string; + ca?: string | undefined; + }; + } | { - type: DynamicSecretProviders.Cassandra; - inputs: { - host: string; - port: number; - keyspace?: string; - localDataCenter: string; - username: string; - password: string; - creationStatement: string; - revocationStatement: string; - renewStatement?: string; - ca?: string | undefined; - }; - } + type: DynamicSecretProviders.Cassandra; + inputs: { + host: string; + port: number; + keyspace?: string; + localDataCenter: string; + username: string; + password: string; + creationStatement: string; + revocationStatement: string; + renewStatement?: string; + ca?: string | undefined; + }; + } | { - type: DynamicSecretProviders.AwsIam; - inputs: { - accessKey: string; - secretAccessKey: string; - region: string; - awsPath?: string; - policyDocument?: string; - userGroups?: string; - policyArns?: string; + type: DynamicSecretProviders.AwsIam; + inputs: { + accessKey: string; + secretAccessKey: string; + region: string; + awsPath?: string; + policyDocument?: string; + userGroups?: string; + policyArns?: string; + }; + } + | { + type: DynamicSecretProviders.Redis; + inputs: { + host: string; + port: number; + username: string; + password?: string; + creationStatement: string; + renewStatement?: string; + revocationStatement: string; + ca?: string | undefined; + }; }; - }; export type TCreateDynamicSecretDTO = { projectSlug: string; diff --git a/frontend/src/views/SecretMainPage/components/ActionBar/CreateDynamicSecretForm/CreateDynamicSecretForm.tsx b/frontend/src/views/SecretMainPage/components/ActionBar/CreateDynamicSecretForm/CreateDynamicSecretForm.tsx index 760e845ea..e2a40ea9a 100644 --- a/frontend/src/views/SecretMainPage/components/ActionBar/CreateDynamicSecretForm/CreateDynamicSecretForm.tsx +++ b/frontend/src/views/SecretMainPage/components/ActionBar/CreateDynamicSecretForm/CreateDynamicSecretForm.tsx @@ -9,6 +9,7 @@ import { DynamicSecretProviders } from "@app/hooks/api/dynamicSecret/types"; import { AwsIamInputForm } from "./AwsIamInputForm"; import { CassandraInputForm } from "./CassandraInputForm"; +import { RedisInputForm } from "./RedisInputForm"; import { SqlDatabaseInputForm } from "./SqlDatabaseInputForm"; type Props = { @@ -35,6 +36,11 @@ const DYNAMIC_SECRET_LIST = [ provider: DynamicSecretProviders.Cassandra, title: "Cassandra" }, + { + icon: faDatabase, + provider: DynamicSecretProviders.Redis, + title: "Redis" + }, { icon: faAws, provider: DynamicSecretProviders.AwsIam, @@ -118,6 +124,24 @@ export const CreateDynamicSecretForm = ({ /> )} + {wizardStep === WizardSteps.ProviderInputs && + selectedProvider === DynamicSecretProviders.Redis && ( + + + + )} {wizardStep === WizardSteps.ProviderInputs && selectedProvider === DynamicSecretProviders.Cassandra && ( { + const valMs = ms(val); + if (valMs < 60 * 1000) + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be a greater than 1min" }); + // a day + if (valMs > 24 * 60 * 60 * 1000) + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than a day" }); + }), + maxTTL: z + .string() + .optional() + .superRefine((val, ctx) => { + if (!val) return; + const valMs = ms(val); + if (valMs < 60 * 1000) + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be a greater than 1min" }); + // a day + if (valMs > 24 * 60 * 60 * 1000) + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "TTL must be less than a day" }); + }), + name: z.string().refine((val) => val.toLowerCase() === val, "Must be lowercase") +}); +type TForm = z.infer; + +type Props = { + onCompleted: () => void; + onCancel: () => void; + secretPath: string; + projectSlug: string; + environment: string; +}; + +export const RedisInputForm = ({ + onCompleted, + onCancel, + environment, + secretPath, + projectSlug +}: Props) => { + const { + control, + formState: { isSubmitting }, + handleSubmit + } = useForm({ + resolver: zodResolver(formSchema), + defaultValues: { + provider: { + username: "default", + creationStatement: "ACL SETUSER {{username}} on >{{password}} ~* &* +@all", + revocationStatement: "ACL DELUSER {{username}}" + } + } + }); + + const createDynamicSecret = useCreateDynamicSecret(); + + const handleCreateDynamicSecret = async ({ name, maxTTL, provider, defaultTTL }: TForm) => { + // wait till previous request is finished + if (createDynamicSecret.isLoading) return; + try { + await createDynamicSecret.mutateAsync({ + provider: { type: DynamicSecretProviders.Redis, inputs: provider }, + maxTTL, + name, + path: secretPath, + defaultTTL, + projectSlug, + environmentSlug: environment + }); + onCompleted(); + } catch (err) { + createNotification({ + type: "error", + text: "Failed to create dynamic secret" + }); + } + }; + + return ( +
+
+
+
+
+ ( + + + + )} + /> +
+
+ ( + } + isError={Boolean(error?.message)} + errorText={error?.message} + > + + + )} + /> +
+
+ ( + } + isError={Boolean(error?.message)} + errorText={error?.message} + > + + + )} + /> +
+
+
+
+ Configuration +
+
+
+ ( + + + + )} + /> + ( + + + + )} + /> +
+
+ ( + + + + )} + /> + ( + + + + )} + /> +
+
+ ( + + + + )} + /> + + + Modify Redis Statements + + ( + +