diff --git a/cli/example-agent-config.yaml b/cli/example-agent-config.yaml new file mode 100644 index 000000000..0afbe66f0 --- /dev/null +++ b/cli/example-agent-config.yaml @@ -0,0 +1,17 @@ +infisical: + address: "http://localhost:8080" +auth: + type: "token" + config: + token-path: "./role-id" +sinks: + - type: "file" + config: + path: "/Users/maidulislam/Desktop/test/infisical-token" + - type: "file" + config: + path: "access-token" + - type: "file" + config: + path: "maiduls-access-token" +templates: diff --git a/cli/packages/api/api.go b/cli/packages/api/api.go index 1be58985c..56d96f3fb 100644 --- a/cli/packages/api/api.go +++ b/cli/packages/api/api.go @@ -3,6 +3,7 @@ package api import ( "fmt" "net/http" + "strings" "github.com/Infisical/infisical-merge/packages/config" "github.com/go-resty/resty/v2" @@ -359,3 +360,50 @@ func CallCreateServiceToken(httpClient *resty.Client, request CreateServiceToken return createServiceTokenResponse, nil } + +func CallServiceTokenV3Refresh(httpClient *resty.Client, request ServiceTokenV3RefreshTokenRequest) (ServiceTokenV3RefreshTokenResponse, error) { + var serviceTokenV3RefreshTokenResponse ServiceTokenV3RefreshTokenResponse + response, err := httpClient. + R(). + SetResult(&serviceTokenV3RefreshTokenResponse). + SetHeader("User-Agent", USER_AGENT). + SetBody(request). + Post(fmt.Sprintf("%v/v3/service-token/me/token", config.INFISICAL_URL)) + + if err != nil { + return ServiceTokenV3RefreshTokenResponse{}, fmt.Errorf("CallServiceTokenV3Refresh: Unable to complete api request [err=%s]", err) + } + + if response.IsError() { + return ServiceTokenV3RefreshTokenResponse{}, fmt.Errorf("CallServiceTokenV3Refresh: Unsuccessful response [%v %v] [status-code=%v] [response=%v]", response.Request.Method, response.Request.URL, response.StatusCode(), response.String()) + } + + return serviceTokenV3RefreshTokenResponse, nil +} + +func CallGetRawSecretsV3(httpClient *resty.Client, request GetRawSecretsV3Request) (GetRawSecretsV3Response, error) { + var getRawSecretsV3Response GetRawSecretsV3Response + response, err := httpClient. + R(). + SetResult(&getRawSecretsV3Response). + SetHeader("User-Agent", USER_AGENT). + SetBody(request). + SetQueryParam("workspaceId", request.WorkspaceId). + SetQueryParam("environment", request.Environment). + SetQueryParam("include_imports", "false"). + Get(fmt.Sprintf("%v/v3/secrets/raw", config.INFISICAL_URL)) + + if err != nil { + return GetRawSecretsV3Response{}, fmt.Errorf("CallGetRawSecretsV3: Unable to complete api request [err=%w]", err) + } + + if response.IsError() && strings.Contains(response.String(), "Failed to find bot key") { + return GetRawSecretsV3Response{}, fmt.Errorf("project with id %s is a legacy project type, please navigate to project settings and disable end to end encryption then try again", request.WorkspaceId) + } + + if response.IsError() { + return GetRawSecretsV3Response{}, fmt.Errorf("CallGetRawSecretsV3: Unsuccessful response [%v %v] [status-code=%v]", response.Request.Method, response.Request.URL, response.StatusCode()) + } + + return getRawSecretsV3Response, nil +} diff --git a/cli/packages/api/model.go b/cli/packages/api/model.go index 1531dce36..14aa48b30 100644 --- a/cli/packages/api/model.go +++ b/cli/packages/api/model.go @@ -421,3 +421,34 @@ type CreateServiceTokenResponse struct { ServiceToken string `json:"serviceToken"` ServiceTokenData ServiceTokenData `json:"serviceTokenData"` } + +type ServiceTokenV3RefreshTokenRequest struct { + RefreshToken string `json:"refresh_token"` +} +type ServiceTokenV3RefreshTokenResponse struct { + RefreshToken string `json:"refresh_token"` + AccessToken string `json:"access_token"` + ExpiresIn int `json:"expires_in"` + TokenType string `json:"token_type"` +} + +type GetRawSecretsV3Request struct { + Environment string `json:"environment"` + WorkspaceId string `json:"workspaceId"` + SecretPath string `json:"secretPath"` + IncludeImport bool `json:"include_imports"` +} + +type GetRawSecretsV3Response struct { + Secrets []struct { + ID string `json:"_id"` + Version int `json:"version"` + Workspace string `json:"workspace"` + Type string `json:"type"` + Environment string `json:"environment"` + SecretKey string `json:"secretKey"` + SecretValue string `json:"secretValue"` + SecretComment string `json:"secretComment"` + } `json:"secrets"` + Imports []any `json:"imports"` +} diff --git a/cli/packages/cmd/agent.go b/cli/packages/cmd/agent.go new file mode 100644 index 000000000..7af471a03 --- /dev/null +++ b/cli/packages/cmd/agent.go @@ -0,0 +1,327 @@ +/* +Copyright (c) 2023 Infisical Inc. +*/ +package cmd + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "os" + "os/signal" + "strings" + "syscall" + "text/template" + "time" + + "github.com/rs/zerolog/log" + "gopkg.in/yaml.v2" + + "github.com/Infisical/infisical-merge/packages/api" + "github.com/Infisical/infisical-merge/packages/config" + "github.com/Infisical/infisical-merge/packages/models" + "github.com/Infisical/infisical-merge/packages/util" + "github.com/go-resty/resty/v2" + "github.com/spf13/cobra" +) + +const DEFAULT_INFISICAL_CLOUD_URL = "https://app.infisical.com" + +type Config struct { + Infisical InfisicalConfig `yaml:"infisical"` + Auth AuthConfig `yaml:"auth"` + Sinks []Sink `yaml:"sinks"` + Templates []Template `yaml:"templates"` +} + +type InfisicalConfig struct { + Address string `yaml:"address"` +} + +type AuthConfig struct { + Type string `yaml:"type"` + Config interface{} `yaml:"config"` +} + +type TokenAuthConfig struct { + TokenPath string `yaml:"token-path"` +} + +type OAuthConfig struct { + ClientID string `yaml:"client-id"` + ClientSecret string `yaml:"client-secret"` +} + +type Sink struct { + Type string `yaml:"type"` + Config SinkDetails `yaml:"config"` +} + +type SinkDetails struct { + Path string `yaml:"path"` +} + +type Template struct { + SourcePath string `yaml:"source-path"` + DestinationPath string `yaml:"destination-path"` +} + +func ReadFile(filePath string) ([]byte, error) { + return ioutil.ReadFile(filePath) +} + +func FileExists(filepath string) bool { + info, err := os.Stat(filepath) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() +} + +// WriteToFile writes data to the specified file path. +func WriteBytesToFile(data *bytes.Buffer, outputPath string) error { + outputFile, err := os.Create(outputPath) + if err != nil { + return err + } + defer outputFile.Close() + + _, err = outputFile.Write(data.Bytes()) + return err +} + +func appendAPIEndpoint(address string) string { + // Ensure the address does not already end with "/api" + if strings.HasSuffix(address, "/api") { + return address + } + + // Check if the address ends with a slash and append accordingly + if address[len(address)-1] == '/' { + return address + "api" + } + return address + "/api" +} + +func ParseAgentConfig(filePath string) (*Config, error) { + data, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + + var rawConfig struct { + Infisical InfisicalConfig `yaml:"infisical"` + Auth struct { + Type string `yaml:"type"` + Config map[string]interface{} `yaml:"config"` + } `yaml:"auth"` + Sinks []Sink `yaml:"sinks"` + Templates []Template `yaml:"templates"` + } + + if err := yaml.Unmarshal(data, &rawConfig); err != nil { + return nil, err + } + + // Set defaults + if rawConfig.Infisical.Address == "" { + rawConfig.Infisical.Address = DEFAULT_INFISICAL_CLOUD_URL + } + + config.INFISICAL_URL = appendAPIEndpoint(rawConfig.Infisical.Address) + + log.Info().Msgf("Infisical instance address set to %s", rawConfig.Infisical.Address) + + config := &Config{ + Infisical: rawConfig.Infisical, + Auth: AuthConfig{ + Type: rawConfig.Auth.Type, + }, + Sinks: rawConfig.Sinks, + Templates: rawConfig.Templates, + } + + // Marshal and then unmarshal the config based on the type + configBytes, err := yaml.Marshal(rawConfig.Auth.Config) + if err != nil { + return nil, err + } + + switch rawConfig.Auth.Type { + case "token": + var tokenConfig TokenAuthConfig + if err := yaml.Unmarshal(configBytes, &tokenConfig); err != nil { + return nil, err + } + config.Auth.Config = tokenConfig + case "oauth": // aws, gcp, k8s service account, etc + var oauthConfig OAuthConfig + if err := yaml.Unmarshal(configBytes, &oauthConfig); err != nil { + return nil, err + } + config.Auth.Config = oauthConfig + default: + return nil, fmt.Errorf("unknown auth type: %s", rawConfig.Auth.Type) + } + + return config, nil +} + +func secretTemplateFunction(accessToken string) func(string, string, string) ([]models.SingleEnvironmentVariable, error) { + return func(projectID, envSlug, secretPath string) ([]models.SingleEnvironmentVariable, error) { + secrets, err := util.GetPlainTextSecretsViaMachineIdentity(accessToken, projectID, envSlug, secretPath, false) + if err != nil { + return nil, err + } + + return secrets, nil + } +} + +func ProcessTemplate(templatePath string, data interface{}, accessToken string) (*bytes.Buffer, error) { + // custom template function to fetch secrets from Infisical + secretFunction := secretTemplateFunction(accessToken) + funcs := template.FuncMap{ + "secret": secretFunction, + } + + tmpl, err := template.New(templatePath).Funcs(funcs).ParseFiles(templatePath) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return nil, err + } + + return &buf, nil +} + +func refreshTokenAndProcessTemplate(refreshToken string, config *Config, errChan chan error) { + for { + httpClient := resty.New() + httpClient.SetRetryCount(10000). + SetRetryMaxWaitTime(20 * time.Second). + SetRetryWaitTime(5 * time.Second) + + tokenResponse, err := api.CallServiceTokenV3Refresh(httpClient, api.ServiceTokenV3RefreshTokenRequest{RefreshToken: refreshToken}) + if err != nil { + errChan <- fmt.Errorf("unable to complete renewal because [%s]", err) + } + + for _, sinkFile := range config.Sinks { + if sinkFile.Type == "file" { + err = ioutil.WriteFile(sinkFile.Config.Path, []byte(tokenResponse.AccessToken), 0644) + if err != nil { + errChan <- err + return + } + } else { + errChan <- errors.New("unsupported sink type. Only 'file' type is supported") + return + } + } + + refreshToken = tokenResponse.RefreshToken + nextRefreshCycle := time.Duration(tokenResponse.ExpiresIn-5) * time.Second // when the next access refresh will happen + + d, err := time.ParseDuration(nextRefreshCycle.String()) + if err != nil { + errChan <- fmt.Errorf("unable to parse refresh time because %s", err) + return + } + + log.Info().Msgf("token refreshed and saved to selected path; next cycle will occur in %s", d.String()) + + for _, secretTemplate := range config.Templates { + processedTemplate, err := ProcessTemplate(secretTemplate.SourcePath, nil, tokenResponse.AccessToken) + if err != nil { + errChan <- err + return + } + + if err := WriteBytesToFile(processedTemplate, secretTemplate.DestinationPath); err != nil { + errChan <- err + return + } + + log.Info().Msgf("secret template at path %s has been rendered and saved to path %s", secretTemplate.SourcePath, secretTemplate.DestinationPath) + } + + time.Sleep(nextRefreshCycle) + } +} + +// runCmd represents the run command +var agentCmd = &cobra.Command{ + Example: ` + infisical agent + `, + Use: "agent", + Short: "agent", + DisableFlagsInUseLine: true, + Run: func(cmd *cobra.Command, args []string) { + + log.Info().Msg("starting Infisical agent...") + + configPath, err := cmd.Flags().GetString("config") + if err != nil { + util.HandleError(err, "Unable to parse flag config") + } + + if !FileExists(configPath) { + log.Error().Msgf("Unable to locate %s. The provided agent config file path is either missing or incorrect", configPath) + return + } + + agentConfig, err := ParseAgentConfig(configPath) + if err != nil { + log.Error().Msgf("Unable to prase %s because %v. Please ensure that is follows the Infisical Agent config structure", configPath, err) + return + } + + errChan := make(chan error) + sigChan := make(chan os.Signal, 1) + + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + + switch configAuthType := agentConfig.Auth.Config.(type) { + case TokenAuthConfig: + content, err := ReadFile(configAuthType.TokenPath) + if err != nil { + log.Error().Msgf("unable to read initial token from file path %s because %v", configAuthType.TokenPath, err) + return + } + + refreshToken := string(content) + go refreshTokenAndProcessTemplate(refreshToken, agentConfig, errChan) + + case OAuthConfig: + // future auth types + default: + log.Error().Msgf("unknown auth config type. Only 'file' type is supported") + return + } + + select { + case err := <-errChan: + log.Fatal().Msgf("agent stopped due to error: %v", err) + os.Exit(1) + case <-sigChan: + log.Info().Msg("agent is gracefully shutting...") + os.Exit(1) + } + + }, +} + +func init() { + agentCmd.SetHelpFunc(func(command *cobra.Command, strings []string) { + command.Flags().MarkHidden("domain") + command.Parent().HelpFunc()(command, strings) + }) + agentCmd.Flags().String("config", "agent-config.yaml", "The path to agent config yaml file") + rootCmd.AddCommand(agentCmd) +} diff --git a/cli/packages/util/secrets.go b/cli/packages/util/secrets.go index 73be43319..a3f794f78 100644 --- a/cli/packages/util/secrets.go +++ b/cli/packages/util/secrets.go @@ -152,6 +152,46 @@ func GetPlainTextSecretsViaJTW(JTWToken string, receiversPrivateKey string, work return plainTextSecrets, nil } +func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool) ([]models.SingleEnvironmentVariable, error) { + httpClient := resty.New() + httpClient.SetAuthToken(accessToken). + SetHeader("Accept", "application/json") + + getSecretsRequest := api.GetEncryptedSecretsV3Request{ + WorkspaceId: workspaceId, + Environment: environmentName, + IncludeImport: includeImports, + // TagSlugs: tagSlugs, + } + + if secretsPath != "" { + getSecretsRequest.SecretPath = secretsPath + } + + rawSecrets, err := api.CallGetRawSecretsV3(httpClient, api.GetRawSecretsV3Request{WorkspaceId: workspaceId, SecretPath: environmentName, Environment: environmentName}) + if err != nil { + return nil, err + } + + plainTextSecrets := []models.SingleEnvironmentVariable{} + if err != nil { + return nil, fmt.Errorf("unable to decrypt your secrets [err=%v]", err) + } + + for _, secret := range rawSecrets.Secrets { + plainTextSecrets = append(plainTextSecrets, models.SingleEnvironmentVariable{Key: secret.SecretKey, Value: secret.SecretValue}) + } + + // if includeImports { + // plainTextSecrets, err = InjectImportedSecret(plainTextWorkspaceKey, plainTextSecrets, encryptedSecrets.ImportedSecrets) + // if err != nil { + // return nil, err + // } + // } + + return plainTextSecrets, nil +} + func InjectImportedSecret(plainTextWorkspaceKey []byte, secrets []models.SingleEnvironmentVariable, importedSecrets []api.ImportedSecretV3) ([]models.SingleEnvironmentVariable, error) { if importedSecrets == nil { return secrets, nil diff --git a/cli/secret-render-template b/cli/secret-render-template new file mode 100644 index 000000000..32ab2331a --- /dev/null +++ b/cli/secret-render-template @@ -0,0 +1,5 @@ +{{- with secret "6553ccb2b7da580d7f6e7260" "dev" "/" }} +{{- range . }} +{{ .Key }}={{ .Value }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/docs/images/agent/infisical-agent-diagram.png b/docs/images/agent/infisical-agent-diagram.png new file mode 100644 index 000000000..27356ba11 Binary files /dev/null and b/docs/images/agent/infisical-agent-diagram.png differ diff --git a/docs/infisical-agent/overview.mdx b/docs/infisical-agent/overview.mdx new file mode 100644 index 000000000..1468a805c --- /dev/null +++ b/docs/infisical-agent/overview.mdx @@ -0,0 +1,93 @@ +--- +title: "Infisical Agent" +--- + +Infisical Agent is a client daemon that simplifies the adoption of Infisical by providing a more scalable and user-friendly approach for applications to interact with Infisical. +It eliminates the need to modify application logic by enabling clients to decide how they want their secrets rendered through the use of templates. + + + +### Key features: +- Token renewal: Automatically authenticates with Infisical and deposits renewed access tokens at specified path for applications to consume +- Templating: Renders secrets via user provided templates to desired formats for applications to consume + +### Token renewal +The Infisical agent can help manage the life cycle of access tokens. The token renewal process is split into two main components: a Method, which is the authentication process suitable for your current setup, and Sinks, which are the places where the agent deposits the new access token whenever it receives updates. + +When the Infisical Agent is started, it will attempts to obtain a valid access token using the authentication method you have configured. If the agent is unable to fetch a valid token, the agent will keep trying, increasing the time between each attempt. + +Once a access token is successfully fetched, the agent will make sure the access token stays valid, continuing to renew it before it expires. + +Every time the agent successfully retrieves a new access token, it writes the new token to the Sinks you've configured. + + + Access tokens can be utilized with Infisical SDKs or directly in API requests to retrieve secrets from Infisical + + +### Templating +The Infisical agent can help deliver formatted secrets to your application in a variety of environments. To achieve this, the agent will retrieve secrets from Infisical, format them using a specified template, and then save these formatted secrets to a designated file path. + +Templating process is done through the use of Go language's [text/template feature](https://pkg.go.dev/text/template). Multiple template definitions can be set in the agent configuration file to generate a variety of formatted secret files. + +When the agent is started and templates are defined in the agent configuration file, the agent will attempt to acquire a valid access token using the set authentication method outlined in the agent's configuration. +If this initial attempt is unsuccessful, the agent will momentarily pauses before continuing to make more attempts. + +Once the agent successfully obtains a valid access token, the agent proceeds to fetch the secrets from Infisical using it. +It then formats these secrets using the user provided templates and writes the formatted data to configured file paths. + +## Agent configuration file + +To set up the authentication method for token renewal and to define secret templates, the Infisical agent requires a YAML configuration file containing properties defined below. +While specifying an authentication method is mandatory to start the agent, configuring sinks and secret templates are optional. + +| Field | Description | +| ---------------------------- | ----------- | +| `infisical.address` | The URL of the Infisical service. Default: `"https://app.infisical.com"`. | +| `auth.type` | The type of authentication method used. Only `"token"` type is currently available | +| `auth.config.token-path` | The file path where the initial token for authentication is stored. | +| `sinks[].type` | The type of sink in a list of sinks. Each item specifies a sink type. Currently, only `"file"` type is available. | +| `sinks[].config.path` | The file path where the access token should be stored for each sink in the list. | +| `templates[].source-path` | The path to the template file that should be used to render secrets. | +| `templates[].destination-path` | The path where the rendered secrets from the source template will be saved to. | + + +## Quick start Infisical Agent +To install the Infisical agent, you must first install the [Infisical CLI](../cli/overview) in the desired environment where you'd like the agent to run. This is because the Infisical agent is a sub-command of the Infisical CLI. + +Once you have the CLI installed, you will need to create a agent configuration file in yaml. + +```yaml example-agent-config-file.yaml +infisical: + address: "https://app.infisical.com" +auth: + type: "token" + config: + token-path: "/path/to/initial/token" +sinks: + - type: "file" + config: + path: "/some/path/to/store/access-token/file-name" +templates: + - source-path: my-dot-ev-secret-template + destination-path: /some/path/.env +``` + +Above is an example agent configuration file that defines the token authentication method, one sink location (where to deposit access tokens after renewal) and a secret template. + + +```text my-dot-ev-secret-template +{{- with secret "6553ccb2b7da580d7f6e7260" "dev" "/" }} +{{- range . }} +{{ .Key }}={{ .Value }} +{{- end }} +{{- end }} +``` + +The secret template above will be used to render the secrets where the key and the value are separated by `=` sign. You'll notice that a custom function named `secret` is used to fetch the secrets. +This function takes the following arguments: `secret "" "" ""`. + +```bash +infisical agent --config example-agent-config-file.yaml +``` + +After defining the agent configuration file, run the command above pointing to the path where the agent configuration is located. \ No newline at end of file diff --git a/docs/mint.json b/docs/mint.json index 6ab1b4624..f196424f9 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -197,6 +197,12 @@ "cli/faq" ] }, + { + "group": "Agent", + "pages": [ + "infisical-agent/overview" + ] + }, { "group": "Integrations", "pages": ["integrations/overview"]