Merge pull request #1196 from Infisical/infisica-agent

Infisical Agent

cli/example-agent-config.yaml

@@ -0,0 +1,17 @@
+infisical:
+  address: "http://localhost:8080"
+auth:
+  type: "token"
+  config:
+    token-path: "./role-id"
+sinks:
+  - type: "file"
+    config:
+      path: "/Users/maidulislam/Desktop/test/infisical-token"
+  - type: "file"
+    config:
+      path: "access-token"
+  - type: "file"
+    config:
+      path: "maiduls-access-token"
+templates:

cli/packages/api/api.go

@@ -3,6 +3,7 @@ package api
 import (
 	"fmt"
 	"net/http"
+	"strings"
 
 	"github.com/Infisical/infisical-merge/packages/config"
 	"github.com/go-resty/resty/v2"
@@ -359,3 +360,50 @@ func CallCreateServiceToken(httpClient *resty.Client, request CreateServiceToken
 
 	return createServiceTokenResponse, nil
 }
+
+func CallServiceTokenV3Refresh(httpClient *resty.Client, request ServiceTokenV3RefreshTokenRequest) (ServiceTokenV3RefreshTokenResponse, error) {
+	var serviceTokenV3RefreshTokenResponse ServiceTokenV3RefreshTokenResponse
+	response, err := httpClient.
+		R().
+		SetResult(&serviceTokenV3RefreshTokenResponse).
+		SetHeader("User-Agent", USER_AGENT).
+		SetBody(request).
+		Post(fmt.Sprintf("%v/v3/service-token/me/token", config.INFISICAL_URL))
+
+	if err != nil {
+		return ServiceTokenV3RefreshTokenResponse{}, fmt.Errorf("CallServiceTokenV3Refresh: Unable to complete api request [err=%s]", err)
+	}
+
+	if response.IsError() {
+		return ServiceTokenV3RefreshTokenResponse{}, fmt.Errorf("CallServiceTokenV3Refresh: Unsuccessful response [%v %v] [status-code=%v] [response=%v]", response.Request.Method, response.Request.URL, response.StatusCode(), response.String())
+	}
+
+	return serviceTokenV3RefreshTokenResponse, nil
+}
+
+func CallGetRawSecretsV3(httpClient *resty.Client, request GetRawSecretsV3Request) (GetRawSecretsV3Response, error) {
+	var getRawSecretsV3Response GetRawSecretsV3Response
+	response, err := httpClient.
+		R().
+		SetResult(&getRawSecretsV3Response).
+		SetHeader("User-Agent", USER_AGENT).
+		SetBody(request).
+		SetQueryParam("workspaceId", request.WorkspaceId).
+		SetQueryParam("environment", request.Environment).
+		SetQueryParam("include_imports", "false").
+		Get(fmt.Sprintf("%v/v3/secrets/raw", config.INFISICAL_URL))
+
+	if err != nil {
+		return GetRawSecretsV3Response{}, fmt.Errorf("CallGetRawSecretsV3: Unable to complete api request [err=%w]", err)
+	}
+
+	if response.IsError() && strings.Contains(response.String(), "Failed to find bot key") {
+		return GetRawSecretsV3Response{}, fmt.Errorf("project with id %s is a legacy project type, please navigate to project settings and disable end to end encryption then try again", request.WorkspaceId)
+	}
+
+	if response.IsError() {
+		return GetRawSecretsV3Response{}, fmt.Errorf("CallGetRawSecretsV3: Unsuccessful response [%v %v] [status-code=%v]", response.Request.Method, response.Request.URL, response.StatusCode())
+	}
+
+	return getRawSecretsV3Response, nil
+}

cli/packages/api/model.go

@@ -421,3 +421,34 @@ type CreateServiceTokenResponse struct {
 	ServiceToken     string           `json:"serviceToken"`
 	ServiceTokenData ServiceTokenData `json:"serviceTokenData"`
 }
+
+type ServiceTokenV3RefreshTokenRequest struct {
+	RefreshToken string `json:"refresh_token"`
+}
+type ServiceTokenV3RefreshTokenResponse struct {
+	RefreshToken string `json:"refresh_token"`
+	AccessToken  string `json:"access_token"`
+	ExpiresIn    int    `json:"expires_in"`
+	TokenType    string `json:"token_type"`
+}
+
+type GetRawSecretsV3Request struct {
+	Environment   string `json:"environment"`
+	WorkspaceId   string `json:"workspaceId"`
+	SecretPath    string `json:"secretPath"`
+	IncludeImport bool   `json:"include_imports"`
+}
+
+type GetRawSecretsV3Response struct {
+	Secrets []struct {
+		ID            string `json:"_id"`
+		Version       int    `json:"version"`
+		Workspace     string `json:"workspace"`
+		Type          string `json:"type"`
+		Environment   string `json:"environment"`
+		SecretKey     string `json:"secretKey"`
+		SecretValue   string `json:"secretValue"`
+		SecretComment string `json:"secretComment"`
+	} `json:"secrets"`
+	Imports []any `json:"imports"`
+}

cli/packages/cmd/agent.go

@@ -0,0 +1,327 @@
+/*
+Copyright (c) 2023 Infisical Inc.
+*/
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"text/template"
+	"time"
+
+	"github.com/rs/zerolog/log"
+	"gopkg.in/yaml.v2"
+
+	"github.com/Infisical/infisical-merge/packages/api"
+	"github.com/Infisical/infisical-merge/packages/config"
+	"github.com/Infisical/infisical-merge/packages/models"
+	"github.com/Infisical/infisical-merge/packages/util"
+	"github.com/go-resty/resty/v2"
+	"github.com/spf13/cobra"
+)
+
+const DEFAULT_INFISICAL_CLOUD_URL = "https://app.infisical.com"
+
+type Config struct {
+	Infisical InfisicalConfig `yaml:"infisical"`
+	Auth      AuthConfig      `yaml:"auth"`
+	Sinks     []Sink          `yaml:"sinks"`
+	Templates []Template      `yaml:"templates"`
+}
+
+type InfisicalConfig struct {
+	Address string `yaml:"address"`
+}
+
+type AuthConfig struct {
+	Type   string      `yaml:"type"`
+	Config interface{} `yaml:"config"`
+}
+
+type TokenAuthConfig struct {
+	TokenPath string `yaml:"token-path"`
+}
+
+type OAuthConfig struct {
+	ClientID     string `yaml:"client-id"`
+	ClientSecret string `yaml:"client-secret"`
+}
+
+type Sink struct {
+	Type   string      `yaml:"type"`
+	Config SinkDetails `yaml:"config"`
+}
+
+type SinkDetails struct {
+	Path string `yaml:"path"`
+}
+
+type Template struct {
+	SourcePath      string `yaml:"source-path"`
+	DestinationPath string `yaml:"destination-path"`
+}
+
+func ReadFile(filePath string) ([]byte, error) {
+	return ioutil.ReadFile(filePath)
+}
+
+func FileExists(filepath string) bool {
+	info, err := os.Stat(filepath)
+	if os.IsNotExist(err) {
+		return false
+	}
+	return !info.IsDir()
+}
+
+// WriteToFile writes data to the specified file path.
+func WriteBytesToFile(data *bytes.Buffer, outputPath string) error {
+	outputFile, err := os.Create(outputPath)
+	if err != nil {
+		return err
+	}
+	defer outputFile.Close()
+
+	_, err = outputFile.Write(data.Bytes())
+	return err
+}
+
+func appendAPIEndpoint(address string) string {
+	// Ensure the address does not already end with "/api"
+	if strings.HasSuffix(address, "/api") {
+		return address
+	}
+
+	// Check if the address ends with a slash and append accordingly
+	if address[len(address)-1] == '/' {
+		return address + "api"
+	}
+	return address + "/api"
+}
+
+func ParseAgentConfig(filePath string) (*Config, error) {
+	data, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	var rawConfig struct {
+		Infisical InfisicalConfig `yaml:"infisical"`
+		Auth      struct {
+			Type   string                 `yaml:"type"`
+			Config map[string]interface{} `yaml:"config"`
+		} `yaml:"auth"`
+		Sinks     []Sink     `yaml:"sinks"`
+		Templates []Template `yaml:"templates"`
+	}
+
+	if err := yaml.Unmarshal(data, &rawConfig); err != nil {
+		return nil, err
+	}
+
+	// Set defaults
+	if rawConfig.Infisical.Address == "" {
+		rawConfig.Infisical.Address = DEFAULT_INFISICAL_CLOUD_URL
+	}
+
+	config.INFISICAL_URL = appendAPIEndpoint(rawConfig.Infisical.Address)
+
+	log.Info().Msgf("Infisical instance address set to %s", rawConfig.Infisical.Address)
+
+	config := &Config{
+		Infisical: rawConfig.Infisical,
+		Auth: AuthConfig{
+			Type: rawConfig.Auth.Type,
+		},
+		Sinks:     rawConfig.Sinks,
+		Templates: rawConfig.Templates,
+	}
+
+	// Marshal and then unmarshal the config based on the type
+	configBytes, err := yaml.Marshal(rawConfig.Auth.Config)
+	if err != nil {
+		return nil, err
+	}
+
+	switch rawConfig.Auth.Type {
+	case "token":
+		var tokenConfig TokenAuthConfig
+		if err := yaml.Unmarshal(configBytes, &tokenConfig); err != nil {
+			return nil, err
+		}
+		config.Auth.Config = tokenConfig
+	case "oauth": // aws, gcp, k8s service account, etc
+		var oauthConfig OAuthConfig
+		if err := yaml.Unmarshal(configBytes, &oauthConfig); err != nil {
+			return nil, err
+		}
+		config.Auth.Config = oauthConfig
+	default:
+		return nil, fmt.Errorf("unknown auth type: %s", rawConfig.Auth.Type)
+	}
+
+	return config, nil
+}
+
+func secretTemplateFunction(accessToken string) func(string, string, string) ([]models.SingleEnvironmentVariable, error) {
+	return func(projectID, envSlug, secretPath string) ([]models.SingleEnvironmentVariable, error) {
+		secrets, err := util.GetPlainTextSecretsViaMachineIdentity(accessToken, projectID, envSlug, secretPath, false)
+		if err != nil {
+			return nil, err
+		}
+
+		return secrets, nil
+	}
+}
+
+func ProcessTemplate(templatePath string, data interface{}, accessToken string) (*bytes.Buffer, error) {
+	// custom template function to fetch secrets from Infisical
+	secretFunction := secretTemplateFunction(accessToken)
+	funcs := template.FuncMap{
+		"secret": secretFunction,
+	}
+
+	tmpl, err := template.New(templatePath).Funcs(funcs).ParseFiles(templatePath)
+	if err != nil {
+		return nil, err
+	}
+
+	var buf bytes.Buffer
+	if err := tmpl.Execute(&buf, data); err != nil {
+		return nil, err
+	}
+
+	return &buf, nil
+}
+
+func refreshTokenAndProcessTemplate(refreshToken string, config *Config, errChan chan error) {
+	for {
+		httpClient := resty.New()
+		httpClient.SetRetryCount(10000).
+			SetRetryMaxWaitTime(20 * time.Second).
+			SetRetryWaitTime(5 * time.Second)
+
+		tokenResponse, err := api.CallServiceTokenV3Refresh(httpClient, api.ServiceTokenV3RefreshTokenRequest{RefreshToken: refreshToken})
+		if err != nil {
+			errChan <- fmt.Errorf("unable to complete renewal because [%s]", err)
+		}
+
+		for _, sinkFile := range config.Sinks {
+			if sinkFile.Type == "file" {
+				err = ioutil.WriteFile(sinkFile.Config.Path, []byte(tokenResponse.AccessToken), 0644)
+				if err != nil {
+					errChan <- err
+					return
+				}
+			} else {
+				errChan <- errors.New("unsupported sink type. Only 'file' type is supported")
+				return
+			}
+		}
+
+		refreshToken = tokenResponse.RefreshToken
+		nextRefreshCycle := time.Duration(tokenResponse.ExpiresIn-5) * time.Second // when the next access refresh will happen
+
+		d, err := time.ParseDuration(nextRefreshCycle.String())
+		if err != nil {
+			errChan <- fmt.Errorf("unable to parse refresh time because %s", err)
+			return
+		}
+
+		log.Info().Msgf("token refreshed and saved to selected path; next cycle will occur in %s", d.String())
+
+		for _, secretTemplate := range config.Templates {
+			processedTemplate, err := ProcessTemplate(secretTemplate.SourcePath, nil, tokenResponse.AccessToken)
+			if err != nil {
+				errChan <- err
+				return
+			}
+
+			if err := WriteBytesToFile(processedTemplate, secretTemplate.DestinationPath); err != nil {
+				errChan <- err
+				return
+			}
+
+			log.Info().Msgf("secret template at path %s has been rendered and saved to path %s", secretTemplate.SourcePath, secretTemplate.DestinationPath)
+		}
+
+		time.Sleep(nextRefreshCycle)
+	}
+}
+
+// runCmd represents the run command
+var agentCmd = &cobra.Command{
+	Example: `
+	infisical agent
+	`,
+	Use:                   "agent",
+	Short:                 "agent",
+	DisableFlagsInUseLine: true,
+	Run: func(cmd *cobra.Command, args []string) {
+
+		log.Info().Msg("starting Infisical agent...")
+
+		configPath, err := cmd.Flags().GetString("config")
+		if err != nil {
+			util.HandleError(err, "Unable to parse flag config")
+		}
+
+		if !FileExists(configPath) {
+			log.Error().Msgf("Unable to locate %s. The provided agent config file path is either missing or incorrect", configPath)
+			return
+		}
+
+		agentConfig, err := ParseAgentConfig(configPath)
+		if err != nil {
+			log.Error().Msgf("Unable to prase %s because %v. Please ensure that is follows the Infisical Agent config structure", configPath, err)
+			return
+		}
+
+		errChan := make(chan error)
+		sigChan := make(chan os.Signal, 1)
+
+		signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+
+		switch configAuthType := agentConfig.Auth.Config.(type) {
+		case TokenAuthConfig:
+			content, err := ReadFile(configAuthType.TokenPath)
+			if err != nil {
+				log.Error().Msgf("unable to read initial token from file path %s because %v", configAuthType.TokenPath, err)
+				return
+			}
+
+			refreshToken := string(content)
+			go refreshTokenAndProcessTemplate(refreshToken, agentConfig, errChan)
+
+		case OAuthConfig:
+			// future auth types
+		default:
+			log.Error().Msgf("unknown auth config type. Only 'file' type is supported")
+			return
+		}
+
+		select {
+		case err := <-errChan:
+			log.Fatal().Msgf("agent stopped due to error: %v", err)
+			os.Exit(1)
+		case <-sigChan:
+			log.Info().Msg("agent is gracefully shutting...")
+			os.Exit(1)
+		}
+
+	},
+}
+
+func init() {
+	agentCmd.SetHelpFunc(func(command *cobra.Command, strings []string) {
+		command.Flags().MarkHidden("domain")
+		command.Parent().HelpFunc()(command, strings)
+	})
+	agentCmd.Flags().String("config", "agent-config.yaml", "The path to agent config yaml file")
+	rootCmd.AddCommand(agentCmd)
+}

cli/packages/util/secrets.go

@@ -152,6 +152,46 @@ func GetPlainTextSecretsViaJTW(JTWToken string, receiversPrivateKey string, work
 	return plainTextSecrets, nil
 }
 
+func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool) ([]models.SingleEnvironmentVariable, error) {
+	httpClient := resty.New()
+	httpClient.SetAuthToken(accessToken).
+		SetHeader("Accept", "application/json")
+
+	getSecretsRequest := api.GetEncryptedSecretsV3Request{
+		WorkspaceId:   workspaceId,
+		Environment:   environmentName,
+		IncludeImport: includeImports,
+		// TagSlugs:    tagSlugs,
+	}
+
+	if secretsPath != "" {
+		getSecretsRequest.SecretPath = secretsPath
+	}
+
+	rawSecrets, err := api.CallGetRawSecretsV3(httpClient, api.GetRawSecretsV3Request{WorkspaceId: workspaceId, SecretPath: environmentName, Environment: environmentName})
+	if err != nil {
+		return nil, err
+	}
+
+	plainTextSecrets := []models.SingleEnvironmentVariable{}
+	if err != nil {
+		return nil, fmt.Errorf("unable to decrypt your secrets [err=%v]", err)
+	}
+
+	for _, secret := range rawSecrets.Secrets {
+		plainTextSecrets = append(plainTextSecrets, models.SingleEnvironmentVariable{Key: secret.SecretKey, Value: secret.SecretValue})
+	}
+
+	// if includeImports {
+	// 	plainTextSecrets, err = InjectImportedSecret(plainTextWorkspaceKey, plainTextSecrets, encryptedSecrets.ImportedSecrets)
+	// 	if err != nil {
+	// 		return nil, err
+	// 	}
+	// }
+
+	return plainTextSecrets, nil
+}
+
 func InjectImportedSecret(plainTextWorkspaceKey []byte, secrets []models.SingleEnvironmentVariable, importedSecrets []api.ImportedSecretV3) ([]models.SingleEnvironmentVariable, error) {
 	if importedSecrets == nil {
 		return secrets, nil

cli/secret-render-template

@@ -0,0 +1,5 @@
+{{- with secret "6553ccb2b7da580d7f6e7260" "dev" "/" }}
+{{- range . }}
+{{ .Key }}={{ .Value }}
+{{- end }}
+{{- end }}

docs/infisical-agent/overview.mdx

@@ -0,0 +1,93 @@
+---
+title: "Infisical Agent"
+---
+
+Infisical Agent is a client daemon that simplifies the adoption of Infisical by providing a more scalable and user-friendly approach for applications to interact with Infisical.
+It eliminates the need to modify application logic by enabling clients to decide how they want their secrets rendered through the use of templates.
+
+<img height="200" src="../images/agent/infisical-agent-diagram.png" />
+
+### Key features:
+- Token renewal: Automatically authenticates with Infisical and deposits renewed access tokens at specified path for applications to consume
+- Templating: Renders secrets via user provided templates to desired formats for applications to consume
+
+### Token renewal
+The Infisical agent can help manage the life cycle of access tokens. The token renewal process is split into two main components: a Method, which is the authentication process suitable for your current setup, and Sinks, which are the places where the agent deposits the new access token whenever it receives updates.
+
+When the Infisical Agent is started, it will attempts to obtain a valid access token using the authentication method you have configured. If the agent is unable to fetch a valid token, the agent will keep trying, increasing the time between each attempt. 
+
+Once a access token is successfully fetched, the agent will make sure the access token stays valid, continuing to renew it before it expires.
+
+Every time the agent successfully retrieves a new access token, it writes the new token to the Sinks you've configured.
+
+<Info>
+  Access tokens can be utilized with Infisical SDKs or directly in API requests to retrieve secrets from Infisical
+</Info>
+
+### Templating
+The Infisical agent can help deliver formatted secrets to your application in a variety of environments. To achieve this, the agent will retrieve secrets from Infisical, format them using a specified template, and then save these formatted secrets to a designated file path. 
+
+Templating process is done through the use of Go language's [text/template feature](https://pkg.go.dev/text/template). Multiple template definitions can be set in the agent configuration file to generate a variety of formatted secret files.
+
+When the agent is started and templates are defined in the agent configuration file, the agent will attempt to acquire a valid access token using the set authentication method outlined in the agent's configuration. 
+If this initial attempt is unsuccessful, the agent will momentarily pauses before continuing to make more attempts.
+
+Once the agent successfully obtains a valid access token, the agent proceeds to fetch the secrets from Infisical using it. 
+It then formats these secrets using the user provided templates and writes the formatted data to configured file paths.
+
+## Agent configuration file 
+
+To set up the authentication method for token renewal and to define secret templates, the Infisical agent requires a YAML configuration file containing properties defined below. 
+While specifying an authentication method is mandatory to start the agent, configuring sinks and secret templates are optional.
+
+| Field                        | Description |
+| ---------------------------- | ----------- |
+| `infisical.address`          | The URL of the Infisical service. Default: `"https://app.infisical.com"`. |
+| `auth.type`                  | The type of authentication method used. Only `"token"` type is currently available |
+| `auth.config.token-path`     | The file path where the initial token for authentication is stored.  |
+| `sinks[].type`               | The type of sink in a list of sinks. Each item specifies a sink type. Currently, only `"file"` type is available. |
+| `sinks[].config.path`        | The file path where the access token should be stored for each sink in the list. |
+| `templates[].source-path`    | The path to the template file that should be used to render secrets. |
+| `templates[].destination-path` | The path where the rendered secrets from the source template will be saved to. |
+
+
+## Quick start Infisical Agent
+To install the Infisical agent, you must first install the [Infisical CLI](../cli/overview) in the desired environment where you'd like the agent to run. This is because the Infisical agent is a sub-command of the Infisical CLI.
+
+Once you have the CLI installed, you will need to create a agent configuration file in yaml. 
+
+```yaml example-agent-config-file.yaml
+infisical:
+  address: "https://app.infisical.com"
+auth:
+  type: "token"
+  config:
+    token-path: "/path/to/initial/token"
+sinks:
+  - type: "file"
+    config:
+      path: "/some/path/to/store/access-token/file-name"
+templates:
+  - source-path: my-dot-ev-secret-template
+    destination-path: /some/path/.env
+```
+
+Above is an example agent configuration file that defines the token authentication method, one sink location (where to deposit access tokens after renewal) and a secret template. 
+
+
+```text my-dot-ev-secret-template
+{{- with secret "6553ccb2b7da580d7f6e7260" "dev" "/" }}
+{{- range . }}
+{{ .Key }}={{ .Value }}
+{{- end }}
+{{- end }}
+```
+
+The secret template above will be used to render the secrets where the key and the value are separated by `=` sign. You'll notice that a custom function named `secret` is used to fetch the secrets. 
+This function takes the following arguments: `secret "<project-id>" "<environment-slug>" "<secret-path>"`. 
+
+```bash 
+infisical agent --config example-agent-config-file.yaml
+```
+
+After defining the agent configuration file, run the command above pointing to the path where the agent configuration is located. 

docs/mint.json

@@ -197,6 +197,12 @@
         "cli/faq"
       ]
     },
+    {
+      "group": "Agent",
+      "pages": [
+        "infisical-agent/overview"
+      ]
+    },
     {
       "group": "Integrations",
       "pages": ["integrations/overview"]