From 79efe64504fbbe6b167483712b2a3c954f634afb Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Thu, 29 Feb 2024 07:11:56 +0100 Subject: [PATCH 01/15] Feat: Agent improvements, get ETag from secrets request --- cli/packages/api/api.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cli/packages/api/api.go b/cli/packages/api/api.go index 9ae356cd0..d438adb33 100644 --- a/cli/packages/api/api.go +++ b/cli/packages/api/api.go @@ -490,5 +490,7 @@ func CallGetRawSecretsV3(httpClient *resty.Client, request GetRawSecretsV3Reques return GetRawSecretsV3Response{}, fmt.Errorf("CallGetRawSecretsV3: Unsuccessful response [%v %v] [status-code=%v] [response=%v]", response.Request.Method, response.Request.URL, response.StatusCode(), response.String()) } + getRawSecretsV3Response.ETag = response.Header().Get(("etag")) + return getRawSecretsV3Response, nil } From c91456838ef4d067f23d2970e9a29857c420a8a7 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Thu, 29 Feb 2024 07:12:01 +0100 Subject: [PATCH 02/15] Update model.go --- cli/packages/api/model.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/packages/api/model.go b/cli/packages/api/model.go index 3c6466382..67ac97c25 100644 --- a/cli/packages/api/model.go +++ b/cli/packages/api/model.go @@ -505,4 +505,5 @@ type GetRawSecretsV3Response struct { SecretComment string `json:"secretComment"` } `json:"secrets"` Imports []any `json:"imports"` + ETag string } From 1ad916a784f47159ef093745e91b5fdba2cb86eb Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Thu, 29 Feb 2024 07:12:10 +0100 Subject: [PATCH 03/15] Feat: Agent improvements, Secrets state manager --- cli/packages/cmd/agent.go | 68 ++++++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/cli/packages/cmd/agent.go b/cli/packages/cmd/agent.go index 8857f5806..72e1d0d9b 100644 --- a/cli/packages/cmd/agent.go +++ b/cli/packages/cmd/agent.go @@ -73,6 +73,19 @@ type Template struct { DestinationPath string `yaml:"destination-path"` } +type SecretsStateManager struct { + // etags should be stored in memory, and the key should be env-secretPath-projectID, and the value should be the actual etag + etags map[string]string + secretMutationChannel chan bool +} + +func NewSecretsStateManager(secretMutationChannel chan bool) *SecretsStateManager { + return &SecretsStateManager{ + etags: make(map[string]string), + secretMutationChannel: secretMutationChannel, + } +} + func ReadFile(filePath string) ([]byte, error) { return ioutil.ReadFile(filePath) } @@ -170,20 +183,29 @@ func ParseAgentConfig(configFile []byte) (*Config, error) { return config, nil } -func secretTemplateFunction(accessToken string) func(string, string, string) ([]models.SingleEnvironmentVariable, error) { +func secretTemplateFunction(accessToken string, secretStateManager *SecretsStateManager) func(string, string, string) ([]models.SingleEnvironmentVariable, error) { return func(projectID, envSlug, secretPath string) ([]models.SingleEnvironmentVariable, error) { - secrets, err := util.GetPlainTextSecretsViaMachineIdentity(accessToken, projectID, envSlug, secretPath, false) + res, err := util.GetPlainTextSecretsViaMachineIdentity(accessToken, projectID, envSlug, secretPath, false) if err != nil { return nil, err } - return secrets, nil + if secretStateManager != nil { + key := fmt.Sprintf("%s-%s-%s", envSlug, secretPath, projectID) + oldEtag, ok := secretStateManager.etags[key] // if there's no etag, it means it's the first time we are fetching this secret. we should only notify the secretMutationChannel if the etag has changed, not if it's the first time we are fetching the secret + if ok && oldEtag != res.Hash { + secretStateManager.secretMutationChannel <- true + } + secretStateManager.etags[key] = res.Hash + } + + return res.Secrets, nil } } -func ProcessTemplate(templatePath string, data interface{}, accessToken string) (*bytes.Buffer, error) { +func ProcessTemplate(templatePath string, data interface{}, accessToken string, secretStateManager *SecretsStateManager) (*bytes.Buffer, error) { // custom template function to fetch secrets from Infisical - secretFunction := secretTemplateFunction(accessToken) + secretFunction := secretTemplateFunction(accessToken, secretStateManager) funcs := template.FuncMap{ "secret": secretFunction, } @@ -203,7 +225,7 @@ func ProcessTemplate(templatePath string, data interface{}, accessToken string) return &buf, nil } -func ProcessBase64Template(encodedTemplate string, data interface{}, accessToken string) (*bytes.Buffer, error) { +func ProcessBase64Template(encodedTemplate string, data interface{}, accessToken string, secretStateManager *SecretsStateManager) (*bytes.Buffer, error) { // custom template function to fetch secrets from Infisical decoded, err := base64.StdEncoding.DecodeString(encodedTemplate) if err != nil { @@ -212,7 +234,7 @@ func ProcessBase64Template(encodedTemplate string, data interface{}, accessToken templateString := string(decoded) - secretFunction := secretTemplateFunction(accessToken) + secretFunction := secretTemplateFunction(accessToken, secretStateManager) // TODO: Fix this funcs := template.FuncMap{ "secret": secretFunction, } @@ -244,13 +266,22 @@ type TokenManager struct { clientIdPath string clientSecretPath string newAccessTokenNotificationChan chan bool - removeClientSecretOnRead bool - cachedClientSecret string - exitAfterAuth bool + + removeClientSecretOnRead bool + cachedClientSecret string + exitAfterAuth bool } func NewTokenManager(fileDeposits []Sink, templates []Template, clientIdPath string, clientSecretPath string, newAccessTokenNotificationChan chan bool, removeClientSecretOnRead bool, exitAfterAuth bool) *TokenManager { - return &TokenManager{filePaths: fileDeposits, templates: templates, clientIdPath: clientIdPath, clientSecretPath: clientSecretPath, newAccessTokenNotificationChan: newAccessTokenNotificationChan, removeClientSecretOnRead: removeClientSecretOnRead, exitAfterAuth: exitAfterAuth} + return &TokenManager{ + filePaths: fileDeposits, + templates: templates, + clientIdPath: clientIdPath, + clientSecretPath: clientSecretPath, + newAccessTokenNotificationChan: newAccessTokenNotificationChan, + removeClientSecretOnRead: removeClientSecretOnRead, + exitAfterAuth: exitAfterAuth, + } } func (tm *TokenManager) SetToken(token string, accessTokenTTL time.Duration, accessTokenMaxTTL time.Duration) { @@ -428,7 +459,7 @@ func (tm *TokenManager) WriteTokenToFiles() { } } -func (tm *TokenManager) FetchSecrets() { +func (tm *TokenManager) FetchSecrets(secretStateManager *SecretsStateManager) { log.Info().Msgf("template engine started...") for { token := tm.GetToken() @@ -437,9 +468,9 @@ func (tm *TokenManager) FetchSecrets() { var processedTemplate *bytes.Buffer var err error if secretTemplate.SourcePath != "" { - processedTemplate, err = ProcessTemplate(secretTemplate.SourcePath, nil, token) + processedTemplate, err = ProcessTemplate(secretTemplate.SourcePath, nil, token, secretStateManager) } else { - processedTemplate, err = ProcessBase64Template(secretTemplate.Base64TemplateContent, nil, token) + processedTemplate, err = ProcessBase64Template(secretTemplate.Base64TemplateContent, nil, token, secretStateManager) } if err != nil { @@ -458,7 +489,7 @@ func (tm *TokenManager) FetchSecrets() { } // fetch new secrets every 5 minutes (TODO: add PubSub in the future ) - time.Sleep(5 * time.Minute) + time.Sleep(5 * time.Second) } } } @@ -537,19 +568,24 @@ var agentCmd = &cobra.Command{ configUniversalAuthType := agentConfig.Auth.Config.(UniversalAuth) tokenRefreshNotifier := make(chan bool) + secretMutationNotifier := make(chan bool) sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) filePaths := agentConfig.Sinks tm := NewTokenManager(filePaths, agentConfig.Templates, configUniversalAuthType.ClientIDPath, configUniversalAuthType.ClientSecretPath, tokenRefreshNotifier, configUniversalAuthType.RemoveClientSecretOnRead, agentConfig.Infisical.ExitAfterAuth) + ssm := NewSecretsStateManager(secretMutationNotifier) + go tm.ManageTokenLifecycle() - go tm.FetchSecrets() + go tm.FetchSecrets(ssm) for { select { case <-tokenRefreshNotifier: go tm.WriteTokenToFiles() + case <-secretMutationNotifier: + log.Info().Msgf("Mashallah, a mutation has occurred") case <-sigChan: log.Info().Msg("agent is gracefully shutting...") // TODO: check if we are in the middle of writing files to disk From 48bf41ac8cb4c613017a14acf94f7286be9ab593 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Thu, 29 Feb 2024 07:12:18 +0100 Subject: [PATCH 04/15] Update cli.go --- cli/packages/models/cli.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cli/packages/models/cli.go b/cli/packages/models/cli.go index 165982a77..9a2f2e545 100644 --- a/cli/packages/models/cli.go +++ b/cli/packages/models/cli.go @@ -20,6 +20,7 @@ type LoggedInUser struct { Domain string `json:"domain"` } + type SingleEnvironmentVariable struct { Key string `json:"key"` Value string `json:"value"` @@ -34,6 +35,11 @@ type SingleEnvironmentVariable struct { Comment string `json:"comment"` } +type PlaintextSecretResult struct { + Secrets []SingleEnvironmentVariable + Hash string +} + type SingleFolder struct { ID string `json:"_id"` Name string `json:"name"` From fb8c4bd415dcaadd719e626aafe0dde7cb1b5ffa Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Thu, 29 Feb 2024 07:12:30 +0100 Subject: [PATCH 05/15] Feat: Agent improvements --- cli/packages/util/secrets.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cli/packages/util/secrets.go b/cli/packages/util/secrets.go index 36cddbef8..fd1fe3b44 100644 --- a/cli/packages/util/secrets.go +++ b/cli/packages/util/secrets.go @@ -152,7 +152,7 @@ func GetPlainTextSecretsViaJTW(JTWToken string, receiversPrivateKey string, work return plainTextSecrets, nil } -func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool) ([]models.SingleEnvironmentVariable, error) { +func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool) (models.PlaintextSecretResult, error) { httpClient := resty.New() httpClient.SetAuthToken(accessToken). SetHeader("Accept", "application/json") @@ -170,12 +170,12 @@ func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId strin rawSecrets, err := api.CallGetRawSecretsV3(httpClient, api.GetRawSecretsV3Request{WorkspaceId: workspaceId, SecretPath: secretsPath, Environment: environmentName}) if err != nil { - return nil, err + return models.PlaintextSecretResult{}, err } plainTextSecrets := []models.SingleEnvironmentVariable{} if err != nil { - return nil, fmt.Errorf("unable to decrypt your secrets [err=%v]", err) + return models.PlaintextSecretResult{}, fmt.Errorf("unable to decrypt your secrets [err=%v]", err) } for _, secret := range rawSecrets.Secrets { @@ -189,7 +189,10 @@ func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId strin // } // } - return plainTextSecrets, nil + return models.PlaintextSecretResult{ + Secrets: plainTextSecrets, + Hash: rawSecrets.ETag, + }, nil } func InjectImportedSecret(plainTextWorkspaceKey []byte, secrets []models.SingleEnvironmentVariable, importedSecrets []api.ImportedSecretV3) ([]models.SingleEnvironmentVariable, error) { From 5096ce3bdc4bcb96ad00d4868b45d324597c4988 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 1 Mar 2024 06:41:17 +0100 Subject: [PATCH 06/15] Feat: Agent improvements --- cli/packages/cmd/agent.go | 148 +++++++++++++++++++++++-------------- cli/packages/cmd/run.go | 35 +++++++++ cli/packages/util/agent.go | 41 ++++++++++ 3 files changed, 167 insertions(+), 57 deletions(-) create mode 100644 cli/packages/util/agent.go diff --git a/cli/packages/cmd/agent.go b/cli/packages/cmd/agent.go index 72e1d0d9b..801a5dcb3 100644 --- a/cli/packages/cmd/agent.go +++ b/cli/packages/cmd/agent.go @@ -71,19 +71,14 @@ type Template struct { SourcePath string `yaml:"source-path"` Base64TemplateContent string `yaml:"base64-template-content"` DestinationPath string `yaml:"destination-path"` -} -type SecretsStateManager struct { - // etags should be stored in memory, and the key should be env-secretPath-projectID, and the value should be the actual etag - etags map[string]string - secretMutationChannel chan bool -} - -func NewSecretsStateManager(secretMutationChannel chan bool) *SecretsStateManager { - return &SecretsStateManager{ - etags: make(map[string]string), - secretMutationChannel: secretMutationChannel, - } + Config struct { // Configurations for the template + PollingInterval string `yaml:"polling-interval"` // How often to poll for changes in the secret + Exec struct { + Command string `yaml:"command"` // Command to execute once the template has been rendered + Timeout int64 `yaml:"timeout"` // Timeout for the command + } `yaml:"exec"` // Command to execute once the template has been rendered + } `yaml:"config"` } func ReadFile(filePath string) ([]byte, error) { @@ -183,29 +178,24 @@ func ParseAgentConfig(configFile []byte) (*Config, error) { return config, nil } -func secretTemplateFunction(accessToken string, secretStateManager *SecretsStateManager) func(string, string, string) ([]models.SingleEnvironmentVariable, error) { +func secretTemplateFunction(accessToken string, existingEtag string, currentEtag *string) func(string, string, string) ([]models.SingleEnvironmentVariable, error) { return func(projectID, envSlug, secretPath string) ([]models.SingleEnvironmentVariable, error) { res, err := util.GetPlainTextSecretsViaMachineIdentity(accessToken, projectID, envSlug, secretPath, false) if err != nil { return nil, err } - if secretStateManager != nil { - key := fmt.Sprintf("%s-%s-%s", envSlug, secretPath, projectID) - oldEtag, ok := secretStateManager.etags[key] // if there's no etag, it means it's the first time we are fetching this secret. we should only notify the secretMutationChannel if the etag has changed, not if it's the first time we are fetching the secret - if ok && oldEtag != res.Hash { - secretStateManager.secretMutationChannel <- true - } - secretStateManager.etags[key] = res.Hash + if existingEtag != res.Hash { + *currentEtag = res.Hash } return res.Secrets, nil } } -func ProcessTemplate(templatePath string, data interface{}, accessToken string, secretStateManager *SecretsStateManager) (*bytes.Buffer, error) { +func ProcessTemplate(templatePath string, data interface{}, accessToken string, existingEtag string, currentEtag *string) (*bytes.Buffer, error) { // custom template function to fetch secrets from Infisical - secretFunction := secretTemplateFunction(accessToken, secretStateManager) + secretFunction := secretTemplateFunction(accessToken, existingEtag, currentEtag) funcs := template.FuncMap{ "secret": secretFunction, } @@ -225,7 +215,7 @@ func ProcessTemplate(templatePath string, data interface{}, accessToken string, return &buf, nil } -func ProcessBase64Template(encodedTemplate string, data interface{}, accessToken string, secretStateManager *SecretsStateManager) (*bytes.Buffer, error) { +func ProcessBase64Template(encodedTemplate string, data interface{}, accessToken string, existingEtag string, currentEtag *string) (*bytes.Buffer, error) { // custom template function to fetch secrets from Infisical decoded, err := base64.StdEncoding.DecodeString(encodedTemplate) if err != nil { @@ -234,7 +224,7 @@ func ProcessBase64Template(encodedTemplate string, data interface{}, accessToken templateString := string(decoded) - secretFunction := secretTemplateFunction(accessToken, secretStateManager) // TODO: Fix this + secretFunction := secretTemplateFunction(accessToken, existingEtag, currentEtag) // TODO: Fix this funcs := template.FuncMap{ "secret": secretFunction, } @@ -266,13 +256,13 @@ type TokenManager struct { clientIdPath string clientSecretPath string newAccessTokenNotificationChan chan bool - - removeClientSecretOnRead bool - cachedClientSecret string - exitAfterAuth bool + removeClientSecretOnRead bool + cachedClientSecret string + exitAfterAuth bool } func NewTokenManager(fileDeposits []Sink, templates []Template, clientIdPath string, clientSecretPath string, newAccessTokenNotificationChan chan bool, removeClientSecretOnRead bool, exitAfterAuth bool) *TokenManager { + log.Info().Msgf("Token manager done, templates: %+v", templates[0]) return &TokenManager{ filePaths: fileDeposits, templates: templates, @@ -282,6 +272,7 @@ func NewTokenManager(fileDeposits []Sink, templates []Template, clientIdPath str removeClientSecretOnRead: removeClientSecretOnRead, exitAfterAuth: exitAfterAuth, } + } func (tm *TokenManager) SetToken(token string, accessTokenTTL time.Duration, accessTokenMaxTTL time.Duration) { @@ -459,38 +450,83 @@ func (tm *TokenManager) WriteTokenToFiles() { } } -func (tm *TokenManager) FetchSecrets(secretStateManager *SecretsStateManager) { +func (tm *TokenManager) WriteTemplateToFile(bytes *bytes.Buffer, template *Template) { + log.Info().Msgf("template engine started...") + + if err := WriteBytesToFile(bytes, template.DestinationPath); err != nil { + log.Error().Msgf("template engine: unable to write secrets to path because %s. Will try again on next cycle", err) + return + } + log.Info().Msgf("template engine: secret template at path %s has been rendered and saved to path %s", template.SourcePath, template.DestinationPath) +} + +func (tm *TokenManager) MonitorSecretChanges(secretTemplate Template, sigChan chan os.Signal) { + + pollingInterval := time.Duration(5 * time.Minute) + + if secretTemplate.Config.PollingInterval != "" { + interval, err := util.ConvertPollingIntervalToTime(secretTemplate.Config.PollingInterval) + + if err != nil { + log.Error().Msgf("unable to convert polling interval to time because %v", err) + sigChan <- syscall.SIGINT + return + + } else { + pollingInterval = interval + } + } + + var existingEtag string + var currentEtag string + var firstRun = true + + execTimeout := secretTemplate.Config.Exec.Timeout + execCommand := secretTemplate.Config.Exec.Command + + // Now you can use the `command` variable, which is guaranteed to be a string + for { + log.Info().Msg("polling") token := tm.GetToken() + if token != "" { - for _, secretTemplate := range tm.templates { - var processedTemplate *bytes.Buffer - var err error - if secretTemplate.SourcePath != "" { - processedTemplate, err = ProcessTemplate(secretTemplate.SourcePath, nil, token, secretStateManager) - } else { - processedTemplate, err = ProcessBase64Template(secretTemplate.Base64TemplateContent, nil, token, secretStateManager) - } - if err != nil { - log.Error().Msgf("template engine: unable to render secrets because %s. Will try again on next cycle", err) + var processedTemplate *bytes.Buffer + var err error - continue - } - - if err := WriteBytesToFile(processedTemplate, secretTemplate.DestinationPath); err != nil { - log.Error().Msgf("template engine: unable to write secrets to path because %s. Will try again on next cycle", err) - - continue - } - - log.Info().Msgf("template engine: secret template at path %s has been rendered and saved to path %s", secretTemplate.SourcePath, secretTemplate.DestinationPath) + if secretTemplate.SourcePath != "" { + processedTemplate, err = ProcessTemplate(secretTemplate.SourcePath, nil, token, existingEtag, ¤tEtag) + } else { + processedTemplate, err = ProcessBase64Template(secretTemplate.Base64TemplateContent, nil, token, existingEtag, ¤tEtag) } - // fetch new secrets every 5 minutes (TODO: add PubSub in the future ) - time.Sleep(5 * time.Second) + if err != nil { + log.Error().Msgf("unable to process template because %v", err) + } else { + if (existingEtag != currentEtag) || firstRun { + + tm.WriteTemplateToFile(processedTemplate, &secretTemplate) + existingEtag = currentEtag + + if !firstRun && execCommand != "" { + log.Info().Msgf("executing command: %s", execCommand) + err := ExecuteCommandWithTimeout(execCommand, execTimeout) + + if err != nil { + log.Error().Msgf("unable to execute command because %v", err) + } + + } + if firstRun { + firstRun = false + } + } + } } + + time.Sleep(pollingInterval) } } @@ -568,24 +604,22 @@ var agentCmd = &cobra.Command{ configUniversalAuthType := agentConfig.Auth.Config.(UniversalAuth) tokenRefreshNotifier := make(chan bool) - secretMutationNotifier := make(chan bool) sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) filePaths := agentConfig.Sinks tm := NewTokenManager(filePaths, agentConfig.Templates, configUniversalAuthType.ClientIDPath, configUniversalAuthType.ClientSecretPath, tokenRefreshNotifier, configUniversalAuthType.RemoveClientSecretOnRead, agentConfig.Infisical.ExitAfterAuth) - ssm := NewSecretsStateManager(secretMutationNotifier) - go tm.ManageTokenLifecycle() - go tm.FetchSecrets(ssm) + + for _, template := range agentConfig.Templates { + go tm.MonitorSecretChanges(template, sigChan) + } for { select { case <-tokenRefreshNotifier: go tm.WriteTokenToFiles() - case <-secretMutationNotifier: - log.Info().Msgf("Mashallah, a mutation has occurred") case <-sigChan: log.Info().Msg("agent is gracefully shutting...") // TODO: check if we are in the middle of writing files to disk diff --git a/cli/packages/cmd/run.go b/cli/packages/cmd/run.go index 2bb043c26..b4e22acb3 100644 --- a/cli/packages/cmd/run.go +++ b/cli/packages/cmd/run.go @@ -4,6 +4,7 @@ Copyright (c) 2023 Infisical Inc. package cmd import ( + "context" "fmt" "os" "os/exec" @@ -11,6 +12,7 @@ import ( "runtime" "strings" "syscall" + "time" "github.com/Infisical/infisical-merge/packages/models" "github.com/Infisical/infisical-merge/packages/util" @@ -270,3 +272,36 @@ func execCmd(cmd *exec.Cmd) error { os.Exit(waitStatus.ExitStatus()) return nil } + +func ExecuteCommandWithTimeout(command string, timeout int64) error { + + shell := [2]string{"sh", "-c"} + if runtime.GOOS == "windows" { + shell = [2]string{"cmd", "/C"} + } else { + currentShell := os.Getenv("SHELL") + if currentShell != "" { + shell[0] = currentShell + } + } + + ctx := context.Background() + if timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) + defer cancel() + } + + cmd := exec.CommandContext(ctx, shell[0], shell[1], command) + + if err := cmd.Run(); err != nil { + if exitError, ok := err.(*exec.ExitError); ok { // type assertion + if exitError.ProcessState.ExitCode() == -1 { + return fmt.Errorf("command timed out") + } + } + return err + } else { + return nil + } +} diff --git a/cli/packages/util/agent.go b/cli/packages/util/agent.go new file mode 100644 index 000000000..188ae5de2 --- /dev/null +++ b/cli/packages/util/agent.go @@ -0,0 +1,41 @@ +package util + +import ( + "fmt" + "strconv" + "time" +) + +// ConvertPollingIntervalToTime converts a string representation of a polling interval to a time.Duration +func ConvertPollingIntervalToTime(pollingInterval string) (time.Duration, error) { + length := len(pollingInterval) + if length < 2 { + return 0, fmt.Errorf("invalid format") + } + + unit := pollingInterval[length-1:] + numberPart := pollingInterval[:length-1] + + number, err := strconv.Atoi(numberPart) + if err != nil { + return 0, err + } + + switch unit { + case "s": + if number < 60 { + return 0, fmt.Errorf("polling interval should be at least 60 seconds") + } + return time.Duration(number) * time.Second, nil + case "m": + return time.Duration(number) * time.Minute, nil + case "h": + return time.Duration(number) * time.Hour, nil + case "d": + return time.Duration(number) * 24 * time.Hour, nil + case "w": + return time.Duration(number) * 7 * 24 * time.Hour, nil + default: + return 0, fmt.Errorf("invalid time unit") + } +} From 1f3f061a06a5f3ac664a699bfff71a8199066781 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 1 Mar 2024 06:46:09 +0100 Subject: [PATCH 07/15] Fix: Agent output --- cli/packages/cmd/run.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cli/packages/cmd/run.go b/cli/packages/cmd/run.go index b4e22acb3..ef8e1c2c7 100644 --- a/cli/packages/cmd/run.go +++ b/cli/packages/cmd/run.go @@ -293,6 +293,9 @@ func ExecuteCommandWithTimeout(command string, timeout int64) error { } cmd := exec.CommandContext(ctx, shell[0], shell[1], command) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { if exitError, ok := err.(*exec.ExitError); ok { // type assertion From e95265941599cbe72b79159749fa5ed67dbf9413 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 1 Mar 2024 07:02:04 +0100 Subject: [PATCH 08/15] Update agent.go --- cli/packages/cmd/agent.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/packages/cmd/agent.go b/cli/packages/cmd/agent.go index 801a5dcb3..62632f3c6 100644 --- a/cli/packages/cmd/agent.go +++ b/cli/packages/cmd/agent.go @@ -488,7 +488,6 @@ func (tm *TokenManager) MonitorSecretChanges(secretTemplate Template, sigChan ch // Now you can use the `command` variable, which is guaranteed to be a string for { - log.Info().Msg("polling") token := tm.GetToken() if token != "" { From 97ac8cb45aaf461e79d6ae7779b0bdc1795f2e56 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 1 Mar 2024 07:02:26 +0100 Subject: [PATCH 09/15] Update agent.go --- cli/packages/cmd/agent.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cli/packages/cmd/agent.go b/cli/packages/cmd/agent.go index 62632f3c6..52bf932f1 100644 --- a/cli/packages/cmd/agent.go +++ b/cli/packages/cmd/agent.go @@ -485,8 +485,6 @@ func (tm *TokenManager) MonitorSecretChanges(secretTemplate Template, sigChan ch execTimeout := secretTemplate.Config.Exec.Timeout execCommand := secretTemplate.Config.Exec.Command - // Now you can use the `command` variable, which is guaranteed to be a string - for { token := tm.GetToken() From 9fcdf17a04a1c871cf0bcb4deb9e7f38ce3d1ddd Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 1 Mar 2024 07:17:27 +0100 Subject: [PATCH 10/15] Update agent.go --- cli/packages/cmd/agent.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cli/packages/cmd/agent.go b/cli/packages/cmd/agent.go index 52bf932f1..14acfa6ab 100644 --- a/cli/packages/cmd/agent.go +++ b/cli/packages/cmd/agent.go @@ -521,9 +521,12 @@ func (tm *TokenManager) MonitorSecretChanges(secretTemplate Template, sigChan ch } } } + time.Sleep(pollingInterval) + } else { + // It fails to get the access token. So we will re-try in 3 seconds. We do this because if we don't, the user will have to wait for the next polling interval to get the first secret render. + time.Sleep(3 * time.Second) } - time.Sleep(pollingInterval) } } From 4a2a5f42a89a0adca63babe47b5fd17f0b3cab70 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 1 Mar 2024 07:26:31 +0100 Subject: [PATCH 11/15] =?UTF-8?q?Renamed=20to=20exec=20to=20execute,=20and?= =?UTF-8?q?=20cleanup=20=F0=9F=A7=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cli/packages/cmd/agent.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/cli/packages/cmd/agent.go b/cli/packages/cmd/agent.go index 14acfa6ab..34c70e381 100644 --- a/cli/packages/cmd/agent.go +++ b/cli/packages/cmd/agent.go @@ -74,10 +74,10 @@ type Template struct { Config struct { // Configurations for the template PollingInterval string `yaml:"polling-interval"` // How often to poll for changes in the secret - Exec struct { + Execute struct { Command string `yaml:"command"` // Command to execute once the template has been rendered Timeout int64 `yaml:"timeout"` // Timeout for the command - } `yaml:"exec"` // Command to execute once the template has been rendered + } `yaml:"execute"` // Command to execute once the template has been rendered } `yaml:"config"` } @@ -262,7 +262,6 @@ type TokenManager struct { } func NewTokenManager(fileDeposits []Sink, templates []Template, clientIdPath string, clientSecretPath string, newAccessTokenNotificationChan chan bool, removeClientSecretOnRead bool, exitAfterAuth bool) *TokenManager { - log.Info().Msgf("Token manager done, templates: %+v", templates[0]) return &TokenManager{ filePaths: fileDeposits, templates: templates, @@ -451,9 +450,6 @@ func (tm *TokenManager) WriteTokenToFiles() { } func (tm *TokenManager) WriteTemplateToFile(bytes *bytes.Buffer, template *Template) { - - log.Info().Msgf("template engine started...") - if err := WriteBytesToFile(bytes, template.DestinationPath); err != nil { log.Error().Msgf("template engine: unable to write secrets to path because %s. Will try again on next cycle", err) return @@ -482,8 +478,8 @@ func (tm *TokenManager) MonitorSecretChanges(secretTemplate Template, sigChan ch var currentEtag string var firstRun = true - execTimeout := secretTemplate.Config.Exec.Timeout - execCommand := secretTemplate.Config.Exec.Command + execTimeout := secretTemplate.Config.Execute.Timeout + execCommand := secretTemplate.Config.Execute.Command for { token := tm.GetToken() @@ -612,7 +608,8 @@ var agentCmd = &cobra.Command{ go tm.ManageTokenLifecycle() - for _, template := range agentConfig.Templates { + for i, template := range agentConfig.Templates { + log.Info().Msgf("template engine started for template %v...", i+1) go tm.MonitorSecretChanges(template, sigChan) } From 041c4a20a0a2298e8523afa8dff853dfe2416ea7 Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Fri, 1 Mar 2024 02:10:26 -0500 Subject: [PATCH 12/15] example config --- cli/agent-config.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cli/agent-config.yaml b/cli/agent-config.yaml index ae130d3a8..e767fdca5 100644 --- a/cli/agent-config.yaml +++ b/cli/agent-config.yaml @@ -1,5 +1,5 @@ infisical: - address: "http://localhost:8080" + address: "https://app.infisical.com/" auth: type: "universal-auth" config: @@ -13,3 +13,12 @@ sinks: templates: - source-path: my-dot-ev-secret-template destination-path: my-dot-env.env + config: + polling-interval: 60s + execute: + command: docker-compose -f docker-compose.prod.yml down && docker-compose -f docker-compose.prod.yml up -d + - source-path: my-dot-ev-secret-template1 + destination-path: my-dot-env-1.env + config: + exec: + command: mkdir hello-world1 From 8dd94a4e10bd58b7c466136e2b376dc053e5ad9f Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Fri, 1 Mar 2024 02:11:03 -0500 Subject: [PATCH 13/15] move ExecuteCommandWithTimeout to agent file --- cli/packages/cmd/agent.go | 43 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/cli/packages/cmd/agent.go b/cli/packages/cmd/agent.go index 34c70e381..750727df1 100644 --- a/cli/packages/cmd/agent.go +++ b/cli/packages/cmd/agent.go @@ -5,12 +5,15 @@ package cmd import ( "bytes" + "context" "encoding/base64" "fmt" "io/ioutil" "os" + "os/exec" "os/signal" "path" + "runtime" "strings" "sync" "syscall" @@ -85,6 +88,42 @@ func ReadFile(filePath string) ([]byte, error) { return ioutil.ReadFile(filePath) } +func ExecuteCommandWithTimeout(command string, timeout int64) error { + + shell := [2]string{"sh", "-c"} + if runtime.GOOS == "windows" { + shell = [2]string{"cmd", "/C"} + } else { + currentShell := os.Getenv("SHELL") + if currentShell != "" { + shell[0] = currentShell + } + } + + ctx := context.Background() + if timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) + defer cancel() + } + + cmd := exec.CommandContext(ctx, shell[0], shell[1], command) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + if exitError, ok := err.(*exec.ExitError); ok { // type assertion + if exitError.ProcessState.ExitCode() == -1 { + return fmt.Errorf("command timed out") + } + } + return err + } else { + return nil + } +} + func FileExists(filepath string) bool { info, err := os.Stat(filepath) if os.IsNotExist(err) { @@ -185,8 +224,8 @@ func secretTemplateFunction(accessToken string, existingEtag string, currentEtag return nil, err } - if existingEtag != res.Hash { - *currentEtag = res.Hash + if existingEtag != res.Etag { + *currentEtag = res.Etag } return res.Secrets, nil From 5c40b538af0e5ce158ecfe75009b0dad98a17031 Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Fri, 1 Mar 2024 02:11:27 -0500 Subject: [PATCH 14/15] remove ExecuteCommandWithTimeout --- cli/packages/cmd/run.go | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/cli/packages/cmd/run.go b/cli/packages/cmd/run.go index ef8e1c2c7..2bb043c26 100644 --- a/cli/packages/cmd/run.go +++ b/cli/packages/cmd/run.go @@ -4,7 +4,6 @@ Copyright (c) 2023 Infisical Inc. package cmd import ( - "context" "fmt" "os" "os/exec" @@ -12,7 +11,6 @@ import ( "runtime" "strings" "syscall" - "time" "github.com/Infisical/infisical-merge/packages/models" "github.com/Infisical/infisical-merge/packages/util" @@ -272,39 +270,3 @@ func execCmd(cmd *exec.Cmd) error { os.Exit(waitStatus.ExitStatus()) return nil } - -func ExecuteCommandWithTimeout(command string, timeout int64) error { - - shell := [2]string{"sh", "-c"} - if runtime.GOOS == "windows" { - shell = [2]string{"cmd", "/C"} - } else { - currentShell := os.Getenv("SHELL") - if currentShell != "" { - shell[0] = currentShell - } - } - - ctx := context.Background() - if timeout > 0 { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) - defer cancel() - } - - cmd := exec.CommandContext(ctx, shell[0], shell[1], command) - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - if exitError, ok := err.(*exec.ExitError); ok { // type assertion - if exitError.ProcessState.ExitCode() == -1 { - return fmt.Errorf("command timed out") - } - } - return err - } else { - return nil - } -} From a0ea2627ed516ccf28cc66cb3266e4fca46fd32d Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Fri, 1 Mar 2024 02:11:50 -0500 Subject: [PATCH 15/15] change hash to etag --- cli/packages/models/cli.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cli/packages/models/cli.go b/cli/packages/models/cli.go index 9a2f2e545..71033fe72 100644 --- a/cli/packages/models/cli.go +++ b/cli/packages/models/cli.go @@ -20,7 +20,6 @@ type LoggedInUser struct { Domain string `json:"domain"` } - type SingleEnvironmentVariable struct { Key string `json:"key"` Value string `json:"value"` @@ -37,7 +36,7 @@ type SingleEnvironmentVariable struct { type PlaintextSecretResult struct { Secrets []SingleEnvironmentVariable - Hash string + Etag string } type SingleFolder struct {