diff --git a/cli/packages/cmd/run.go b/cli/packages/cmd/run.go index 22a4ca65b..fa5176d89 100644 --- a/cli/packages/cmd/run.go +++ b/cli/packages/cmd/run.go @@ -4,22 +4,27 @@ Copyright (c) 2023 Infisical Inc. package cmd import ( + "errors" "fmt" "os" "os/exec" "os/signal" "runtime" "strings" + "sync" "syscall" + "time" "github.com/Infisical/infisical-merge/packages/models" "github.com/Infisical/infisical-merge/packages/util" "github.com/fatih/color" - "github.com/posthog/posthog-go" "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) +var ErrManualSignalInterrupt = errors.New("signal: interrupt") +var watcherWaitGroup = new(sync.WaitGroup) + // runCmd represents the run command var runCmd = &cobra.Command{ Example: ` @@ -77,11 +82,35 @@ var runCmd = &cobra.Command{ util.HandleError(err, "Unable to parse flag") } + command, err := cmd.Flags().GetString("command") + if err != nil { + util.HandleError(err, "Unable to parse flag") + } + + if err != nil { + util.HandleError(err, "Unable to parse flag") + } + secretOverriding, err := cmd.Flags().GetBool("secret-overriding") if err != nil { util.HandleError(err, "Unable to parse flag") } + watchMode, err := cmd.Flags().GetBool("watch") + if err != nil { + util.HandleError(err, "Unable to parse flag") + } + + watchModeInterval, err := cmd.Flags().GetInt("watch-interval") + if err != nil { + util.HandleError(err, "Unable to parse flag") + } + + // If the --watch flag has been set, the --watch-interval flag should also be set + if watchMode && watchModeInterval < 5 { + util.HandleError(fmt.Errorf("watch interval must be at least 5 seconds, you passed %d seconds", watchModeInterval)) + } + shouldExpandSecrets, err := cmd.Flags().GetBool("expand") if err != nil { util.HandleError(err, "Unable to parse flag") @@ -116,108 +145,50 @@ var runCmd = &cobra.Command{ Recursive: recursive, } - if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { - request.InfisicalToken = token.Token - } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { - request.UniversalAuthAccessToken = token.Token - } - - secrets, err := util.GetAllEnvironmentVariables(request, projectConfigDir) - + injectableEnvironment, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, shouldExpandSecrets, token) if err != nil { util.HandleError(err, "Could not fetch secrets", "If you are using a service token to fetch secrets, please ensure it is valid") } - if secretOverriding { - secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_PERSONAL) + log.Debug().Msgf("injecting the following environment variables into shell: %v", injectableEnvironment.Variables) + + if watchMode { + executeCommandWithWatchMode(command, args, watchModeInterval, request, projectConfigDir, shouldExpandSecrets, secretOverriding, token) } else { - secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED) - } + if cmd.Flags().Changed("command") { + command := cmd.Flag("command").Value.String() + err = executeMultipleCommandWithEnvs(command, injectableEnvironment.SecretsCount, injectableEnvironment.Variables) + if err != nil { + fmt.Println(err) + os.Exit(1) + } - if shouldExpandSecrets { - - authParams := models.ExpandSecretsAuthentication{} - - if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { - authParams.InfisicalToken = token.Token - } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { - authParams.UniversalAuthAccessToken = token.Token - } - - secrets = util.ExpandSecrets(secrets, authParams, projectConfigDir) - } - - secretsByKey := getSecretsByKeys(secrets) - environmentVariables := make(map[string]string) - - // add all existing environment vars - for _, s := range os.Environ() { - kv := strings.SplitN(s, "=", 2) - key := kv[0] - value := kv[1] - environmentVariables[key] = value - } - - // check to see if there are any reserved key words in secrets to inject - filterReservedEnvVars(secretsByKey) - - // now add infisical secrets - for k, v := range secretsByKey { - environmentVariables[k] = v.Value - } - - // turn it back into a list of envs - var env []string - for key, value := range environmentVariables { - s := key + "=" + value - env = append(env, s) - } - - log.Debug().Msgf("injecting the following environment variables into shell: %v", env) - - Telemetry.CaptureEvent("cli-command:run", - posthog.NewProperties(). - Set("secretsCount", len(secrets)). - Set("environment", environmentName). - Set("isUsingServiceToken", token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER). - Set("isUsingUniversalAuthToken", token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER). - Set("single-command", strings.Join(args, " ")). - Set("multi-command", cmd.Flag("command").Value.String()). - Set("version", util.CLI_VERSION)) - - if cmd.Flags().Changed("command") { - command := cmd.Flag("command").Value.String() - - err = executeMultipleCommandWithEnvs(command, len(secretsByKey), env) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - } else { - err = executeSingleCommandWithEnvs(args, len(secretsByKey), env) - if err != nil { - fmt.Println(err) - os.Exit(1) + } else { + err = executeSingleCommandWithEnvs(args, injectableEnvironment.SecretsCount, injectableEnvironment.Variables) + if err != nil { + fmt.Println(err) + os.Exit(1) + } } } + }, } -var ( - reservedEnvVars = []string{ - "HOME", "PATH", "PS1", "PS2", - "PWD", "EDITOR", "XAUTHORITY", "USER", - "TERM", "TERMINFO", "SHELL", "MAIL", - } - - reservedEnvVarPrefixes = []string{ - "XDG_", - "LC_", - } -) - func filterReservedEnvVars(env map[string]models.SingleEnvironmentVariable) { + var ( + reservedEnvVars = []string{ + "HOME", "PATH", "PS1", "PS2", + "PWD", "EDITOR", "XAUTHORITY", "USER", + "TERM", "TERMINFO", "SHELL", "MAIL", + } + + reservedEnvVarPrefixes = []string{ + "XDG_", + "LC_", + } + ) + for _, reservedEnvName := range reservedEnvVars { if _, ok := env[reservedEnvName]; ok { delete(env, reservedEnvName) @@ -237,13 +208,15 @@ func filterReservedEnvVars(env map[string]models.SingleEnvironmentVariable) { func init() { rootCmd.AddCommand(runCmd) - runCmd.Flags().String("token", "", "Fetch secrets using service token or machine identity access token") + runCmd.Flags().String("token", "", "fetch secrets using service token or machine identity access token") runCmd.Flags().String("projectId", "", "manually set the project ID to fetch secrets from when using machine identity based auth") - runCmd.Flags().StringP("env", "e", "dev", "Set the environment (dev, prod, etc.) from which your secrets should be pulled from") - runCmd.Flags().Bool("expand", true, "Parse shell parameter expansions in your secrets") - runCmd.Flags().Bool("include-imports", true, "Import linked secrets ") - runCmd.Flags().Bool("recursive", false, "Fetch secrets from all sub-folders") - runCmd.Flags().Bool("secret-overriding", true, "Prioritizes personal secrets, if any, with the same name over shared secrets") + runCmd.Flags().StringP("env", "e", "dev", "set the environment (dev, prod, etc.) from which your secrets should be pulled from") + runCmd.Flags().Bool("expand", true, "parse shell parameter expansions in your secrets") + runCmd.Flags().Bool("include-imports", true, "import linked secrets ") + runCmd.Flags().Bool("recursive", false, "fetch secrets from all sub-folders") + runCmd.Flags().Bool("secret-overriding", true, "prioritizes personal secrets, if any, with the same name over shared secrets") + runCmd.Flags().Bool("watch", false, "enable reload of application when secrets change") + runCmd.Flags().Int("watch-interval", 10, "interval in seconds to check for secret changes") runCmd.Flags().StringP("command", "c", "", "chained commands to execute (e.g. \"npm install && npm run dev; echo ...\")") runCmd.Flags().StringP("tags", "t", "", "filter secrets by tag slugs ") runCmd.Flags().String("path", "/", "get secrets within a folder path") @@ -263,7 +236,7 @@ func executeSingleCommandWithEnvs(args []string, secretsCount int, env []string) cmd.Stderr = os.Stderr cmd.Env = env - return execCmd(cmd) + return execBasicCmd(cmd) } func executeMultipleCommandWithEnvs(fullCommand string, secretsCount int, env []string) error { @@ -286,11 +259,10 @@ func executeMultipleCommandWithEnvs(fullCommand string, secretsCount int, env [] log.Info().Msgf(color.GreenString("Injecting %v Infisical secrets into your application process", secretsCount)) log.Debug().Msgf("executing command: %s %s %s \n", shell[0], shell[1], fullCommand) - return execCmd(cmd) + return execBasicCmd(cmd) } -// Credit: inspired by AWS Valut -func execCmd(cmd *exec.Cmd) error { +func execBasicCmd(cmd *exec.Cmd) error { sigChannel := make(chan os.Signal, 1) signal.Notify(sigChannel) @@ -314,3 +286,217 @@ func execCmd(cmd *exec.Cmd) error { os.Exit(waitStatus.ExitStatus()) return nil } + +func waitForExitCommand(cmd *exec.Cmd) (int, error) { + if err := cmd.Wait(); err != nil { + // ignore errors + cmd.Process.Signal(os.Kill) // #nosec G104 + + if exitError, ok := err.(*exec.ExitError); ok { + return exitError.ExitCode(), exitError + } + + return 2, err + } + + waitStatus, ok := cmd.ProcessState.Sys().(syscall.WaitStatus) + if !ok { + return 2, fmt.Errorf("unexpected ProcessState type, expected syscall.WaitStatus, got %T", waitStatus) + } + return waitStatus.ExitStatus(), nil +} + +func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInterval int, request models.GetAllSecretsParameters, projectConfigDir string, expandSecrets bool, secretOverriding bool, token *models.TokenDetails) { + + var cmd *exec.Cmd + var err error + var lastSecretsFetch time.Time + var lastUpdateEvent time.Time + var watchMutex sync.Mutex + var processMutex sync.Mutex + var beingTerminated = false + var currentETag string + + if err != nil { + util.HandleError(err, "Failed to fetch secrets") + } + + runCommandWithWatcher := func(environmentVariables models.InjectableEnvironmentResult) { + currentETag = environmentVariables.ETag + secretsFetchedAt := time.Now() + if secretsFetchedAt.After(lastSecretsFetch) { + lastSecretsFetch = secretsFetchedAt + } + + shouldRestartProcess := cmd != nil + // terminate the old process before starting a new one + if shouldRestartProcess { + log.Info().Msg(color.HiMagentaString("[HOT RELOAD] Environment changes detected. Reloading process...")) + beingTerminated = true + + log.Debug().Msgf(color.HiMagentaString("[HOT RELOAD] Sending SIGTERM to PID %d", cmd.Process.Pid)) + if e := cmd.Process.Signal(syscall.SIGTERM); e != nil { + log.Error().Err(e).Msg(color.HiMagentaString("[HOT RELOAD] Failed to send SIGTERM")) + } + // wait up to 10 sec for the process to exit + for i := 0; i < 10; i++ { + if !util.IsProcessRunning(cmd.Process) { + // process has been killed so we break out + break + } + if i == 5 { + log.Debug().Msg(color.HiMagentaString("[HOT RELOAD] Still waiting for process exit status")) + } + time.Sleep(time.Second) + } + + // SIGTERM may not work on Windows so we try SIGKILL + if util.IsProcessRunning(cmd.Process) { + log.Debug().Msg(color.HiMagentaString("[HOT RELOAD] Process still hasn't fully exited, attempting SIGKILL")) + if e := cmd.Process.Kill(); e != nil { + log.Error().Err(e).Msg(color.HiMagentaString("[HOT RELOAD] Failed to send SIGKILL")) + } + } + + cmd = nil + } else { + // If `cmd` is nil, we know this is the first time we are starting the process + log.Info().Msg(color.HiMagentaString("[HOT RELOAD] Watching for secret changes...")) + } + + processMutex.Lock() + + if lastUpdateEvent.After(secretsFetchedAt) { + processMutex.Unlock() + return + } + + beingTerminated = false + watcherWaitGroup.Add(1) + + // start the process + log.Info().Msgf(color.GreenString("Injecting %v Infisical secrets into your application process", environmentVariables.SecretsCount)) + + cmd, err = util.RunCommand(commandFlag, args, environmentVariables.Variables, false) + if err != nil { + defer watcherWaitGroup.Done() + util.HandleError(err) + } + + go func() { + defer processMutex.Unlock() + defer watcherWaitGroup.Done() + + exitCode, err := waitForExitCommand(cmd) + + // ignore errors if we are being terminated + if !beingTerminated { + if err != nil { + if strings.HasPrefix(err.Error(), "exec") || strings.HasPrefix(err.Error(), "fork/exec") { + log.Error().Err(err).Msg("Failed to execute command") + } + if err.Error() != ErrManualSignalInterrupt.Error() { + log.Error().Err(err).Msg("Process exited with error") + } + } + + os.Exit(exitCode) + } + }() + } + + recheckSecretsChannel := make(chan bool, 1) + recheckSecretsChannel <- true + + // a simple goroutine that triggers the recheckSecretsChan every watch interval (defaults to 10 seconds) + go func() { + for { + time.Sleep(time.Duration(watchModeInterval) * time.Second) + recheckSecretsChannel <- true + } + }() + + for { + <-recheckSecretsChannel + watchMutex.Lock() + + newEnvironmentVariables, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, expandSecrets, token) + if err != nil { + log.Error().Err(err).Msg("[HOT RELOAD] Failed to fetch secrets") + continue + } + + if newEnvironmentVariables.ETag != currentETag { + runCommandWithWatcher(newEnvironmentVariables) + } else { + log.Debug().Msg("[HOT RELOAD] No changes detected in secrets, not reloading process") + } + + watchMutex.Unlock() + + } +} + +func fetchAndFormatSecretsForShell(request models.GetAllSecretsParameters, projectConfigDir string, secretOverriding bool, shouldExpandSecrets bool, token *models.TokenDetails) (models.InjectableEnvironmentResult, error) { + + if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { + request.InfisicalToken = token.Token + } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { + request.UniversalAuthAccessToken = token.Token + } + + secrets, err := util.GetAllEnvironmentVariables(request, projectConfigDir) + + if err != nil { + return models.InjectableEnvironmentResult{}, err + } + + if secretOverriding { + secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_PERSONAL) + } else { + secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED) + } + + if shouldExpandSecrets { + + authParams := models.ExpandSecretsAuthentication{} + + if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { + authParams.InfisicalToken = token.Token + } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { + authParams.UniversalAuthAccessToken = token.Token + } + + secrets = util.ExpandSecrets(secrets, authParams, projectConfigDir) + } + + secretsByKey := getSecretsByKeys(secrets) + environmentVariables := make(map[string]string) + + // add all existing environment vars + for _, s := range os.Environ() { + kv := strings.SplitN(s, "=", 2) + key := kv[0] + value := kv[1] + environmentVariables[key] = value + } + + // check to see if there are any reserved key words in secrets to inject + filterReservedEnvVars(secretsByKey) + + // now add infisical secrets + for k, v := range secretsByKey { + environmentVariables[k] = v.Value + } + + env := make([]string, 0, len(environmentVariables)) + for key, value := range environmentVariables { + env = append(env, key+"="+value) + } + + return models.InjectableEnvironmentResult{ + Variables: env, + ETag: util.GenerateETagFromSecrets(secrets), + SecretsCount: len(secretsByKey), + }, nil +} diff --git a/cli/packages/models/cli.go b/cli/packages/models/cli.go index c4bbd0175..1bad5e327 100644 --- a/cli/packages/models/cli.go +++ b/cli/packages/models/cli.go @@ -104,6 +104,12 @@ type GetAllSecretsParameters struct { Recursive bool } +type InjectableEnvironmentResult struct { + Variables []string + ETag string + SecretsCount int +} + type GetAllFoldersParameters struct { WorkspaceId string Environment string diff --git a/cli/packages/util/exec.go b/cli/packages/util/exec.go new file mode 100644 index 000000000..2cdb50f42 --- /dev/null +++ b/cli/packages/util/exec.go @@ -0,0 +1,92 @@ +package util + +import ( + "fmt" + "os" + "os/exec" + "os/signal" + "runtime" + "syscall" +) + +func RunCommand(singleCommand string, args []string, env []string, waitForExit bool) (*exec.Cmd, error) { + var c *exec.Cmd + var err error + + if singleCommand != "" { + c, err = RunCommandFromString(singleCommand, env, waitForExit) + } else { + c, err = RunCommandFromArgs(args, env, waitForExit) + } + + return c, err +} + +func IsProcessRunning(p *os.Process) bool { + err := p.Signal(syscall.Signal(0)) + return err == nil +} + +// For "infisical run -- COMMAND" +func RunCommandFromArgs(args []string, env []string, waitForExit bool) (*exec.Cmd, error) { + cmd := exec.Command(args[0], args[1:]...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = env + + err := execCommand(cmd, waitForExit) + + return cmd, err +} + +func execCommand(cmd *exec.Cmd, waitForExit bool) error { + sigChannel := make(chan os.Signal, 1) + signal.Notify(sigChannel) + + if err := cmd.Start(); err != nil { + return err + } + + go func() { + for { + sig := <-sigChannel + _ = cmd.Process.Signal(sig) // process all sigs + } + }() + + if !waitForExit { + return nil + } + + if err := cmd.Wait(); err != nil { + _ = cmd.Process.Signal(os.Kill) + return fmt.Errorf("failed to wait for command termination: %v", err) + } + + waitStatus := cmd.ProcessState.Sys().(syscall.WaitStatus) + os.Exit(waitStatus.ExitStatus()) + return nil +} + +// For "infisical run --command=COMMAND" +func RunCommandFromString(command string, env []string, waitForExit bool) (*exec.Cmd, error) { + shell := [2]string{"sh", "-c"} + if runtime.GOOS == "windows" { + shell = [2]string{"cmd", "/C"} + } else { + currentShell := os.Getenv("SHELL") + if currentShell != "" { + shell[0] = currentShell + } + } + + cmd := exec.Command(shell[0], shell[1], command) // #nosec G204 nosemgrep: semgrep_configs.prohibit-exec-command + cmd.Env = env + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := execCommand(cmd, waitForExit) + return cmd, err +} diff --git a/cli/packages/util/helper.go b/cli/packages/util/helper.go index 69a310efa..b758ebd9d 100644 --- a/cli/packages/util/helper.go +++ b/cli/packages/util/helper.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/sha256" "encoding/base64" + "encoding/hex" "fmt" "math/rand" "os" @@ -298,3 +299,16 @@ func GenerateRandomString(length int) string { } return string(b) } + +func GenerateETagFromSecrets(secrets []models.SingleEnvironmentVariable) string { + sortedSecrets := SortSecretsByKeys(secrets) + content := []byte{} + + for _, secret := range sortedSecrets { + content = append(content, []byte(secret.Key)...) + content = append(content, []byte(secret.Value)...) + } + + hash := sha256.Sum256(content) + return fmt.Sprintf(`"%s"`, hex.EncodeToString(hash[:])) +} diff --git a/docs/cli/commands/run.mdx b/docs/cli/commands/run.mdx index 74aa84947..bfa4bd693 100644 --- a/docs/cli/commands/run.mdx +++ b/docs/cli/commands/run.mdx @@ -47,20 +47,20 @@ $ infisical run -- npm run dev Used to fetch secrets via a [machine identity](/documentation/platform/identities/machine-identities) apposed to logged in credentials. Simply, export this variable in the terminal before running this command. ```bash - # Example - export INFISICAL_TOKEN=$(infisical login --method=universal-auth --client-id= --client-secret= --silent --plain) # --plain flag will output only the token, so it can be fed to an environment variable. --silent will disable any update messages. + # Example + export INFISICAL_TOKEN=$(infisical login --method=universal-auth --client-id= --client-secret= --silent --plain) # --plain flag will output only the token, so it can be fed to an environment variable. --silent will disable any update messages. ``` Alternatively, you may use service tokens. Please note, however, that service tokens are being deprecated in favor of [machine identities](/documentation/platform/identities/machine-identities). They will be removed in the future in accordance with the deprecation notice and timeline stated [here](https://infisical.com/blog/deprecating-api-keys). + ```bash - # Example - export INFISICAL_TOKEN= + # Example + export INFISICAL_TOKEN= ``` - - + @@ -69,22 +69,30 @@ $ infisical run -- npm run dev To use, simply export this variable in the terminal before running this command. ```bash - # Example - export INFISICAL_DISABLE_UPDATE_CHECK=true + # Example + export INFISICAL_DISABLE_UPDATE_CHECK=true ``` - ### Flags - + + By passing the `watch` flag, you are telling the CLI to watch for changes that happen in your Infisical project. + If secret changes happen, the command you provided will automatically be restarted with the new environment variables attached. + + ```bash + # Example + infisical run --watch -- printenv + ``` + + + Explicitly set the directory where the .infisical.json resides. This is useful for some monorepo setups. ```bash - # Example - infisical run --project-config-dir=/some-dir -- printenv + # Example + infisical run --project-config-dir=/some-dir -- printenv ``` - @@ -172,3 +180,19 @@ $ infisical run -- npm run dev + + +## Automatically reload command when secrets change + +To automatically reload your command when secrets change, use the `--watch` flag. + +```bash +infisical run --watch -- npm run dev +``` + +This will watch for changes in your secrets and automatically restart your command with the new secrets. +When your command restarts, it will have the new environment variables injeceted into it. + + + Please note that this feature is intended for development purposes. It is not recommended to use this in production environments. Generally it's not recommended to automatically reload your application in production when remote changes are made. + \ No newline at end of file