awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2353 from Infisical/daniel/cli-run-watch-mode

Browse Source 
feat(cli): `run` command watch mode
This commit is contained in:
Maidul Islam
2024-09-10 10:39:06 -04:00
committed by GitHub
parent 1bc1feb843 5e0b78b104
commit a549c8b9e3
5 changed files with 436 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

388
cli/packages/cmd/run.go
View File

@@ -4,22 +4,27 @@ Copyright (c) 2023 Infisical Inc.
package cmd
import (
"errors"
"fmt"
"os"
"os/exec"
"os/signal"
"runtime"
"strings"
"sync"
"syscall"
"time"
"github.com/Infisical/infisical-merge/packages/models"
"github.com/Infisical/infisical-merge/packages/util"
"github.com/fatih/color"
"github.com/posthog/posthog-go"
"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
)
var ErrManualSignalInterrupt = errors.New("signal: interrupt")
var watcherWaitGroup = new(sync.WaitGroup)
// runCmd represents the run command
var runCmd = &cobra.Command{
Example: `
@@ -77,11 +82,35 @@ var runCmd = &cobra.Command{
util.HandleError(err, "Unable to parse flag")
}
command, err := cmd.Flags().GetString("command")
if err != nil {
util.HandleError(err, "Unable to parse flag")
}
if err != nil {
util.HandleError(err, "Unable to parse flag")
}
secretOverriding, err := cmd.Flags().GetBool("secret-overriding")
if err != nil {
util.HandleError(err, "Unable to parse flag")
}
watchMode, err := cmd.Flags().GetBool("watch")
if err != nil {
util.HandleError(err, "Unable to parse flag")
}
watchModeInterval, err := cmd.Flags().GetInt("watch-interval")
if err != nil {
util.HandleError(err, "Unable to parse flag")
}
// If the --watch flag has been set, the --watch-interval flag should also be set
if watchMode && watchModeInterval < 5 {
util.HandleError(fmt.Errorf("watch interval must be at least 5 seconds, you passed %d seconds", watchModeInterval))
}
shouldExpandSecrets, err := cmd.Flags().GetBool("expand")
if err != nil {
util.HandleError(err, "Unable to parse flag")
@@ -116,108 +145,50 @@ var runCmd = &cobra.Command{
Recursive: recursive,
}
if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
request.InfisicalToken = token.Token
} else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER {
request.UniversalAuthAccessToken = token.Token
}
secrets, err := util.GetAllEnvironmentVariables(request, projectConfigDir)
injectableEnvironment, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, shouldExpandSecrets, token)
if err != nil {
util.HandleError(err, "Could not fetch secrets", "If you are using a service token to fetch secrets, please ensure it is valid")
}
if secretOverriding {
secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_PERSONAL)
log.Debug().Msgf("injecting the following environment variables into shell: %v", injectableEnvironment.Variables)
if watchMode {
executeCommandWithWatchMode(command, args, watchModeInterval, request, projectConfigDir, shouldExpandSecrets, secretOverriding, token)
} else {
secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED)
}
if cmd.Flags().Changed("command") {
command := cmd.Flag("command").Value.String()
err = executeMultipleCommandWithEnvs(command, injectableEnvironment.SecretsCount, injectableEnvironment.Variables)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
if shouldExpandSecrets {
authParams := models.ExpandSecretsAuthentication{}
if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
authParams.InfisicalToken = token.Token
} else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER {
authParams.UniversalAuthAccessToken = token.Token
}
secrets = util.ExpandSecrets(secrets, authParams, projectConfigDir)
}
secretsByKey := getSecretsByKeys(secrets)
environmentVariables := make(map[string]string)
// add all existing environment vars
for _, s := range os.Environ() {
kv := strings.SplitN(s, "=", 2)
key := kv[0]
value := kv[1]
environmentVariables[key] = value
}
// check to see if there are any reserved key words in secrets to inject
filterReservedEnvVars(secretsByKey)
// now add infisical secrets
for k, v := range secretsByKey {
environmentVariables[k] = v.Value
}
// turn it back into a list of envs
var env []string
for key, value := range environmentVariables {
s := key + "=" + value
env = append(env, s)
}
log.Debug().Msgf("injecting the following environment variables into shell: %v", env)
Telemetry.CaptureEvent("cli-command:run",
posthog.NewProperties().
Set("secretsCount", len(secrets)).
Set("environment", environmentName).
Set("isUsingServiceToken", token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER).
Set("isUsingUniversalAuthToken", token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER).
Set("single-command", strings.Join(args, " ")).
Set("multi-command", cmd.Flag("command").Value.String()).
Set("version", util.CLI_VERSION))
if cmd.Flags().Changed("command") {
command := cmd.Flag("command").Value.String()
err = executeMultipleCommandWithEnvs(command, len(secretsByKey), env)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
} else {
err = executeSingleCommandWithEnvs(args, len(secretsByKey), env)
if err != nil {
fmt.Println(err)
os.Exit(1)
} else {
err = executeSingleCommandWithEnvs(args, injectableEnvironment.SecretsCount, injectableEnvironment.Variables)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
}
}
},
}
var (
reservedEnvVars = []string{
"HOME", "PATH", "PS1", "PS2",
"PWD", "EDITOR", "XAUTHORITY", "USER",
"TERM", "TERMINFO", "SHELL", "MAIL",
}
reservedEnvVarPrefixes = []string{
"XDG_",
"LC_",
}
)
func filterReservedEnvVars(env map[string]models.SingleEnvironmentVariable) {
var (
reservedEnvVars = []string{
"HOME", "PATH", "PS1", "PS2",
"PWD", "EDITOR", "XAUTHORITY", "USER",
"TERM", "TERMINFO", "SHELL", "MAIL",
}
reservedEnvVarPrefixes = []string{
"XDG_",
"LC_",
}
)
for _, reservedEnvName := range reservedEnvVars {
if _, ok := env[reservedEnvName]; ok {
delete(env, reservedEnvName)
@@ -237,13 +208,15 @@ func filterReservedEnvVars(env map[string]models.SingleEnvironmentVariable) {
func init() {
rootCmd.AddCommand(runCmd)
runCmd.Flags().String("token", "", "Fetch secrets using service token or machine identity access token")
runCmd.Flags().String("token", "", "fetch secrets using service token or machine identity access token")
runCmd.Flags().String("projectId", "", "manually set the project ID to fetch secrets from when using machine identity based auth")
runCmd.Flags().StringP("env", "e", "dev", "Set the environment (dev, prod, etc.) from which your secrets should be pulled from")
runCmd.Flags().Bool("expand", true, "Parse shell parameter expansions in your secrets")
runCmd.Flags().Bool("include-imports", true, "Import linked secrets ")
runCmd.Flags().Bool("recursive", false, "Fetch secrets from all sub-folders")
runCmd.Flags().Bool("secret-overriding", true, "Prioritizes personal secrets, if any, with the same name over shared secrets")
runCmd.Flags().StringP("env", "e", "dev", "set the environment (dev, prod, etc.) from which your secrets should be pulled from")
runCmd.Flags().Bool("expand", true, "parse shell parameter expansions in your secrets")
runCmd.Flags().Bool("include-imports", true, "import linked secrets ")
runCmd.Flags().Bool("recursive", false, "fetch secrets from all sub-folders")
runCmd.Flags().Bool("secret-overriding", true, "prioritizes personal secrets, if any, with the same name over shared secrets")
runCmd.Flags().Bool("watch", false, "enable reload of application when secrets change")
runCmd.Flags().Int("watch-interval", 10, "interval in seconds to check for secret changes")
runCmd.Flags().StringP("command", "c", "", "chained commands to execute (e.g. \"npm install && npm run dev; echo ...\")")
runCmd.Flags().StringP("tags", "t", "", "filter secrets by tag slugs ")
runCmd.Flags().String("path", "/", "get secrets within a folder path")
@@ -263,7 +236,7 @@ func executeSingleCommandWithEnvs(args []string, secretsCount int, env []string)
cmd.Stderr = os.Stderr
cmd.Env = env
return execCmd(cmd)
return execBasicCmd(cmd)
}
func executeMultipleCommandWithEnvs(fullCommand string, secretsCount int, env []string) error {
@@ -286,11 +259,10 @@ func executeMultipleCommandWithEnvs(fullCommand string, secretsCount int, env []
log.Info().Msgf(color.GreenString("Injecting %v Infisical secrets into your application process", secretsCount))
log.Debug().Msgf("executing command: %s %s %s \n", shell[0], shell[1], fullCommand)
return execCmd(cmd)
return execBasicCmd(cmd)
}
// Credit: inspired by AWS Valut
func execCmd(cmd *exec.Cmd) error {
func execBasicCmd(cmd *exec.Cmd) error {
sigChannel := make(chan os.Signal, 1)
signal.Notify(sigChannel)
@@ -314,3 +286,217 @@ func execCmd(cmd *exec.Cmd) error {
os.Exit(waitStatus.ExitStatus())
return nil
}
func waitForExitCommand(cmd *exec.Cmd) (int, error) {
if err := cmd.Wait(); err != nil {
// ignore errors
cmd.Process.Signal(os.Kill) // #nosec G104
if exitError, ok := err.(*exec.ExitError); ok {
return exitError.ExitCode(), exitError
}
return 2, err
}
waitStatus, ok := cmd.ProcessState.Sys().(syscall.WaitStatus)
if !ok {
return 2, fmt.Errorf("unexpected ProcessState type, expected syscall.WaitStatus, got %T", waitStatus)
}
return waitStatus.ExitStatus(), nil
}
func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInterval int, request models.GetAllSecretsParameters, projectConfigDir string, expandSecrets bool, secretOverriding bool, token *models.TokenDetails) {
var cmd *exec.Cmd
var err error
var lastSecretsFetch time.Time
var lastUpdateEvent time.Time
var watchMutex sync.Mutex
var processMutex sync.Mutex
var beingTerminated = false
var currentETag string
if err != nil {
util.HandleError(err, "Failed to fetch secrets")
}
runCommandWithWatcher := func(environmentVariables models.InjectableEnvironmentResult) {
currentETag = environmentVariables.ETag
secretsFetchedAt := time.Now()
if secretsFetchedAt.After(lastSecretsFetch) {
lastSecretsFetch = secretsFetchedAt
}
shouldRestartProcess := cmd != nil
// terminate the old process before starting a new one
if shouldRestartProcess {
log.Info().Msg(color.HiMagentaString("[HOT RELOAD] Environment changes detected. Reloading process..."))
beingTerminated = true
log.Debug().Msgf(color.HiMagentaString("[HOT RELOAD] Sending SIGTERM to PID %d", cmd.Process.Pid))
if e := cmd.Process.Signal(syscall.SIGTERM); e != nil {
log.Error().Err(e).Msg(color.HiMagentaString("[HOT RELOAD] Failed to send SIGTERM"))
}
// wait up to 10 sec for the process to exit
for i := 0; i < 10; i++ {
if !util.IsProcessRunning(cmd.Process) {
// process has been killed so we break out
break
}
if i == 5 {
log.Debug().Msg(color.HiMagentaString("[HOT RELOAD] Still waiting for process exit status"))
}
time.Sleep(time.Second)
}
// SIGTERM may not work on Windows so we try SIGKILL
if util.IsProcessRunning(cmd.Process) {
log.Debug().Msg(color.HiMagentaString("[HOT RELOAD] Process still hasn't fully exited, attempting SIGKILL"))
if e := cmd.Process.Kill(); e != nil {
log.Error().Err(e).Msg(color.HiMagentaString("[HOT RELOAD] Failed to send SIGKILL"))
}
}
cmd = nil
} else {
// If `cmd` is nil, we know this is the first time we are starting the process
log.Info().Msg(color.HiMagentaString("[HOT RELOAD] Watching for secret changes..."))
}
processMutex.Lock()
if lastUpdateEvent.After(secretsFetchedAt) {
processMutex.Unlock()
return
}
beingTerminated = false
watcherWaitGroup.Add(1)
// start the process
log.Info().Msgf(color.GreenString("Injecting %v Infisical secrets into your application process", environmentVariables.SecretsCount))
cmd, err = util.RunCommand(commandFlag, args, environmentVariables.Variables, false)
if err != nil {
defer watcherWaitGroup.Done()
util.HandleError(err)
}
go func() {
defer processMutex.Unlock()
defer watcherWaitGroup.Done()
exitCode, err := waitForExitCommand(cmd)
// ignore errors if we are being terminated
if !beingTerminated {
if err != nil {
if strings.HasPrefix(err.Error(), "exec") || strings.HasPrefix(err.Error(), "fork/exec") {
log.Error().Err(err).Msg("Failed to execute command")
}
if err.Error() != ErrManualSignalInterrupt.Error() {
log.Error().Err(err).Msg("Process exited with error")
}
}
os.Exit(exitCode)
}
}()
}
recheckSecretsChannel := make(chan bool, 1)
recheckSecretsChannel <- true
// a simple goroutine that triggers the recheckSecretsChan every watch interval (defaults to 10 seconds)
go func() {
for {
time.Sleep(time.Duration(watchModeInterval) * time.Second)
recheckSecretsChannel <- true
}
}()
for {
<-recheckSecretsChannel
watchMutex.Lock()
newEnvironmentVariables, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, expandSecrets, token)
if err != nil {
log.Error().Err(err).Msg("[HOT RELOAD] Failed to fetch secrets")
continue
}
if newEnvironmentVariables.ETag != currentETag {
runCommandWithWatcher(newEnvironmentVariables)
} else {
log.Debug().Msg("[HOT RELOAD] No changes detected in secrets, not reloading process")
}
watchMutex.Unlock()
}
}
func fetchAndFormatSecretsForShell(request models.GetAllSecretsParameters, projectConfigDir string, secretOverriding bool, shouldExpandSecrets bool, token *models.TokenDetails) (models.InjectableEnvironmentResult, error) {
if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
request.InfisicalToken = token.Token
} else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER {
request.UniversalAuthAccessToken = token.Token
}
secrets, err := util.GetAllEnvironmentVariables(request, projectConfigDir)
if err != nil {
return models.InjectableEnvironmentResult{}, err
}
if secretOverriding {
secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_PERSONAL)
} else {
secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED)
}
if shouldExpandSecrets {
authParams := models.ExpandSecretsAuthentication{}
if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
authParams.InfisicalToken = token.Token
} else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER {
authParams.UniversalAuthAccessToken = token.Token
}
secrets = util.ExpandSecrets(secrets, authParams, projectConfigDir)
}
secretsByKey := getSecretsByKeys(secrets)
environmentVariables := make(map[string]string)
// add all existing environment vars
for _, s := range os.Environ() {
kv := strings.SplitN(s, "=", 2)
key := kv[0]
value := kv[1]
environmentVariables[key] = value
}
// check to see if there are any reserved key words in secrets to inject
filterReservedEnvVars(secretsByKey)
// now add infisical secrets
for k, v := range secretsByKey {
environmentVariables[k] = v.Value
}
env := make([]string, 0, len(environmentVariables))
for key, value := range environmentVariables {
env = append(env, key+"="+value)
}
return models.InjectableEnvironmentResult{
Variables: env,
ETag: util.GenerateETagFromSecrets(secrets),
SecretsCount: len(secretsByKey),
}, nil
}

6
cli/packages/models/cli.go
View File

@@ -104,6 +104,12 @@ type GetAllSecretsParameters struct {
Recursive bool
}
type InjectableEnvironmentResult struct {
Variables []string
ETag string
SecretsCount int
}
type GetAllFoldersParameters struct {
WorkspaceId string
Environment string

92
cli/packages/util/exec.go Normal file
View File

@@ -0,0 +1,92 @@
package util
import (
"fmt"
"os"
"os/exec"
"os/signal"
"runtime"
"syscall"
)
func RunCommand(singleCommand string, args []string, env []string, waitForExit bool) (*exec.Cmd, error) {
var c *exec.Cmd
var err error
if singleCommand != "" {
c, err = RunCommandFromString(singleCommand, env, waitForExit)
} else {
c, err = RunCommandFromArgs(args, env, waitForExit)
}
return c, err
}
func IsProcessRunning(p *os.Process) bool {
err := p.Signal(syscall.Signal(0))
return err == nil
}
// For "infisical run -- COMMAND"
func RunCommandFromArgs(args []string, env []string, waitForExit bool) (*exec.Cmd, error) {
cmd := exec.Command(args[0], args[1:]...)
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Env = env
err := execCommand(cmd, waitForExit)
return cmd, err
}
func execCommand(cmd *exec.Cmd, waitForExit bool) error {
sigChannel := make(chan os.Signal, 1)
signal.Notify(sigChannel)
if err := cmd.Start(); err != nil {
return err
}
go func() {
for {
sig := <-sigChannel
_ = cmd.Process.Signal(sig) // process all sigs
}
}()
if !waitForExit {
return nil
}
if err := cmd.Wait(); err != nil {
_ = cmd.Process.Signal(os.Kill)
return fmt.Errorf("failed to wait for command termination: %v", err)
}
waitStatus := cmd.ProcessState.Sys().(syscall.WaitStatus)
os.Exit(waitStatus.ExitStatus())
return nil
}
// For "infisical run --command=COMMAND"
func RunCommandFromString(command string, env []string, waitForExit bool) (*exec.Cmd, error) {
shell := [2]string{"sh", "-c"}
if runtime.GOOS == "windows" {
shell = [2]string{"cmd", "/C"}
} else {
currentShell := os.Getenv("SHELL")
if currentShell != "" {
shell[0] = currentShell
}
}
cmd := exec.Command(shell[0], shell[1], command) // #nosec G204 nosemgrep: semgrep_configs.prohibit-exec-command
cmd.Env = env
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := execCommand(cmd, waitForExit)
return cmd, err
}

14
cli/packages/util/helper.go
View File

@@ -4,6 +4,7 @@ import (
"bytes"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"math/rand"
"os"
@@ -298,3 +299,16 @@ func GenerateRandomString(length int) string {
}
return string(b)
}
func GenerateETagFromSecrets(secrets []models.SingleEnvironmentVariable) string {
sortedSecrets := SortSecretsByKeys(secrets)
content := []byte{}
for _, secret := range sortedSecrets {
content = append(content, []byte(secret.Key)...)
content = append(content, []byte(secret.Value)...)
}
hash := sha256.Sum256(content)
return fmt.Sprintf(`"%s"`, hex.EncodeToString(hash[:]))
}

50
docs/cli/commands/run.mdx
View File

@@ -47,20 +47,20 @@ $ infisical run -- npm run dev
Used to fetch secrets via a [machine identity](/documentation/platform/identities/machine-identities) apposed to logged in credentials. Simply, export this variable in the terminal before running this command.
```bash
# Example
export INFISICAL_TOKEN=$(infisical login --method=universal-auth --client-id=<identity-client-id> --client-secret=<identity-client-secret> --silent --plain) # --plain flag will output only the token, so it can be fed to an environment variable. --silent will disable any update messages.
# Example
export INFISICAL_TOKEN=$(infisical login --method=universal-auth --client-id=<identity-client-id> --client-secret=<identity-client-secret> --silent --plain) # --plain flag will output only the token, so it can be fed to an environment variable. --silent will disable any update messages.
```
<Info>
Alternatively, you may use service tokens.
Please note, however, that service tokens are being deprecated in favor of [machine identities](/documentation/platform/identities/machine-identities). They will be removed in the future in accordance with the deprecation notice and timeline stated [here](https://infisical.com/blog/deprecating-api-keys).
```bash
# Example
export INFISICAL_TOKEN=<service-token>
# Example
export INFISICAL_TOKEN=<service-token>
```
</Info>
</Info>
</Accordion>
<Accordion title="INFISICAL_DISABLE_UPDATE_CHECK">
@@ -69,22 +69,30 @@ $ infisical run -- npm run dev
To use, simply export this variable in the terminal before running this command.
```bash
# Example
export INFISICAL_DISABLE_UPDATE_CHECK=true
# Example
export INFISICAL_DISABLE_UPDATE_CHECK=true
```
</Accordion>
### Flags
<Accordion title="--project-config-dir">
<Accordion title="--watch">
By passing the `watch` flag, you are telling the CLI to watch for changes that happen in your Infisical project.
If secret changes happen, the command you provided will automatically be restarted with the new environment variables attached.
```bash
# Example
infisical run --watch -- printenv
```
</Accordion>
<Accordion title="--project-config-dir">
Explicitly set the directory where the .infisical.json resides. This is useful for some monorepo setups.
```bash
# Example
infisical run --project-config-dir=/some-dir -- printenv
# Example
infisical run --project-config-dir=/some-dir -- printenv
```
</Accordion>
<Accordion title="--command">
@@ -172,3 +180,19 @@ $ infisical run -- npm run dev
</Accordion>
</Accordion>
## Automatically reload command when secrets change
To automatically reload your command when secrets change, use the `--watch` flag.
```bash
infisical run --watch -- npm run dev
```
This will watch for changes in your secrets and automatically restart your command with the new secrets.
When your command restarts, it will have the new environment variables injeceted into it.
<Note>
Please note that this feature is intended for development purposes. It is not recommended to use this in production environments. Generally it's not recommended to automatically reload your application in production when remote changes are made.
</Note>