diff --git a/cli/packages/api/api.go b/cli/packages/api/api.go index c6ecd8bcd..995385b65 100644 --- a/cli/packages/api/api.go +++ b/cli/packages/api/api.go @@ -415,6 +415,10 @@ func CallGetRawSecretsV3(httpClient *resty.Client, request GetRawSecretsV3Reques req.SetQueryParam("recursive", "true") } + if request.ExpandSecretReferences { + req.SetQueryParam("expandSecretReferences", "true") + } + response, err := req.Get(fmt.Sprintf("%v/v3/secrets/raw", config.INFISICAL_URL)) if err != nil { diff --git a/cli/packages/api/model.go b/cli/packages/api/model.go index 5d89bf3f0..4b6c5a761 100644 --- a/cli/packages/api/model.go +++ b/cli/packages/api/model.go @@ -569,12 +569,13 @@ type CreateDynamicSecretLeaseV1Response struct { } type GetRawSecretsV3Request struct { - Environment string `json:"environment"` - WorkspaceId string `json:"workspaceId"` - SecretPath string `json:"secretPath"` - IncludeImport bool `json:"include_imports"` - Recursive bool `json:"recursive"` - TagSlugs string `json:"tagSlugs,omitempty"` + Environment string `json:"environment"` + WorkspaceId string `json:"workspaceId"` + SecretPath string `json:"secretPath"` + IncludeImport bool `json:"include_imports"` + Recursive bool `json:"recursive"` + TagSlugs string `json:"tagSlugs,omitempty"` + ExpandSecretReferences bool `json:"expandSecretReferences,omitempty"` } type GetRawSecretsV3Response struct { diff --git a/cli/packages/cmd/agent.go b/cli/packages/cmd/agent.go index e489c198c..10ace5efe 100644 --- a/cli/packages/cmd/agent.go +++ b/cli/packages/cmd/agent.go @@ -313,7 +313,15 @@ func ParseAgentConfig(configFile []byte) (*Config, error) { } type secretArguments struct { - IsRecursive bool `json:"recursive"` + IsRecursive bool `json:"recursive"` + ShouldExpandSecretReferences *bool `json:"expandSecretReferences,omitempty"` +} + +func (s *secretArguments) SetDefaults() { + if s.ShouldExpandSecretReferences == nil { + var bool = true + s.ShouldExpandSecretReferences = &bool + } } func secretTemplateFunction(accessToken string, existingEtag string, currentEtag *string) func(string, string, string, ...string) ([]models.SingleEnvironmentVariable, error) { @@ -329,7 +337,9 @@ func secretTemplateFunction(accessToken string, existingEtag string, currentEtag } } - res, err := util.GetPlainTextSecretsV3(accessToken, projectID, envSlug, secretPath, false, parsedArguments.IsRecursive, "") + parsedArguments.SetDefaults() + + res, err := util.GetPlainTextSecretsV3(accessToken, projectID, envSlug, secretPath, false, parsedArguments.IsRecursive, "", *parsedArguments.ShouldExpandSecretReferences) if err != nil { return nil, err } @@ -338,9 +348,7 @@ func secretTemplateFunction(accessToken string, existingEtag string, currentEtag *currentEtag = res.Etag } - expandedSecrets := util.ExpandSecrets(res.Secrets, models.ExpandSecretsAuthentication{UniversalAuthAccessToken: accessToken}, "") - - return expandedSecrets, nil + return res.Secrets, nil } } diff --git a/cli/packages/cmd/export.go b/cli/packages/cmd/export.go index f6b028b7a..6f02408fd 100644 --- a/cli/packages/cmd/export.go +++ b/cli/packages/cmd/export.go @@ -87,11 +87,12 @@ var exportCmd = &cobra.Command{ } request := models.GetAllSecretsParameters{ - Environment: environmentName, - TagSlugs: tagSlugs, - WorkspaceId: projectId, - SecretsPath: secretsPath, - IncludeImport: includeImports, + Environment: environmentName, + TagSlugs: tagSlugs, + WorkspaceId: projectId, + SecretsPath: secretsPath, + IncludeImport: includeImports, + ExpandSecretReferences: shouldExpandSecrets, } if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { @@ -137,18 +138,6 @@ var exportCmd = &cobra.Command{ } var output string - if shouldExpandSecrets { - - authParams := models.ExpandSecretsAuthentication{} - - if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { - authParams.InfisicalToken = token.Token - } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { - authParams.UniversalAuthAccessToken = token.Token - } - - secrets = util.ExpandSecrets(secrets, authParams, "") - } secrets = util.FilterSecretsByTag(secrets, tagSlugs) secrets = util.SortSecretsByKeys(secrets) diff --git a/cli/packages/cmd/run.go b/cli/packages/cmd/run.go index fa5176d89..a232896f1 100644 --- a/cli/packages/cmd/run.go +++ b/cli/packages/cmd/run.go @@ -137,15 +137,16 @@ var runCmd = &cobra.Command{ } request := models.GetAllSecretsParameters{ - Environment: environmentName, - WorkspaceId: projectId, - TagSlugs: tagSlugs, - SecretsPath: secretsPath, - IncludeImport: includeImports, - Recursive: recursive, + Environment: environmentName, + WorkspaceId: projectId, + TagSlugs: tagSlugs, + SecretsPath: secretsPath, + IncludeImport: includeImports, + Recursive: recursive, + ExpandSecretReferences: shouldExpandSecrets, } - injectableEnvironment, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, shouldExpandSecrets, token) + injectableEnvironment, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, token) if err != nil { util.HandleError(err, "Could not fetch secrets", "If you are using a service token to fetch secrets, please ensure it is valid") } @@ -153,7 +154,7 @@ var runCmd = &cobra.Command{ log.Debug().Msgf("injecting the following environment variables into shell: %v", injectableEnvironment.Variables) if watchMode { - executeCommandWithWatchMode(command, args, watchModeInterval, request, projectConfigDir, shouldExpandSecrets, secretOverriding, token) + executeCommandWithWatchMode(command, args, watchModeInterval, request, projectConfigDir, secretOverriding, token) } else { if cmd.Flags().Changed("command") { command := cmd.Flag("command").Value.String() @@ -306,7 +307,7 @@ func waitForExitCommand(cmd *exec.Cmd) (int, error) { return waitStatus.ExitStatus(), nil } -func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInterval int, request models.GetAllSecretsParameters, projectConfigDir string, expandSecrets bool, secretOverriding bool, token *models.TokenDetails) { +func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInterval int, request models.GetAllSecretsParameters, projectConfigDir string, secretOverriding bool, token *models.TokenDetails) { var cmd *exec.Cmd var err error @@ -420,7 +421,7 @@ func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInt <-recheckSecretsChannel watchMutex.Lock() - newEnvironmentVariables, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, expandSecrets, token) + newEnvironmentVariables, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, token) if err != nil { log.Error().Err(err).Msg("[HOT RELOAD] Failed to fetch secrets") continue @@ -437,7 +438,7 @@ func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInt } } -func fetchAndFormatSecretsForShell(request models.GetAllSecretsParameters, projectConfigDir string, secretOverriding bool, shouldExpandSecrets bool, token *models.TokenDetails) (models.InjectableEnvironmentResult, error) { +func fetchAndFormatSecretsForShell(request models.GetAllSecretsParameters, projectConfigDir string, secretOverriding bool, token *models.TokenDetails) (models.InjectableEnvironmentResult, error) { if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { request.InfisicalToken = token.Token @@ -457,19 +458,6 @@ func fetchAndFormatSecretsForShell(request models.GetAllSecretsParameters, proje secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED) } - if shouldExpandSecrets { - - authParams := models.ExpandSecretsAuthentication{} - - if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { - authParams.InfisicalToken = token.Token - } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { - authParams.UniversalAuthAccessToken = token.Token - } - - secrets = util.ExpandSecrets(secrets, authParams, projectConfigDir) - } - secretsByKey := getSecretsByKeys(secrets) environmentVariables := make(map[string]string) diff --git a/cli/packages/cmd/secrets.go b/cli/packages/cmd/secrets.go index e2987cc7d..eff011c5e 100644 --- a/cli/packages/cmd/secrets.go +++ b/cli/packages/cmd/secrets.go @@ -79,12 +79,13 @@ var secretsCmd = &cobra.Command{ } request := models.GetAllSecretsParameters{ - Environment: environmentName, - WorkspaceId: projectId, - TagSlugs: tagSlugs, - SecretsPath: secretsPath, - IncludeImport: includeImports, - Recursive: recursive, + Environment: environmentName, + WorkspaceId: projectId, + TagSlugs: tagSlugs, + SecretsPath: secretsPath, + IncludeImport: includeImports, + Recursive: recursive, + ExpandSecretReferences: shouldExpandSecrets, } if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { @@ -104,17 +105,6 @@ var secretsCmd = &cobra.Command{ secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED) } - if shouldExpandSecrets { - authParams := models.ExpandSecretsAuthentication{} - if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { - authParams.InfisicalToken = token.Token - } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { - authParams.UniversalAuthAccessToken = token.Token - } - - secrets = util.ExpandSecrets(secrets, authParams, "") - } - // Sort the secrets by key so we can create a consistent output secrets = util.SortSecretsByKeys(secrets) @@ -382,12 +372,13 @@ func getSecretsByNames(cmd *cobra.Command, args []string) { } request := models.GetAllSecretsParameters{ - Environment: environmentName, - WorkspaceId: projectId, - TagSlugs: tagSlugs, - SecretsPath: secretsPath, - IncludeImport: includeImports, - Recursive: recursive, + Environment: environmentName, + WorkspaceId: projectId, + TagSlugs: tagSlugs, + SecretsPath: secretsPath, + IncludeImport: includeImports, + Recursive: recursive, + ExpandSecretReferences: shouldExpand, } if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { @@ -407,17 +398,6 @@ func getSecretsByNames(cmd *cobra.Command, args []string) { secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED) } - if shouldExpand { - authParams := models.ExpandSecretsAuthentication{} - if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER { - authParams.InfisicalToken = token.Token - } else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER { - authParams.UniversalAuthAccessToken = token.Token - } - - secrets = util.ExpandSecrets(secrets, authParams, "") - } - requestedSecrets := []models.SingleEnvironmentVariable{} secretsMap := getSecretsByKeys(secrets) diff --git a/cli/packages/models/cli.go b/cli/packages/models/cli.go index 08e7c3957..8b9fef6f6 100644 --- a/cli/packages/models/cli.go +++ b/cli/packages/models/cli.go @@ -104,6 +104,7 @@ type GetAllSecretsParameters struct { SecretsPath string IncludeImport bool Recursive bool + ExpandSecretReferences bool } type InjectableEnvironmentResult struct { diff --git a/cli/packages/util/secrets.go b/cli/packages/util/secrets.go index f375ad8aa..5e19ea664 100644 --- a/cli/packages/util/secrets.go +++ b/cli/packages/util/secrets.go @@ -8,8 +8,6 @@ import ( "errors" "fmt" "os" - "path" - "regexp" "strings" "unicode" @@ -21,7 +19,7 @@ import ( "github.com/zalando/go-keyring" ) -func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment string, secretPath string, includeImports bool, recursive bool, tagSlugs string) ([]models.SingleEnvironmentVariable, error) { +func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment string, secretPath string, includeImports bool, recursive bool, tagSlugs string, expandSecretReferences bool) ([]models.SingleEnvironmentVariable, error) { serviceTokenParts := strings.SplitN(fullServiceToken, ".", 4) if len(serviceTokenParts) < 4 { return nil, fmt.Errorf("invalid service token entered. Please double check your service token and try again") @@ -49,12 +47,13 @@ func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment str } rawSecrets, err := api.CallGetRawSecretsV3(httpClient, api.GetRawSecretsV3Request{ - WorkspaceId: serviceTokenDetails.Workspace, - Environment: environment, - SecretPath: secretPath, - IncludeImport: includeImports, - Recursive: recursive, - TagSlugs: tagSlugs, + WorkspaceId: serviceTokenDetails.Workspace, + Environment: environment, + SecretPath: secretPath, + IncludeImport: includeImports, + Recursive: recursive, + TagSlugs: tagSlugs, + ExpandSecretReferences: expandSecretReferences, }) if err != nil { @@ -78,17 +77,18 @@ func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment str } -func GetPlainTextSecretsV3(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool, recursive bool, tagSlugs string) (models.PlaintextSecretResult, error) { +func GetPlainTextSecretsV3(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool, recursive bool, tagSlugs string, expandSecretReferences bool) (models.PlaintextSecretResult, error) { httpClient := resty.New() httpClient.SetAuthToken(accessToken). SetHeader("Accept", "application/json") getSecretsRequest := api.GetRawSecretsV3Request{ - WorkspaceId: workspaceId, - Environment: environmentName, - IncludeImport: includeImports, - Recursive: recursive, - TagSlugs: tagSlugs, + WorkspaceId: workspaceId, + Environment: environmentName, + IncludeImport: includeImports, + Recursive: recursive, + TagSlugs: tagSlugs, + ExpandSecretReferences: expandSecretReferences, } if secretsPath != "" { @@ -284,7 +284,7 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo } res, err := GetPlainTextSecretsV3(loggedInUserDetails.UserCredentials.JTWToken, infisicalDotJson.WorkspaceId, - params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs) + params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, true) log.Debug().Msgf("GetAllEnvironmentVariables: Trying to fetch secrets JTW token [err=%s]", err) if err == nil { @@ -313,7 +313,7 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo } else { if params.InfisicalToken != "" { log.Debug().Msg("Trying to fetch secrets using service token") - secretsToReturn, errorToReturn = GetPlainTextSecretsViaServiceToken(params.InfisicalToken, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs) + secretsToReturn, errorToReturn = GetPlainTextSecretsViaServiceToken(params.InfisicalToken, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, params.ExpandSecretReferences) } else if params.UniversalAuthAccessToken != "" { if params.WorkspaceId == "" { @@ -321,7 +321,7 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo } log.Debug().Msg("Trying to fetch secrets using universal auth") - res, err := GetPlainTextSecretsV3(params.UniversalAuthAccessToken, params.WorkspaceId, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs) + res, err := GetPlainTextSecretsV3(params.UniversalAuthAccessToken, params.WorkspaceId, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, params.ExpandSecretReferences) errorToReturn = err secretsToReturn = res.Secrets @@ -331,44 +331,6 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo return secretsToReturn, errorToReturn } -var secRefRegex = regexp.MustCompile(`\${([^\}]*)}`) - -func recursivelyExpandSecret(expandedSecs map[string]string, interpolatedSecs map[string]string, crossSecRefFetch func(env string, path []string, key string) string, key string) string { - if v, ok := expandedSecs[key]; ok { - return v - } - - interpolatedVal, ok := interpolatedSecs[key] - if !ok { - HandleError(fmt.Errorf("could not find refered secret - %s", key), "Kindly check whether its provided") - } - - refs := secRefRegex.FindAllStringSubmatch(interpolatedVal, -1) - for _, val := range refs { - // key: "${something}" val: [${something},something] - interpolatedExp, interpolationKey := val[0], val[1] - ref := strings.Split(interpolationKey, ".") - - // ${KEY1} => [key1] - if len(ref) == 1 { - val := recursivelyExpandSecret(expandedSecs, interpolatedSecs, crossSecRefFetch, interpolationKey) - interpolatedVal = strings.ReplaceAll(interpolatedVal, interpolatedExp, val) - continue - } - - // cross board reference ${env.folder.key1} => [env folder key1] - if len(ref) > 1 { - secEnv, tmpSecPath, secKey := ref[0], ref[1:len(ref)-1], ref[len(ref)-1] - interpolatedSecs[interpolationKey] = crossSecRefFetch(secEnv, tmpSecPath, secKey) // get the reference value - val := recursivelyExpandSecret(expandedSecs, interpolatedSecs, crossSecRefFetch, interpolationKey) - interpolatedVal = strings.ReplaceAll(interpolatedVal, interpolatedExp, val) - } - - } - expandedSecs[key] = interpolatedVal - return interpolatedVal -} - func getSecretsByKeys(secrets []models.SingleEnvironmentVariable) map[string]models.SingleEnvironmentVariable { secretMapByName := make(map[string]models.SingleEnvironmentVariable, len(secrets)) @@ -379,70 +341,6 @@ func getSecretsByKeys(secrets []models.SingleEnvironmentVariable) map[string]mod return secretMapByName } -func ExpandSecrets(secrets []models.SingleEnvironmentVariable, auth models.ExpandSecretsAuthentication, projectConfigPathDir string) []models.SingleEnvironmentVariable { - expandedSecs := make(map[string]string) - interpolatedSecs := make(map[string]string) - // map[env.secret-path][keyname]Secret - crossEnvRefSecs := make(map[string]map[string]models.SingleEnvironmentVariable) // a cache to hold all cross board reference secrets - - for _, sec := range secrets { - // get all references in a secret - refs := secRefRegex.FindAllStringSubmatch(sec.Value, -1) - // nil means its a secret without reference - if refs == nil { - expandedSecs[sec.Key] = sec.Value // atomic secrets without any interpolation - } else { - interpolatedSecs[sec.Key] = sec.Value - } - } - - for i, sec := range secrets { - // already present pick that up - if expandedVal, ok := expandedSecs[sec.Key]; ok { - secrets[i].Value = expandedVal - continue - } - - expandedVal := recursivelyExpandSecret(expandedSecs, interpolatedSecs, func(env string, secPaths []string, secKey string) string { - secPaths = append([]string{"/"}, secPaths...) - secPath := path.Join(secPaths...) - - secPathDot := strings.Join(secPaths, ".") - uniqKey := fmt.Sprintf("%s.%s", env, secPathDot) - - if crossRefSec, ok := crossEnvRefSecs[uniqKey]; !ok { - - var refSecs []models.SingleEnvironmentVariable - var err error - - // if not in cross reference cache, fetch it from server - if auth.InfisicalToken != "" { - refSecs, err = GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: env, InfisicalToken: auth.InfisicalToken, SecretsPath: secPath}, projectConfigPathDir) - } else if auth.UniversalAuthAccessToken != "" { - refSecs, err = GetAllEnvironmentVariables((models.GetAllSecretsParameters{Environment: env, UniversalAuthAccessToken: auth.UniversalAuthAccessToken, SecretsPath: secPath, WorkspaceId: sec.WorkspaceId}), projectConfigPathDir) - } else if IsLoggedIn() { - refSecs, err = GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: env, SecretsPath: secPath}, projectConfigPathDir) - } else { - HandleError(errors.New("no authentication provided"), "Please provide authentication to fetch secrets") - } - if err != nil { - HandleError(err, fmt.Sprintf("Could not fetch secrets in environment: %s secret-path: %s", env, secPath), "If you are using a service token to fetch secrets, please ensure it is valid") - } - refSecsByKey := getSecretsByKeys(refSecs) - // save it to avoid calling api again for same environment and folder path - crossEnvRefSecs[uniqKey] = refSecsByKey - return refSecsByKey[secKey].Value - - } else { - return crossRefSec[secKey].Value - } - }, sec.Key) - - secrets[i].Value = expandedVal - } - return secrets -} - func OverrideSecrets(secrets []models.SingleEnvironmentVariable, secretType string) []models.SingleEnvironmentVariable { personalSecrets := make(map[string]models.SingleEnvironmentVariable) sharedSecrets := make(map[string]models.SingleEnvironmentVariable) diff --git a/cli/secret-render-template b/cli/secret-render-template index 537b68e03..41489a074 100644 --- a/cli/secret-render-template +++ b/cli/secret-render-template @@ -1,4 +1,4 @@ -{{- with secret "ae06a663-2740-49c5-b998-bf4ceca2c120" "dev" "/" `{"recursive":true}` }} +{{- with secret "8fac9f01-4a81-44d7-8ff0-3d7be684f56f" "staging" "/" `{"recursive":true, "expandSecretReferences": false}` }} {{- range . }} {{ .Key }}={{ .Value }} {{- end }}