feat: switched expandSecretReferences to server based one and added same support in template too

cli/packages/api/api.go

@@ -415,6 +415,10 @@ func CallGetRawSecretsV3(httpClient *resty.Client, request GetRawSecretsV3Reques
 		req.SetQueryParam("recursive", "true")
 	}
 
+	if request.ExpandSecretReferences {
+		req.SetQueryParam("expandSecretReferences", "true")
+	}
+
 	response, err := req.Get(fmt.Sprintf("%v/v3/secrets/raw", config.INFISICAL_URL))
 
 	if err != nil {

cli/packages/api/model.go

@@ -569,12 +569,13 @@ type CreateDynamicSecretLeaseV1Response struct {
 }
 
 type GetRawSecretsV3Request struct {
-	Environment   string `json:"environment"`
+	Environment            string `json:"environment"`
-	WorkspaceId   string `json:"workspaceId"`
+	WorkspaceId            string `json:"workspaceId"`
-	SecretPath    string `json:"secretPath"`
+	SecretPath             string `json:"secretPath"`
-	IncludeImport bool   `json:"include_imports"`
+	IncludeImport          bool   `json:"include_imports"`
-	Recursive     bool   `json:"recursive"`
+	Recursive              bool   `json:"recursive"`
-	TagSlugs      string `json:"tagSlugs,omitempty"`
+	TagSlugs               string `json:"tagSlugs,omitempty"`
+	ExpandSecretReferences bool   `json:"expandSecretReferences,omitempty"`
 }
 
 type GetRawSecretsV3Response struct {

cli/packages/cmd/agent.go

@@ -313,7 +313,15 @@ func ParseAgentConfig(configFile []byte) (*Config, error) {
 }
 
 type secretArguments struct {
-	IsRecursive bool `json:"recursive"`
+	IsRecursive                  bool  `json:"recursive"`
+	ShouldExpandSecretReferences *bool `json:"expandSecretReferences,omitempty"`
+}
+
+func (s *secretArguments) SetDefaults() {
+	if s.ShouldExpandSecretReferences == nil {
+		var bool = true
+		s.ShouldExpandSecretReferences = &bool
+	}
 }
 
 func secretTemplateFunction(accessToken string, existingEtag string, currentEtag *string) func(string, string, string, ...string) ([]models.SingleEnvironmentVariable, error) {
@@ -329,7 +337,9 @@ func secretTemplateFunction(accessToken string, existingEtag string, currentEtag
 			}
 		}
 
-		res, err := util.GetPlainTextSecretsV3(accessToken, projectID, envSlug, secretPath, false, parsedArguments.IsRecursive, "")
+		parsedArguments.SetDefaults()
+
+		res, err := util.GetPlainTextSecretsV3(accessToken, projectID, envSlug, secretPath, false, parsedArguments.IsRecursive, "", *parsedArguments.ShouldExpandSecretReferences)
 		if err != nil {
 			return nil, err
 		}
@@ -338,9 +348,7 @@ func secretTemplateFunction(accessToken string, existingEtag string, currentEtag
 			*currentEtag = res.Etag
 		}
 
-		expandedSecrets := util.ExpandSecrets(res.Secrets, models.ExpandSecretsAuthentication{UniversalAuthAccessToken: accessToken}, "")
+		return res.Secrets, nil
-
-		return expandedSecrets, nil
 	}
 }
 

cli/packages/cmd/export.go

@@ -87,11 +87,12 @@ var exportCmd = &cobra.Command{
 		}
 
 		request := models.GetAllSecretsParameters{
-			Environment:   environmentName,
+			Environment:            environmentName,
-			TagSlugs:      tagSlugs,
+			TagSlugs:               tagSlugs,
-			WorkspaceId:   projectId,
+			WorkspaceId:            projectId,
-			SecretsPath:   secretsPath,
+			SecretsPath:            secretsPath,
-			IncludeImport: includeImports,
+			IncludeImport:          includeImports,
+			ExpandSecretReferences: shouldExpandSecrets,
 		}
 
 		if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
@@ -137,18 +138,6 @@ var exportCmd = &cobra.Command{
 		}
 
 		var output string
-		if shouldExpandSecrets {
-
-			authParams := models.ExpandSecretsAuthentication{}
-
-			if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
-				authParams.InfisicalToken = token.Token
-			} else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER {
-				authParams.UniversalAuthAccessToken = token.Token
-			}
-
-			secrets = util.ExpandSecrets(secrets, authParams, "")
-		}
 		secrets = util.FilterSecretsByTag(secrets, tagSlugs)
 		secrets = util.SortSecretsByKeys(secrets)
 

cli/packages/cmd/run.go

@@ -137,15 +137,16 @@ var runCmd = &cobra.Command{
 		}
 
 		request := models.GetAllSecretsParameters{
-			Environment:   environmentName,
+			Environment:            environmentName,
-			WorkspaceId:   projectId,
+			WorkspaceId:            projectId,
-			TagSlugs:      tagSlugs,
+			TagSlugs:               tagSlugs,
-			SecretsPath:   secretsPath,
+			SecretsPath:            secretsPath,
-			IncludeImport: includeImports,
+			IncludeImport:          includeImports,
-			Recursive:     recursive,
+			Recursive:              recursive,
+			ExpandSecretReferences: shouldExpandSecrets,
 		}
 
-		injectableEnvironment, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, shouldExpandSecrets, token)
+		injectableEnvironment, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, token)
 		if err != nil {
 			util.HandleError(err, "Could not fetch secrets", "If you are using a service token to fetch secrets, please ensure it is valid")
 		}
@@ -153,7 +154,7 @@ var runCmd = &cobra.Command{
 		log.Debug().Msgf("injecting the following environment variables into shell: %v", injectableEnvironment.Variables)
 
 		if watchMode {
-			executeCommandWithWatchMode(command, args, watchModeInterval, request, projectConfigDir, shouldExpandSecrets, secretOverriding, token)
+			executeCommandWithWatchMode(command, args, watchModeInterval, request, projectConfigDir, secretOverriding, token)
 		} else {
 			if cmd.Flags().Changed("command") {
 				command := cmd.Flag("command").Value.String()
@@ -306,7 +307,7 @@ func waitForExitCommand(cmd *exec.Cmd) (int, error) {
 	return waitStatus.ExitStatus(), nil
 }
 
-func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInterval int, request models.GetAllSecretsParameters, projectConfigDir string, expandSecrets bool, secretOverriding bool, token *models.TokenDetails) {
+func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInterval int, request models.GetAllSecretsParameters, projectConfigDir string, secretOverriding bool, token *models.TokenDetails) {
 
 	var cmd *exec.Cmd
 	var err error
@@ -420,7 +421,7 @@ func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInt
 		<-recheckSecretsChannel
 		watchMutex.Lock()
 
-		newEnvironmentVariables, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, expandSecrets, token)
+		newEnvironmentVariables, err := fetchAndFormatSecretsForShell(request, projectConfigDir, secretOverriding, token)
 		if err != nil {
 			log.Error().Err(err).Msg("[HOT RELOAD] Failed to fetch secrets")
 			continue
@@ -437,7 +438,7 @@ func executeCommandWithWatchMode(commandFlag string, args []string, watchModeInt
 	}
 }
 
-func fetchAndFormatSecretsForShell(request models.GetAllSecretsParameters, projectConfigDir string, secretOverriding bool, shouldExpandSecrets bool, token *models.TokenDetails) (models.InjectableEnvironmentResult, error) {
+func fetchAndFormatSecretsForShell(request models.GetAllSecretsParameters, projectConfigDir string, secretOverriding bool, token *models.TokenDetails) (models.InjectableEnvironmentResult, error) {
 
 	if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
 		request.InfisicalToken = token.Token
@@ -457,19 +458,6 @@ func fetchAndFormatSecretsForShell(request models.GetAllSecretsParameters, proje
 		secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED)
 	}
 
-	if shouldExpandSecrets {
-
-		authParams := models.ExpandSecretsAuthentication{}
-
-		if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
-			authParams.InfisicalToken = token.Token
-		} else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER {
-			authParams.UniversalAuthAccessToken = token.Token
-		}
-
-		secrets = util.ExpandSecrets(secrets, authParams, projectConfigDir)
-	}
-
 	secretsByKey := getSecretsByKeys(secrets)
 	environmentVariables := make(map[string]string)
 

cli/packages/cmd/secrets.go

@@ -79,12 +79,13 @@ var secretsCmd = &cobra.Command{
 		}
 
 		request := models.GetAllSecretsParameters{
-			Environment:   environmentName,
+			Environment:            environmentName,
-			WorkspaceId:   projectId,
+			WorkspaceId:            projectId,
-			TagSlugs:      tagSlugs,
+			TagSlugs:               tagSlugs,
-			SecretsPath:   secretsPath,
+			SecretsPath:            secretsPath,
-			IncludeImport: includeImports,
+			IncludeImport:          includeImports,
-			Recursive:     recursive,
+			Recursive:              recursive,
+			ExpandSecretReferences: shouldExpandSecrets,
 		}
 
 		if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
@@ -104,17 +105,6 @@ var secretsCmd = &cobra.Command{
 			secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED)
 		}
 
-		if shouldExpandSecrets {
-			authParams := models.ExpandSecretsAuthentication{}
-			if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
-				authParams.InfisicalToken = token.Token
-			} else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER {
-				authParams.UniversalAuthAccessToken = token.Token
-			}
-
-			secrets = util.ExpandSecrets(secrets, authParams, "")
-		}
-
 		// Sort the secrets by key so we can create a consistent output
 		secrets = util.SortSecretsByKeys(secrets)
 
@@ -382,12 +372,13 @@ func getSecretsByNames(cmd *cobra.Command, args []string) {
 	}
 
 	request := models.GetAllSecretsParameters{
-		Environment:   environmentName,
+		Environment:            environmentName,
-		WorkspaceId:   projectId,
+		WorkspaceId:            projectId,
-		TagSlugs:      tagSlugs,
+		TagSlugs:               tagSlugs,
-		SecretsPath:   secretsPath,
+		SecretsPath:            secretsPath,
-		IncludeImport: includeImports,
+		IncludeImport:          includeImports,
-		Recursive:     recursive,
+		Recursive:              recursive,
+		ExpandSecretReferences: shouldExpand,
 	}
 
 	if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
@@ -407,17 +398,6 @@ func getSecretsByNames(cmd *cobra.Command, args []string) {
 		secrets = util.OverrideSecrets(secrets, util.SECRET_TYPE_SHARED)
 	}
 
-	if shouldExpand {
-		authParams := models.ExpandSecretsAuthentication{}
-		if token != nil && token.Type == util.SERVICE_TOKEN_IDENTIFIER {
-			authParams.InfisicalToken = token.Token
-		} else if token != nil && token.Type == util.UNIVERSAL_AUTH_TOKEN_IDENTIFIER {
-			authParams.UniversalAuthAccessToken = token.Token
-		}
-
-		secrets = util.ExpandSecrets(secrets, authParams, "")
-	}
-
 	requestedSecrets := []models.SingleEnvironmentVariable{}
 
 	secretsMap := getSecretsByKeys(secrets)

cli/packages/models/cli.go

@@ -104,6 +104,7 @@ type GetAllSecretsParameters struct {
 	SecretsPath              string
 	IncludeImport            bool
 	Recursive                bool
+	ExpandSecretReferences   bool
 }
 
 type InjectableEnvironmentResult struct {

cli/packages/util/secrets.go

@@ -8,8 +8,6 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"path"
-	"regexp"
 	"strings"
 	"unicode"
 
@@ -21,7 +19,7 @@ import (
 	"github.com/zalando/go-keyring"
 )
 
-func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment string, secretPath string, includeImports bool, recursive bool, tagSlugs string) ([]models.SingleEnvironmentVariable, error) {
+func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment string, secretPath string, includeImports bool, recursive bool, tagSlugs string, expandSecretReferences bool) ([]models.SingleEnvironmentVariable, error) {
 	serviceTokenParts := strings.SplitN(fullServiceToken, ".", 4)
 	if len(serviceTokenParts) < 4 {
 		return nil, fmt.Errorf("invalid service token entered. Please double check your service token and try again")
@@ -49,12 +47,13 @@ func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment str
 	}
 
 	rawSecrets, err := api.CallGetRawSecretsV3(httpClient, api.GetRawSecretsV3Request{
-		WorkspaceId:   serviceTokenDetails.Workspace,
+		WorkspaceId:            serviceTokenDetails.Workspace,
-		Environment:   environment,
+		Environment:            environment,
-		SecretPath:    secretPath,
+		SecretPath:             secretPath,
-		IncludeImport: includeImports,
+		IncludeImport:          includeImports,
-		Recursive:     recursive,
+		Recursive:              recursive,
-		TagSlugs:      tagSlugs,
+		TagSlugs:               tagSlugs,
+		ExpandSecretReferences: expandSecretReferences,
 	})
 
 	if err != nil {
@@ -78,17 +77,18 @@ func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment str
 
 }
 
-func GetPlainTextSecretsV3(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool, recursive bool, tagSlugs string) (models.PlaintextSecretResult, error) {
+func GetPlainTextSecretsV3(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool, recursive bool, tagSlugs string, expandSecretReferences bool) (models.PlaintextSecretResult, error) {
 	httpClient := resty.New()
 	httpClient.SetAuthToken(accessToken).
 		SetHeader("Accept", "application/json")
 
 	getSecretsRequest := api.GetRawSecretsV3Request{
-		WorkspaceId:   workspaceId,
+		WorkspaceId:            workspaceId,
-		Environment:   environmentName,
+		Environment:            environmentName,
-		IncludeImport: includeImports,
+		IncludeImport:          includeImports,
-		Recursive:     recursive,
+		Recursive:              recursive,
-		TagSlugs:      tagSlugs,
+		TagSlugs:               tagSlugs,
+		ExpandSecretReferences: expandSecretReferences,
 	}
 
 	if secretsPath != "" {
@@ -284,7 +284,7 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo
 		}
 
 		res, err := GetPlainTextSecretsV3(loggedInUserDetails.UserCredentials.JTWToken, infisicalDotJson.WorkspaceId,
-			params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs)
+			params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, true)
 		log.Debug().Msgf("GetAllEnvironmentVariables: Trying to fetch secrets JTW token [err=%s]", err)
 
 		if err == nil {
@@ -313,7 +313,7 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo
 	} else {
 		if params.InfisicalToken != "" {
 			log.Debug().Msg("Trying to fetch secrets using service token")
-			secretsToReturn, errorToReturn = GetPlainTextSecretsViaServiceToken(params.InfisicalToken, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs)
+			secretsToReturn, errorToReturn = GetPlainTextSecretsViaServiceToken(params.InfisicalToken, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, params.ExpandSecretReferences)
 		} else if params.UniversalAuthAccessToken != "" {
 
 			if params.WorkspaceId == "" {
@@ -321,7 +321,7 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo
 			}
 
 			log.Debug().Msg("Trying to fetch secrets using universal auth")
-			res, err := GetPlainTextSecretsV3(params.UniversalAuthAccessToken, params.WorkspaceId, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs)
+			res, err := GetPlainTextSecretsV3(params.UniversalAuthAccessToken, params.WorkspaceId, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive, params.TagSlugs, params.ExpandSecretReferences)
 
 			errorToReturn = err
 			secretsToReturn = res.Secrets
@@ -331,44 +331,6 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo
 	return secretsToReturn, errorToReturn
 }
 
-var secRefRegex = regexp.MustCompile(`\${([^\}]*)}`)
-
-func recursivelyExpandSecret(expandedSecs map[string]string, interpolatedSecs map[string]string, crossSecRefFetch func(env string, path []string, key string) string, key string) string {
-	if v, ok := expandedSecs[key]; ok {
-		return v
-	}
-
-	interpolatedVal, ok := interpolatedSecs[key]
-	if !ok {
-		HandleError(fmt.Errorf("could not find refered secret -  %s", key), "Kindly check whether its provided")
-	}
-
-	refs := secRefRegex.FindAllStringSubmatch(interpolatedVal, -1)
-	for _, val := range refs {
-		// key: "${something}" val: [${something},something]
-		interpolatedExp, interpolationKey := val[0], val[1]
-		ref := strings.Split(interpolationKey, ".")
-
-		// ${KEY1} => [key1]
-		if len(ref) == 1 {
-			val := recursivelyExpandSecret(expandedSecs, interpolatedSecs, crossSecRefFetch, interpolationKey)
-			interpolatedVal = strings.ReplaceAll(interpolatedVal, interpolatedExp, val)
-			continue
-		}
-
-		// cross board reference ${env.folder.key1} => [env folder key1]
-		if len(ref) > 1 {
-			secEnv, tmpSecPath, secKey := ref[0], ref[1:len(ref)-1], ref[len(ref)-1]
-			interpolatedSecs[interpolationKey] = crossSecRefFetch(secEnv, tmpSecPath, secKey) // get the reference value
-			val := recursivelyExpandSecret(expandedSecs, interpolatedSecs, crossSecRefFetch, interpolationKey)
-			interpolatedVal = strings.ReplaceAll(interpolatedVal, interpolatedExp, val)
-		}
-
-	}
-	expandedSecs[key] = interpolatedVal
-	return interpolatedVal
-}
-
 func getSecretsByKeys(secrets []models.SingleEnvironmentVariable) map[string]models.SingleEnvironmentVariable {
 	secretMapByName := make(map[string]models.SingleEnvironmentVariable, len(secrets))
 
@@ -379,70 +341,6 @@ func getSecretsByKeys(secrets []models.SingleEnvironmentVariable) map[string]mod
 	return secretMapByName
 }
 
-func ExpandSecrets(secrets []models.SingleEnvironmentVariable, auth models.ExpandSecretsAuthentication, projectConfigPathDir string) []models.SingleEnvironmentVariable {
-	expandedSecs := make(map[string]string)
-	interpolatedSecs := make(map[string]string)
-	// map[env.secret-path][keyname]Secret
-	crossEnvRefSecs := make(map[string]map[string]models.SingleEnvironmentVariable) // a cache to hold all cross board reference secrets
-
-	for _, sec := range secrets {
-		// get all references in a secret
-		refs := secRefRegex.FindAllStringSubmatch(sec.Value, -1)
-		// nil means its a secret without reference
-		if refs == nil {
-			expandedSecs[sec.Key] = sec.Value // atomic secrets without any interpolation
-		} else {
-			interpolatedSecs[sec.Key] = sec.Value
-		}
-	}
-
-	for i, sec := range secrets {
-		// already present pick that up
-		if expandedVal, ok := expandedSecs[sec.Key]; ok {
-			secrets[i].Value = expandedVal
-			continue
-		}
-
-		expandedVal := recursivelyExpandSecret(expandedSecs, interpolatedSecs, func(env string, secPaths []string, secKey string) string {
-			secPaths = append([]string{"/"}, secPaths...)
-			secPath := path.Join(secPaths...)
-
-			secPathDot := strings.Join(secPaths, ".")
-			uniqKey := fmt.Sprintf("%s.%s", env, secPathDot)
-
-			if crossRefSec, ok := crossEnvRefSecs[uniqKey]; !ok {
-
-				var refSecs []models.SingleEnvironmentVariable
-				var err error
-
-				// if not in cross reference cache, fetch it from server
-				if auth.InfisicalToken != "" {
-					refSecs, err = GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: env, InfisicalToken: auth.InfisicalToken, SecretsPath: secPath}, projectConfigPathDir)
-				} else if auth.UniversalAuthAccessToken != "" {
-					refSecs, err = GetAllEnvironmentVariables((models.GetAllSecretsParameters{Environment: env, UniversalAuthAccessToken: auth.UniversalAuthAccessToken, SecretsPath: secPath, WorkspaceId: sec.WorkspaceId}), projectConfigPathDir)
-				} else if IsLoggedIn() {
-					refSecs, err = GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: env, SecretsPath: secPath}, projectConfigPathDir)
-				} else {
-					HandleError(errors.New("no authentication provided"), "Please provide authentication to fetch secrets")
-				}
-				if err != nil {
-					HandleError(err, fmt.Sprintf("Could not fetch secrets in environment: %s secret-path: %s", env, secPath), "If you are using a service token to fetch secrets, please ensure it is valid")
-				}
-				refSecsByKey := getSecretsByKeys(refSecs)
-				// save it to avoid calling api again for same environment and folder path
-				crossEnvRefSecs[uniqKey] = refSecsByKey
-				return refSecsByKey[secKey].Value
-
-			} else {
-				return crossRefSec[secKey].Value
-			}
-		}, sec.Key)
-
-		secrets[i].Value = expandedVal
-	}
-	return secrets
-}
-
 func OverrideSecrets(secrets []models.SingleEnvironmentVariable, secretType string) []models.SingleEnvironmentVariable {
 	personalSecrets := make(map[string]models.SingleEnvironmentVariable)
 	sharedSecrets := make(map[string]models.SingleEnvironmentVariable)

cli/secret-render-template

@@ -1,4 +1,4 @@
-{{- with secret "ae06a663-2740-49c5-b998-bf4ceca2c120" "dev" "/" `{"recursive":true}` }}
+{{- with secret "8fac9f01-4a81-44d7-8ff0-3d7be684f56f" "staging" "/" `{"recursive":true, "expandSecretReferences": false}` }}
 {{- range . }}
 {{ .Key }}={{ .Value }}
 {{- end }}