feat(secret-ref): implemented cli changes for secret reference

cli/packages/cmd/export.go

@@ -83,7 +83,7 @@ var exportCmd = &cobra.Command{
 
 		var output string
 		if shouldExpandSecrets {
-			substitutions := util.SubstituteSecrets(secrets)
+			substitutions := util.ExpandSecrets(secrets, infisicalToken)
 			output, err = formatEnvs(substitutions, format)
 			if err != nil {
 				util.HandleError(err)

cli/packages/cmd/run.go

@@ -100,7 +100,7 @@ var runCmd = &cobra.Command{
 		}
 
 		if shouldExpandSecrets {
-			secrets = util.SubstituteSecrets(secrets)
+			secrets = util.ExpandSecrets(secrets, infisicalToken)
 		}
 
 		secretsByKey := getSecretsByKeys(secrets)

cli/packages/cmd/secrets.go

@@ -65,7 +65,7 @@ var secretsCmd = &cobra.Command{
 		}
 
 		if shouldExpandSecrets {
-			secrets = util.SubstituteSecrets(secrets)
+			secrets = util.ExpandSecrets(secrets, infisicalToken)
 		}
 
 		visualize.PrintAllSecretDetails(secrets)

cli/packages/util/log.go

@@ -45,5 +45,5 @@ func PrintErrorMessageAndExit(messages ...string) {
 }
 
 func printError(e error) {
-	color.New(color.FgRed).Fprintf(os.Stderr, "Hmm, we ran into an error: %v", e)
+	color.New(color.FgRed).Fprintf(os.Stderr, "Hmm, we ran into an error: %v\n", e)
 }

cli/packages/util/secrets.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path"
 	"regexp"
 	"strings"
 
@@ -279,22 +280,100 @@ func getExpandedEnvVariable(secrets []models.SingleEnvironmentVariable, variable
 	return "${" + variableWeAreLookingFor + "}"
 }
 
-func SubstituteSecrets(secrets []models.SingleEnvironmentVariable) []models.SingleEnvironmentVariable {
+var secRefRegex = regexp.MustCompile(`\${([^\}]*)}`)
-	hashMapOfCompleteVariables := make(map[string]string)
-	hashMapOfSelfRefs := make(map[string]string)
-	expandedSecrets := []models.SingleEnvironmentVariable{}
-
-	for _, secret := range secrets {
-		expandedVariable := getExpandedEnvVariable(secrets, secret.Key, hashMapOfCompleteVariables, hashMapOfSelfRefs)
-		expandedSecrets = append(expandedSecrets, models.SingleEnvironmentVariable{
-			Key:   secret.Key,
-			Value: expandedVariable,
-			Type:  secret.Type,
-		})
 
+func recursivelyExpandSecret(expandedSecs map[string]string, interpolatedSecs map[string]string, crossSecRefFetch func(env string, path []string, key string) string, key string) string {
+	if v, ok := expandedSecs[key]; ok {
+		return v
 	}
 
-	return expandedSecrets
+	interpolatedVal := interpolatedSecs[key]
+	refs := secRefRegex.FindAllStringSubmatch(interpolatedVal, -1)
+	for _, val := range refs {
+		// key: "${something}" val: [${something},something]
+		interpolatedExp, interpolationKey := val[0], val[1]
+		ref := strings.Split(interpolationKey, ".")
+
+		// ${KEY1} => [key1]
+		if len(ref) == 1 {
+			val := recursivelyExpandSecret(expandedSecs, interpolatedSecs, crossSecRefFetch, interpolationKey)
+			interpolatedVal = strings.ReplaceAll(interpolatedVal, interpolatedExp, val)
+			continue
+		}
+
+		// cross board reference ${env.folder.key1} => [env folder key1]
+		if len(ref) > 1 {
+			secEnv, tmpSecPath, secKey := ref[0], ref[1:len(ref)-1], ref[len(ref)-1]
+			interpolatedSecs[interpolationKey] = crossSecRefFetch(secEnv, tmpSecPath, secKey) // get the reference value
+			val := recursivelyExpandSecret(expandedSecs, interpolatedSecs, crossSecRefFetch, interpolationKey)
+			interpolatedVal = strings.ReplaceAll(interpolatedVal, interpolatedExp, val)
+		}
+
+	}
+	expandedSecs[key] = interpolatedVal
+	return interpolatedVal
+}
+
+func getSecretsByKeys(secrets []models.SingleEnvironmentVariable) map[string]models.SingleEnvironmentVariable {
+	secretMapByName := make(map[string]models.SingleEnvironmentVariable, len(secrets))
+
+	for _, secret := range secrets {
+		secretMapByName[secret.Key] = secret
+	}
+
+	return secretMapByName
+}
+
+func ExpandSecrets(secrets []models.SingleEnvironmentVariable, infisicalToken string) []models.SingleEnvironmentVariable {
+	expandedSecs := make(map[string]string)
+	interpolatedSecs := make(map[string]string)
+	// map[env.secret-path][keyname]Secret
+	crossEnvRefSecs := make(map[string]map[string]models.SingleEnvironmentVariable) // a cache to hold all cross board reference secrets
+
+	for _, sec := range secrets {
+		// get all references in a secret
+		refs := secRefRegex.FindAllStringSubmatch(sec.Value, -1)
+		// nil means its a secret without reference
+		if refs == nil {
+			expandedSecs[sec.Key] = sec.Value // atomic secrets without any interpolation
+		} else {
+			interpolatedSecs[sec.Key] = sec.Value
+		}
+	}
+
+	for i, sec := range secrets {
+		// already present pick that up
+		if expandedVal, ok := expandedSecs[sec.Key]; ok {
+			secrets[i].Value = expandedVal
+			continue
+		}
+
+		expandedVal := recursivelyExpandSecret(expandedSecs, interpolatedSecs, func(env string, secPaths []string, secKey string) string {
+			secPath := path.Join(secPaths...)
+			if secPath == "" {
+				secPath = "/"
+			}
+			secPathDot := strings.Join(secPaths, ".")
+			uniqKey := fmt.Sprintf("%s.%s", env, secPathDot)
+
+			if crossRefSec, ok := crossEnvRefSecs[uniqKey]; !ok {
+				// if not in cross reference cache, fetch it from server
+				refSecs, err := GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: env, InfisicalToken: infisicalToken, SecretsPath: secPath})
+				if err != nil {
+					HandleError(err, fmt.Sprintf("Could not fetch secrets in environment: %s secret-path: %s", env, secPath), "If you are using a service token to fetch secrets, please ensure it is valid")
+				}
+				refSecsByKey := getSecretsByKeys(refSecs)
+				// save it to avoid calling api again for same environment and folder path
+				crossEnvRefSecs[uniqKey] = refSecsByKey
+				return refSecsByKey[secKey].Value
+			} else {
+				return crossRefSec[secKey].Value
+			}
+		}, sec.Key)
+
+		secrets[i].Value = expandedVal
+	}
+	return secrets
 }
 
 func OverrideSecrets(secrets []models.SingleEnvironmentVariable, secretType string) []models.SingleEnvironmentVariable {