Merge pull request #2254 from akhilmhdh/fix/backup

Resolved keyring dataset too big by keeping only the encryption key

cli/packages/cmd/vault.go

@@ -31,37 +31,52 @@ var AvailableVaults = []VaultBackendType{
 }
 
 var vaultSetCmd = &cobra.Command{
-	Example:               `infisical vault set file --passphrase <your-passphrase>`,
+	Example:               `infisical vault set file`,
-	Use:                   "set [file|auto] [flags]",
+	Use:                   "set [file|auto]",
 	Short:                 "Used to configure the vault backends",
 	DisableFlagsInUseLine: true,
 	Args:                  cobra.MinimumNArgs(1),
 	Run: func(cmd *cobra.Command, args []string) {
-
+		wantedVaultTypeName := args[0]
-		vaultType := args[0]
+		currentVaultBackend, err := util.GetCurrentVaultBackend()
-
-		passphrase, err := cmd.Flags().GetString("passphrase")
 		if err != nil {
-			util.HandleError(err, "Unable to get passphrase flag")
+			log.Error().Msgf("Unable to set vault to [%s] because of [err=%s]", wantedVaultTypeName, err)
-		}
-
-		if vaultType == util.VAULT_BACKEND_FILE_MODE && passphrase != "" {
-			setFileVaultPassphrase(passphrase)
 			return
 		}
 
-		util.PrintWarning("This command has been deprecated. Please use 'infisical vault use [file|auto]' to select which vault to use.\n")
+		if wantedVaultTypeName == string(currentVaultBackend) {
-		selectVaultTypeCmd(cmd, args)
+			log.Error().Msgf("You are already on vault backend [%s]", currentVaultBackend)
-	},
+			return
-}
+		}
 
-var vaultUseCmd = &cobra.Command{
+		if wantedVaultTypeName == util.VAULT_BACKEND_AUTO_MODE || wantedVaultTypeName == util.VAULT_BACKEND_FILE_MODE {
-	Example:               `infisical vault use [file|auto]`,
+			configFile, err := util.GetConfigFile()
-	Use:                   "use [file|auto]",
+			if err != nil {
-	Short:                 "Used to select the the type of vault backend to store sensitive data securely at rest",
+				log.Error().Msgf("Unable to set vault to [%s] because of [err=%s]", wantedVaultTypeName, err)
-	DisableFlagsInUseLine: true,
+				return
-	Args:                  cobra.MinimumNArgs(1),
+			}
-	Run:                   selectVaultTypeCmd,
+
+			configFile.VaultBackendType = wantedVaultTypeName
+			configFile.LoggedInUserEmail = ""
+			configFile.VaultBackendPassphrase = base64.StdEncoding.EncodeToString([]byte(util.GenerateRandomString(10)))
+
+			err = util.WriteConfigFile(&configFile)
+			if err != nil {
+				log.Error().Msgf("Unable to set vault to [%s] because an error occurred when saving the config file [err=%s]", wantedVaultTypeName, err)
+				return
+			}
+
+			fmt.Printf("\nSuccessfully, switched vault backend from [%s] to [%s]. Please login in again to store your login details in the new vault with [infisical login]\n", currentVaultBackend, wantedVaultTypeName)
+
+			Telemetry.CaptureEvent("cli-command:vault set", posthog.NewProperties().Set("currentVault", currentVaultBackend).Set("wantedVault", wantedVaultTypeName).Set("version", util.CLI_VERSION))
+		} else {
+			var availableVaultsNames []string
+			for _, vault := range AvailableVaults {
+				availableVaultsNames = append(availableVaultsNames, vault.Name)
+			}
+			log.Error().Msgf("The requested vault type [%s] is not available on this system. Only the following vault backends are available for you system: %s", wantedVaultTypeName, strings.Join(availableVaultsNames, ", "))
+		}
+	},
 }
 
 // runCmd represents the run command
@@ -75,26 +90,6 @@ var vaultCmd = &cobra.Command{
 	},
 }
 
-func setFileVaultPassphrase(passphrase string) {
-	configFile, err := util.GetConfigFile()
-	if err != nil {
-		log.Error().Msgf("Unable to set passphrase for file vault because of [err=%s]", err)
-		return
-	}
-
-	// encode with base64
-	encodedPassphrase := base64.StdEncoding.EncodeToString([]byte(passphrase))
-	configFile.VaultBackendPassphrase = encodedPassphrase
-
-	err = util.WriteConfigFile(&configFile)
-	if err != nil {
-		log.Error().Msgf("Unable to set passphrase for file vault because of [err=%s]", err)
-		return
-	}
-
-	util.PrintSuccessMessage("\nSuccessfully, set passphrase for file vault.\n")
-}
-
 func printAvailableVaultBackends() {
 	fmt.Printf("Vaults are used to securely store your login details locally. Available vaults:")
 	for _, vaultType := range AvailableVaults {
@@ -111,53 +106,8 @@ func printAvailableVaultBackends() {
 	fmt.Printf("\n\nYou are currently using [%s] vault to store your login credentials\n", string(currentVaultBackend))
 }
 
-func selectVaultTypeCmd(cmd *cobra.Command, args []string) {
-	wantedVaultTypeName := args[0]
-	currentVaultBackend, err := util.GetCurrentVaultBackend()
-	if err != nil {
-		log.Error().Msgf("Unable to set vault to [%s] because of [err=%s]", wantedVaultTypeName, err)
-		return
-	}
-
-	if wantedVaultTypeName == string(currentVaultBackend) {
-		log.Error().Msgf("You are already on vault backend [%s]", currentVaultBackend)
-		return
-	}
-
-	if wantedVaultTypeName == util.VAULT_BACKEND_AUTO_MODE || wantedVaultTypeName == util.VAULT_BACKEND_FILE_MODE {
-		configFile, err := util.GetConfigFile()
-		if err != nil {
-			log.Error().Msgf("Unable to set vault to [%s] because of [err=%s]", wantedVaultTypeName, err)
-			return
-		}
-
-		configFile.VaultBackendType = wantedVaultTypeName // save selected vault
-		configFile.LoggedInUserEmail = ""                 // reset the logged in user to prompt them to re login
-
-		err = util.WriteConfigFile(&configFile)
-		if err != nil {
-			log.Error().Msgf("Unable to set vault to [%s] because an error occurred when saving the config file [err=%s]", wantedVaultTypeName, err)
-			return
-		}
-
-		fmt.Printf("\nSuccessfully, switched vault backend from [%s] to [%s]. Please login in again to store your login details in the new vault with [infisical login]\n", currentVaultBackend, wantedVaultTypeName)
-
-		Telemetry.CaptureEvent("cli-command:vault set", posthog.NewProperties().Set("currentVault", currentVaultBackend).Set("wantedVault", wantedVaultTypeName).Set("version", util.CLI_VERSION))
-	} else {
-		var availableVaultsNames []string
-		for _, vault := range AvailableVaults {
-			availableVaultsNames = append(availableVaultsNames, vault.Name)
-		}
-		log.Error().Msgf("The requested vault type [%s] is not available on this system. Only the following vault backends are available for you system: %s", wantedVaultTypeName, strings.Join(availableVaultsNames, ", "))
-	}
-}
-
 func init() {
-
-	vaultSetCmd.Flags().StringP("passphrase", "p", "", "Set the passphrase for the file vault")
-
 	vaultCmd.AddCommand(vaultSetCmd)
-	vaultCmd.AddCommand(vaultUseCmd)
 
 	rootCmd.AddCommand(vaultCmd)
 }

cli/packages/util/constants.go

@@ -38,7 +38,8 @@ const (
 	SERVICE_TOKEN_IDENTIFIER        = "service-token"
 	UNIVERSAL_AUTH_TOKEN_IDENTIFIER = "universal-auth-token"
 
-	INFISICAL_BACKUP_SECRET = "infisical-backup-secrets"
+	INFISICAL_BACKUP_SECRET                = "infisical-backup-secrets" // akhilmhdh: @depreciated remove in version v0.30
+	INFISICAL_BACKUP_SECRET_ENCRYPTION_KEY = "infisical-backup-secret-encryption-key"
 )
 
 var (

cli/packages/util/helper.go

@@ -5,6 +5,7 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"fmt"
+	"math/rand"
 	"os"
 	"os/exec"
 	"path"
@@ -25,6 +26,8 @@ type DecodedSymmetricEncryptionDetails = struct {
 	Key    []byte
 }
 
+const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
 func GetBase64DecodedSymmetricEncryptionDetails(key string, cipher string, IV string, tag string) (DecodedSymmetricEncryptionDetails, error) {
 	cipherx, err := base64.StdEncoding.DecodeString(cipher)
 	if err != nil {
@@ -287,3 +290,11 @@ func GetCmdFlagOrEnv(cmd *cobra.Command, flag, envName string) (string, error) {
 	}
 	return value, nil
 }
+
+func GenerateRandomString(length int) string {
+	b := make([]byte, length)
+	for i := range b {
+		b[i] = charset[rand.Intn(len(charset))]
+	}
+	return string(b)
+}

cli/packages/util/keyringwrapper.go

@@ -4,7 +4,6 @@ import (
 	"encoding/base64"
 	"fmt"
 
-	"github.com/manifoldco/promptui"
 	"github.com/rs/zerolog/log"
 	"github.com/zalando/go-keyring"
 )
@@ -32,17 +31,9 @@ func SetValueInKeyring(key, value string) error {
 		configFile, _ := GetConfigFile()
 
 		if configFile.VaultBackendPassphrase == "" {
-			PrintWarning("System keyring could not be used, falling back to `file` vault for sensitive data storage.")
+			encodedPassphrase := base64.StdEncoding.EncodeToString([]byte(GenerateRandomString(10))) // generate random passphrase
-			passphrasePrompt := promptui.Prompt{
-				Label: "Enter the passphrase to use for keyring encryption",
-			}
-			passphrase, err := passphrasePrompt.Run()
-			if err != nil {
-				return err
-			}
-
-			encodedPassphrase := base64.StdEncoding.EncodeToString([]byte(passphrase))
 			configFile.VaultBackendPassphrase = encodedPassphrase
+			configFile.VaultBackendType = VAULT_BACKEND_FILE_MODE
 			err = WriteConfigFile(&configFile)
 			if err != nil {
 				return err
@@ -64,13 +55,7 @@ func GetValueInKeyring(key string) (string, error) {
 	if err != nil {
 		PrintErrorAndExit(1, err, "Unable to get current vault. Tip: run [infisical reset] then try again")
 	}
-
+	return keyring.Get(currentVaultBackend, MAIN_KEYRING_SERVICE, key)
-	value, err := keyring.Get(currentVaultBackend, MAIN_KEYRING_SERVICE, key)
-
-	if err != nil {
-		value, err = keyring.Get(VAULT_BACKEND_FILE_MODE, MAIN_KEYRING_SERVICE, key)
-	}
-	return value, err
 
 }
 
@@ -80,11 +65,5 @@ func DeleteValueInKeyring(key string) error {
 		return err
 	}
 
-	err = keyring.Delete(currentVaultBackend, MAIN_KEYRING_SERVICE, key)
+	return keyring.Delete(currentVaultBackend, MAIN_KEYRING_SERVICE, key)
-
-	if err != nil {
-		err = keyring.Delete(VAULT_BACKEND_FILE_MODE, MAIN_KEYRING_SERVICE, key)
-	}
-
-	return err
 }

cli/packages/util/secrets.go

@@ -1,14 +1,15 @@
 package util
 
 import (
+	"crypto/rand"
 	"encoding/base64"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
 	"path"
 	"regexp"
-	"slices"
 	"strings"
 	"unicode"
 
@@ -285,18 +286,25 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo
 		log.Debug().Msgf("GetAllEnvironmentVariables: Trying to fetch secrets JTW token [err=%s]", err)
 
 		if err == nil {
-			WriteBackupSecrets(infisicalDotJson.WorkspaceId, params.Environment, params.SecretsPath, res.Secrets)
+			backupEncryptionKey, err := GetBackupEncryptionKey()
+			if err != nil {
+				return nil, err
+			}
+			WriteBackupSecrets(infisicalDotJson.WorkspaceId, params.Environment, params.SecretsPath, backupEncryptionKey, res.Secrets)
 		}
 
 		secretsToReturn = res.Secrets
 		errorToReturn = err
 		// only attempt to serve cached secrets if no internet connection and if at least one secret cached
 		if !isConnected {
-			backedSecrets, err := ReadBackupSecrets(infisicalDotJson.WorkspaceId, params.Environment, params.SecretsPath)
+			backupEncryptionKey, _ := GetBackupEncryptionKey()
-			if len(backedSecrets) > 0 {
+			if backupEncryptionKey != nil {
-				PrintWarning("Unable to fetch latest secret(s) due to connection error, serving secrets from last successful fetch. For more info, run with --debug")
+				backedUpSecrets, err := ReadBackupSecrets(infisicalDotJson.WorkspaceId, params.Environment, params.SecretsPath, backupEncryptionKey)
-				secretsToReturn = backedSecrets
+				if len(backedUpSecrets) > 0 {
-				errorToReturn = err
+					PrintWarning("Unable to fetch the latest secret(s) due to connection error, serving secrets from last successful fetch. For more info, run with --debug")
+					secretsToReturn = backedUpSecrets
+					errorToReturn = err
+				}
 			}
 		}
 
@@ -476,71 +484,99 @@ func OverrideSecrets(secrets []models.SingleEnvironmentVariable, secretType stri
 	return secretsToReturn
 }
 
-func WriteBackupSecrets(workspace string, environment string, secretsPath string, secrets []models.SingleEnvironmentVariable) error {
+func GetBackupEncryptionKey() ([]byte, error) {
-	var backedUpSecrets []models.BackupSecretKeyRing
+	encryptionKey, err := GetValueInKeyring(INFISICAL_BACKUP_SECRET_ENCRYPTION_KEY)
-	secretValueInKeyRing, err := GetValueInKeyring(INFISICAL_BACKUP_SECRET)
 	if err != nil {
 		if err == keyring.ErrUnsupportedPlatform {
-			return errors.New("your OS does not support keyring. Consider using a service token https://infisical.com/docs/documentation/platform/token")
+			return nil, errors.New("your OS does not support keyring. Consider using a service token https://infisical.com/docs/documentation/platform/token")
-		} else if err != keyring.ErrNotFound {
+		} else if err == keyring.ErrNotFound {
-			return fmt.Errorf("something went wrong, failed to retrieve value from system keyring [error=%v]", err)
+			// generate a new key
+			randomizedKey := make([]byte, 16)
+			rand.Read(randomizedKey)
+			encryptionKey = hex.EncodeToString(randomizedKey)
+			if err := SetValueInKeyring(INFISICAL_BACKUP_SECRET_ENCRYPTION_KEY, encryptionKey); err != nil {
+				return nil, err
+			}
+			return []byte(encryptionKey), nil
+		} else {
+			return nil, fmt.Errorf("something went wrong, failed to retrieve value from system keyring [error=%v]", err)
 		}
 	}
-	_ = json.Unmarshal([]byte(secretValueInKeyRing), &backedUpSecrets)
+	return []byte(encryptionKey), nil
+}
 
-	backedUpSecrets = slices.DeleteFunc(backedUpSecrets, func(e models.BackupSecretKeyRing) bool {
+func WriteBackupSecrets(workspace string, environment string, secretsPath string, encryptionKey []byte, secrets []models.SingleEnvironmentVariable) error {
-		return e.SecretPath == secretsPath && e.ProjectID == workspace && e.Environment == environment
+	formattedPath := strings.ReplaceAll(secretsPath, "/", "-")
-	})
+	fileName := fmt.Sprintf("project_secrets_%s_%s_%s.json", workspace, environment, formattedPath)
-	newBackupSecret := models.BackupSecretKeyRing{
+	secrets_backup_folder_name := "secrets-backup"
-		ProjectID:   workspace,
-		Environment: environment,
-		SecretPath:  secretsPath,
-		Secrets:     secrets,
-	}
-	backedUpSecrets = append(backedUpSecrets, newBackupSecret)
 
-	listOfSecretsMarshalled, err := json.Marshal(backedUpSecrets)
+	_, fullConfigFileDirPath, err := GetFullConfigFilePath()
 	if err != nil {
-		return err
+		return fmt.Errorf("WriteBackupSecrets: unable to get full config folder path [err=%s]", err)
 	}
 
-	err = SetValueInKeyring(INFISICAL_BACKUP_SECRET, string(listOfSecretsMarshalled))
+	// create secrets backup directory
+	fullPathToSecretsBackupFolder := fmt.Sprintf("%s/%s", fullConfigFileDirPath, secrets_backup_folder_name)
+	if _, err := os.Stat(fullPathToSecretsBackupFolder); errors.Is(err, os.ErrNotExist) {
+		err := os.Mkdir(fullPathToSecretsBackupFolder, os.ModePerm)
+		if err != nil {
+			return err
+		}
+	}
+	marshaledSecrets, _ := json.Marshal(secrets)
+	result, err := crypto.EncryptSymmetric(marshaledSecrets, encryptionKey)
 	if err != nil {
-		return fmt.Errorf("StoreUserCredsInKeyRing: unable to store user credentials because [err=%s]", err)
+		return fmt.Errorf("WriteBackupSecrets: Unable to encrypt local secret backup to file [err=%s]", err)
+	}
+	listOfSecretsMarshalled, _ := json.Marshal(result)
+	err = os.WriteFile(fmt.Sprintf("%s/%s", fullPathToSecretsBackupFolder, fileName), listOfSecretsMarshalled, 0600)
+	if err != nil {
+		return fmt.Errorf("WriteBackupSecrets: Unable to write backup secrets to file [err=%s]", err)
 	}
 
 	return nil
 }
 
-func ReadBackupSecrets(workspace string, environment string, secretsPath string) ([]models.SingleEnvironmentVariable, error) {
+func ReadBackupSecrets(workspace string, environment string, secretsPath string, encryptionKey []byte) ([]models.SingleEnvironmentVariable, error) {
-	secretValueInKeyRing, err := GetValueInKeyring(INFISICAL_BACKUP_SECRET)
+	formattedPath := strings.ReplaceAll(secretsPath, "/", "-")
+	fileName := fmt.Sprintf("project_secrets_%s_%s_%s.json", workspace, environment, formattedPath)
+	secrets_backup_folder_name := "secrets-backup"
+
+	_, fullConfigFileDirPath, err := GetFullConfigFilePath()
 	if err != nil {
-		if err == keyring.ErrUnsupportedPlatform {
+		return nil, fmt.Errorf("ReadBackupSecrets: unable to write config file because an error occurred when getting config file path [err=%s]", err)
-			return nil, errors.New("your OS does not support keyring. Consider using a service token https://infisical.com/docs/documentation/platform/token")
-		} else if err == keyring.ErrNotFound {
-			return nil, errors.New("credentials not found in system keyring")
-		} else {
-			return nil, fmt.Errorf("something went wrong, failed to retrieve value from system keyring [error=%v]", err)
-		}
 	}
 
-	var backedUpSecrets []models.BackupSecretKeyRing
+	fullPathToSecretsBackupFolder := fmt.Sprintf("%s/%s", fullConfigFileDirPath, secrets_backup_folder_name)
-	err = json.Unmarshal([]byte(secretValueInKeyRing), &backedUpSecrets)
+	if _, err := os.Stat(fullPathToSecretsBackupFolder); errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+
+	encryptedBackupSecretsFilePath := fmt.Sprintf("%s/%s", fullPathToSecretsBackupFolder, fileName)
+
+	encryptedBackupSecretsAsBytes, err := os.ReadFile(encryptedBackupSecretsFilePath)
 	if err != nil {
-		return nil, fmt.Errorf("getUserCredsFromKeyRing: Something went wrong when unmarshalling user creds [err=%s]", err)
+		return nil, err
 	}
 
-	for _, backupSecret := range backedUpSecrets {
+	var encryptedBackUpSecrets models.SymmetricEncryptionResult
-		if backupSecret.Environment == environment && backupSecret.ProjectID == workspace && backupSecret.SecretPath == secretsPath {
+	err = json.Unmarshal(encryptedBackupSecretsAsBytes, &encryptedBackUpSecrets)
-			return backupSecret.Secrets, nil
+	if err != nil {
-		}
+		return nil, fmt.Errorf("ReadBackupSecrets: unable to parse encrypted backup secrets. The secrets backup may be malformed [err=%s]", err)
 	}
 
-	return nil, nil
+	result, err := crypto.DecryptSymmetric(encryptionKey, encryptedBackUpSecrets.CipherText, encryptedBackUpSecrets.AuthTag, encryptedBackUpSecrets.Nonce)
+	if err != nil {
+		return nil, fmt.Errorf("ReadBackupSecrets: unable to decrypt encrypted backup secrets [err=%s]", err)
+	}
+	var plainTextSecrets []models.SingleEnvironmentVariable
+	_ = json.Unmarshal(result, &plainTextSecrets)
+
+	return plainTextSecrets, nil
+
 }
 
 func DeleteBackupSecrets() error {
-	// keeping this logic for now. Need to remove it later as more users migrate keyring would be used and this folder will be removed completely by then
 	secrets_backup_folder_name := "secrets-backup"
 
 	_, fullConfigFileDirPath, err := GetFullConfigFilePath()
@@ -549,8 +585,8 @@ func DeleteBackupSecrets() error {
 	}
 
 	fullPathToSecretsBackupFolder := fmt.Sprintf("%s/%s", fullConfigFileDirPath, secrets_backup_folder_name)
-
 	DeleteValueInKeyring(INFISICAL_BACKUP_SECRET)
+	DeleteValueInKeyring(INFISICAL_BACKUP_SECRET_ENCRYPTION_KEY)
 
 	return os.RemoveAll(fullPathToSecretsBackupFolder)
 }

docs/cli/commands/vault.mdx

@@ -30,8 +30,5 @@ description: "Change the vault type in Infisical"
 
 ## Description
 
-To safeguard your login details when using the CLI, Infisical places them in a system vault or an encrypted text file, protected by a passphrase that only the user knows.
+To safeguard your login details when using the CLI, Infisical attempts to store them in a system keyring. If a system keyring cannot be found on your machine, the data is stored in a config file.
-
-<Tip>To avoid constantly entering your passphrase when using the `file` vault type, use the `infisical vault set file --passphrase <your-passphrase>` CLI command to specify your password once.</Tip>
-