feat(k8-operator): multiple managed secrets

This commit is contained in:
Daniel Hougaard
2025-01-14 16:31:31 +01:00
parent c3970d1ea2
commit f358e8942d
147 changed files with 3591 additions and 3573 deletions

View File

@@ -1,108 +0,0 @@
package controllers
import (
"context"
"fmt"
"sync"
"github.com/Infisical/infisical/k8-operator/api/v1alpha1"
"github.com/Infisical/infisical/k8-operator/packages/constants"
"github.com/go-logr/logr"
v1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)
const DEPLOYMENT_SECRET_NAME_ANNOTATION_PREFIX = "secrets.infisical.com/managed-secret"
const AUTO_RELOAD_DEPLOYMENT_ANNOTATION = "secrets.infisical.com/auto-reload" // needs to be set to true for a deployment to start auto redeploying
func (r *InfisicalSecretReconciler) ReconcileDeploymentsWithManagedSecrets(ctx context.Context, logger logr.Logger, infisicalSecret v1alpha1.InfisicalSecret) (int, error) {
listOfDeployments := &v1.DeploymentList{}
err := r.Client.List(ctx, listOfDeployments, &client.ListOptions{Namespace: infisicalSecret.Spec.ManagedSecretReference.SecretNamespace})
if err != nil {
return 0, fmt.Errorf("unable to get deployments in the [namespace=%v] [err=%v]", infisicalSecret.Spec.ManagedSecretReference.SecretNamespace, err)
}
managedKubeSecretNameAndNamespace := types.NamespacedName{
Namespace: infisicalSecret.Spec.ManagedSecretReference.SecretNamespace,
Name: infisicalSecret.Spec.ManagedSecretReference.SecretName,
}
managedKubeSecret := &corev1.Secret{}
err = r.Client.Get(ctx, managedKubeSecretNameAndNamespace, managedKubeSecret)
if err != nil {
return 0, fmt.Errorf("unable to fetch Kubernetes secret to update deployment: %v", err)
}
var wg sync.WaitGroup
// Iterate over the deployments and check if they use the managed secret
for _, deployment := range listOfDeployments.Items {
deployment := deployment
if deployment.Annotations[AUTO_RELOAD_DEPLOYMENT_ANNOTATION] == "true" && r.IsDeploymentUsingManagedSecret(deployment, infisicalSecret) {
// Start a goroutine to reconcile the deployment
wg.Add(1)
go func(d v1.Deployment, s corev1.Secret) {
defer wg.Done()
if err := r.ReconcileDeployment(ctx, logger, d, s); err != nil {
logger.Error(err, fmt.Sprintf("unable to reconcile deployment with [name=%v]. Will try next requeue", deployment.ObjectMeta.Name))
}
}(deployment, *managedKubeSecret)
}
}
wg.Wait()
return 0, nil
}
// Check if the deployment uses managed secrets
func (r *InfisicalSecretReconciler) IsDeploymentUsingManagedSecret(deployment v1.Deployment, infisicalSecret v1alpha1.InfisicalSecret) bool {
managedSecretName := infisicalSecret.Spec.ManagedSecretReference.SecretName
for _, container := range deployment.Spec.Template.Spec.Containers {
for _, envFrom := range container.EnvFrom {
if envFrom.SecretRef != nil && envFrom.SecretRef.LocalObjectReference.Name == managedSecretName {
return true
}
}
for _, env := range container.Env {
if env.ValueFrom != nil && env.ValueFrom.SecretKeyRef != nil && env.ValueFrom.SecretKeyRef.LocalObjectReference.Name == managedSecretName {
return true
}
}
}
for _, volume := range deployment.Spec.Template.Spec.Volumes {
if volume.Secret != nil && volume.Secret.SecretName == managedSecretName {
return true
}
}
return false
}
// This function ensures that a deployment is in sync with a Kubernetes secret by comparing their versions.
// If the version of the secret is different from the version annotation on the deployment, the annotation is updated to trigger a restart of the deployment.
func (r *InfisicalSecretReconciler) ReconcileDeployment(ctx context.Context, logger logr.Logger, deployment v1.Deployment, secret corev1.Secret) error {
annotationKey := fmt.Sprintf("%s.%s", DEPLOYMENT_SECRET_NAME_ANNOTATION_PREFIX, secret.Name)
annotationValue := secret.Annotations[constants.SECRET_VERSION_ANNOTATION]
if deployment.Annotations[annotationKey] == annotationValue &&
deployment.Spec.Template.Annotations[annotationKey] == annotationValue {
logger.Info(fmt.Sprintf("The [deploymentName=%v] is already using the most up to date managed secrets. No action required.", deployment.ObjectMeta.Name))
return nil
}
logger.Info(fmt.Sprintf("Deployment is using outdated managed secret. Starting re-deployment [deploymentName=%v]", deployment.ObjectMeta.Name))
if deployment.Spec.Template.Annotations == nil {
deployment.Spec.Template.Annotations = make(map[string]string)
}
deployment.Annotations[annotationKey] = annotationValue
deployment.Spec.Template.Annotations[annotationKey] = annotationValue
if err := r.Client.Update(ctx, &deployment); err != nil {
return fmt.Errorf("failed to update deployment annotation: %v", err)
}
return nil
}

View File

@@ -13,6 +13,8 @@ import (
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/predicate"
defaultErrors "errors"
secretsv1alpha1 "github.com/Infisical/infisical/k8-operator/api/v1alpha1"
"github.com/Infisical/infisical/k8-operator/packages/api"
controllerhelpers "github.com/Infisical/infisical/k8-operator/packages/controllerhelpers"
@@ -35,8 +37,6 @@ func (r *InfisicalSecretReconciler) GetLogger(req ctrl.Request) logr.Logger {
return r.BaseLogger.WithValues("infisicalsecret", req.NamespacedName)
}
var resourceVariablesMap map[string]util.ResourceVariables = make(map[string]util.ResourceVariables)
//+kubebuilder:rbac:groups=secrets.infisical.com,resources=infisicalsecrets,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=secrets.infisical.com,resources=infisicalsecrets/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=secrets.infisical.com,resources=infisicalsecrets/finalizers,verbs=update
@@ -71,6 +71,24 @@ func (r *InfisicalSecretReconciler) Reconcile(ctx context.Context, req ctrl.Requ
}
}
// It's important we don't directly modify the CRD object, so we create a copy of it and move existing data into it.
managedSecretReferences := infisicalSecretCRD.Spec.ManagedSecretReferences
if infisicalSecretCRD.Spec.ManagedSecretReference.SecretName != "" && managedSecretReferences != nil && len(managedSecretReferences) > 0 {
errMessage := "InfisicalSecret CRD cannot have both managedSecretReference and managedSecretReferences"
logger.Error(defaultErrors.New(errMessage), errMessage)
return ctrl.Result{}, defaultErrors.New(errMessage)
}
if infisicalSecretCRD.Spec.ManagedSecretReference.SecretName != "" {
logger.Info("\n\n\nThe field `managedSecretReference` will be deprecated in the near future, please use `managedSecretReferences` instead.\n\nRefer to the documentation for more information: https://infisical.com/docs/integrations/platforms/kubernetes/infisical-secret-crd\n\n\n")
if managedSecretReferences == nil {
managedSecretReferences = []secretsv1alpha1.ManagedKubeSecretConfig{}
}
managedSecretReferences = append(managedSecretReferences, infisicalSecretCRD.Spec.ManagedSecretReference)
}
// Remove finalizers if they exist. This is to support previous InfisicalSecret CRD's that have finalizers on them.
// In order to delete secrets with finalizers, we first remove the finalizers so we can use the simplified and improved deletion process
if !infisicalSecretCRD.ObjectMeta.DeletionTimestamp.IsZero() && len(infisicalSecretCRD.ObjectMeta.Finalizers) > 0 {
@@ -127,7 +145,7 @@ func (r *InfisicalSecretReconciler) Reconcile(ctx context.Context, req ctrl.Requ
api.API_CA_CERTIFICATE = ""
}
err = r.ReconcileInfisicalSecret(ctx, logger, infisicalSecretCRD)
err = r.ReconcileInfisicalSecret(ctx, logger, infisicalSecretCRD, managedSecretReferences)
r.SetReadyToSyncSecretsConditions(ctx, &infisicalSecretCRD, err)
if err != nil {
@@ -138,7 +156,7 @@ func (r *InfisicalSecretReconciler) Reconcile(ctx context.Context, req ctrl.Requ
}, nil
}
numDeployments, err := controllerhelpers.ReconcileDeploymentsWithManagedSecrets(ctx, r.Client, logger, infisicalSecretCRD.Spec.ManagedSecretReference)
numDeployments, err := controllerhelpers.ReconcileDeploymentsWithMultipleManagedSecrets(ctx, r.Client, logger, managedSecretReferences)
r.SetInfisicalAutoRedeploymentReady(ctx, logger, &infisicalSecretCRD, numDeployments, err)
if err != nil {
logger.Error(err, fmt.Sprintf("unable to reconcile auto redeployment. Will requeue after [requeueTime=%v]", requeueTime))

View File

@@ -165,10 +165,10 @@ var infisicalSecretTemplateFunctions = template.FuncMap{
},
}
func (r *InfisicalSecretReconciler) createInfisicalManagedKubeSecret(ctx context.Context, logger logr.Logger, infisicalSecret v1alpha1.InfisicalSecret, secretsFromAPI []model.SingleEnvironmentVariable, ETag string) error {
func (r *InfisicalSecretReconciler) createInfisicalManagedKubeSecret(ctx context.Context, logger logr.Logger, infisicalSecret v1alpha1.InfisicalSecret, managedSecretReference v1alpha1.ManagedKubeSecretConfig, secretsFromAPI []model.SingleEnvironmentVariable, ETag string) error {
plainProcessedSecrets := make(map[string][]byte)
secretType := infisicalSecret.Spec.ManagedSecretReference.SecretType
managedTemplateData := infisicalSecret.Spec.ManagedSecretReference.Template
secretType := managedSecretReference.SecretType
managedTemplateData := managedSecretReference.Template
if managedTemplateData == nil || managedTemplateData.IncludeAllSecrets {
for _, secret := range secretsFromAPI {
@@ -226,8 +226,8 @@ func (r *InfisicalSecretReconciler) createInfisicalManagedKubeSecret(ctx context
// create a new secret as specified by the managed secret spec of CRD
newKubeSecretInstance := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: infisicalSecret.Spec.ManagedSecretReference.SecretName,
Namespace: infisicalSecret.Spec.ManagedSecretReference.SecretNamespace,
Name: managedSecretReference.SecretName,
Namespace: managedSecretReference.SecretNamespace,
Annotations: annotations,
Labels: labels,
},
@@ -235,7 +235,7 @@ func (r *InfisicalSecretReconciler) createInfisicalManagedKubeSecret(ctx context
Data: plainProcessedSecrets,
}
if infisicalSecret.Spec.ManagedSecretReference.CreationPolicy == "Owner" {
if managedSecretReference.CreationPolicy == "Owner" {
// Set InfisicalSecret instance as the owner and controller of the managed secret
err := ctrl.SetControllerReference(&infisicalSecret, newKubeSecretInstance, r.Scheme)
if err != nil {
@@ -252,8 +252,8 @@ func (r *InfisicalSecretReconciler) createInfisicalManagedKubeSecret(ctx context
return nil
}
func (r *InfisicalSecretReconciler) updateInfisicalManagedKubeSecret(ctx context.Context, logger logr.Logger, infisicalSecret v1alpha1.InfisicalSecret, managedKubeSecret corev1.Secret, secretsFromAPI []model.SingleEnvironmentVariable, ETag string) error {
managedTemplateData := infisicalSecret.Spec.ManagedSecretReference.Template
func (r *InfisicalSecretReconciler) updateInfisicalManagedKubeSecret(ctx context.Context, logger logr.Logger, managedSecretReference v1alpha1.ManagedKubeSecretConfig, managedKubeSecret corev1.Secret, secretsFromAPI []model.SingleEnvironmentVariable, ETag string) error {
managedTemplateData := managedSecretReference.Template
plainProcessedSecrets := make(map[string][]byte)
if managedTemplateData == nil || managedTemplateData.IncludeAllSecrets {
@@ -337,7 +337,7 @@ func (r *InfisicalSecretReconciler) updateResourceVariables(infisicalSecret v1al
infisicalSecretResourceVariablesMap[string(infisicalSecret.UID)] = resourceVariables
}
func (r *InfisicalSecretReconciler) ReconcileInfisicalSecret(ctx context.Context, logger logr.Logger, infisicalSecret v1alpha1.InfisicalSecret) error {
func (r *InfisicalSecretReconciler) ReconcileInfisicalSecret(ctx context.Context, logger logr.Logger, infisicalSecret v1alpha1.InfisicalSecret, managedSecretReferences []v1alpha1.ManagedKubeSecretConfig) error {
resourceVariables := r.getResourceVariables(infisicalSecret)
infisicalClient := resourceVariables.InfisicalClient
@@ -361,72 +361,84 @@ func (r *InfisicalSecretReconciler) ReconcileInfisicalSecret(ctx context.Context
})
}
// Look for managed secret by name and namespace
managedKubeSecret, err := util.GetKubeSecretByNamespacedName(ctx, r.Client, types.NamespacedName{
Name: infisicalSecret.Spec.ManagedSecretReference.SecretName,
Namespace: infisicalSecret.Spec.ManagedSecretReference.SecretNamespace,
})
for _, managedSecretReference := range managedSecretReferences {
// Look for managed secret by name and namespace
managedKubeSecret, err := util.GetKubeSecretByNamespacedName(ctx, r.Client, types.NamespacedName{
Name: managedSecretReference.SecretName,
Namespace: managedSecretReference.SecretNamespace,
})
if err != nil && !k8Errors.IsNotFound(err) {
return fmt.Errorf("something went wrong when fetching the managed Kubernetes secret [%w]", err)
}
// Get exiting Etag if exists
secretVersionBasedOnETag := ""
if managedKubeSecret != nil {
secretVersionBasedOnETag = managedKubeSecret.Annotations[constants.SECRET_VERSION_ANNOTATION]
}
var plainTextSecretsFromApi []model.SingleEnvironmentVariable
var updateDetails model.RequestUpdateUpdateDetails
if authDetails.AuthStrategy == util.AuthStrategy.SERVICE_ACCOUNT { // Service Account // ! Legacy auth method
serviceAccountCreds, err := r.getInfisicalServiceAccountCredentialsFromKubeSecret(ctx, infisicalSecret)
if err != nil {
return fmt.Errorf("ReconcileInfisicalSecret: unable to get service account creds from kube secret [err=%s]", err)
}
plainTextSecretsFromApi, updateDetails, err = util.GetPlainTextSecretsViaServiceAccount(infisicalClient, serviceAccountCreds, infisicalSecret.Spec.Authentication.ServiceAccount.ProjectId, infisicalSecret.Spec.Authentication.ServiceAccount.EnvironmentName, secretVersionBasedOnETag)
if err != nil {
return fmt.Errorf("\nfailed to get secrets because [err=%v]", err)
}
logger.Info("ReconcileInfisicalSecret: Fetched secrets via service account")
} else if authDetails.AuthStrategy == util.AuthStrategy.SERVICE_TOKEN { // Service Tokens // ! Legacy / Deprecated auth method
infisicalToken, err := r.getInfisicalTokenFromKubeSecret(ctx, infisicalSecret)
if err != nil {
return fmt.Errorf("ReconcileInfisicalSecret: unable to get service token from kube secret [err=%s]", err)
}
envSlug := infisicalSecret.Spec.Authentication.ServiceToken.SecretsScope.EnvSlug
secretsPath := infisicalSecret.Spec.Authentication.ServiceToken.SecretsScope.SecretsPath
recursive := infisicalSecret.Spec.Authentication.ServiceToken.SecretsScope.Recursive
plainTextSecretsFromApi, updateDetails, err = util.GetPlainTextSecretsViaServiceToken(infisicalClient, infisicalToken, secretVersionBasedOnETag, envSlug, secretsPath, recursive)
if err != nil {
return fmt.Errorf("\nfailed to get secrets because [err=%v]", err)
}
logger.Info("ReconcileInfisicalSecret: Fetched secrets via [type=SERVICE_TOKEN]")
} else if authDetails.IsMachineIdentityAuth { // * Machine Identity authentication, the SDK will be authenticated at this point
plainTextSecretsFromApi, updateDetails, err = util.GetPlainTextSecretsViaMachineIdentity(infisicalClient, secretVersionBasedOnETag, authDetails.MachineIdentityScope)
if err != nil {
return fmt.Errorf("\nfailed to get secrets because [err=%v]", err)
}
logger.Info(fmt.Sprintf("ReconcileInfisicalSecret: Fetched secrets via machine identity [type=%v]", authDetails.AuthStrategy))
} else {
return errors.New("no authentication method provided yet. Please configure a authentication method then try again")
}
if managedKubeSecret == nil {
return r.createInfisicalManagedKubeSecret(ctx, logger, infisicalSecret, plainTextSecretsFromApi, updateDetails.ETag)
} else {
return r.updateInfisicalManagedKubeSecret(ctx, logger, infisicalSecret, *managedKubeSecret, plainTextSecretsFromApi, updateDetails.ETag)
if err != nil && !k8Errors.IsNotFound(err) {
return fmt.Errorf("something went wrong when fetching the managed Kubernetes secret [%w]", err)
}
// Get exiting Etag if exists
secretVersionBasedOnETag := ""
if managedKubeSecret != nil {
secretVersionBasedOnETag = managedKubeSecret.Annotations[constants.SECRET_VERSION_ANNOTATION]
}
var plainTextSecretsFromApi []model.SingleEnvironmentVariable
var updateDetails model.RequestUpdateUpdateDetails
if authDetails.AuthStrategy == util.AuthStrategy.SERVICE_ACCOUNT { // Service Account // ! Legacy auth method
serviceAccountCreds, err := r.getInfisicalServiceAccountCredentialsFromKubeSecret(ctx, infisicalSecret)
if err != nil {
return fmt.Errorf("ReconcileInfisicalSecret: unable to get service account creds from kube secret [err=%s]", err)
}
plainTextSecretsFromApi, updateDetails, err = util.GetPlainTextSecretsViaServiceAccount(infisicalClient, serviceAccountCreds, infisicalSecret.Spec.Authentication.ServiceAccount.ProjectId, infisicalSecret.Spec.Authentication.ServiceAccount.EnvironmentName, secretVersionBasedOnETag)
if err != nil {
return fmt.Errorf("\nfailed to get secrets because [err=%v]", err)
}
logger.Info("ReconcileInfisicalSecret: Fetched secrets via service account")
} else if authDetails.AuthStrategy == util.AuthStrategy.SERVICE_TOKEN { // Service Tokens // ! Legacy / Deprecated auth method
infisicalToken, err := r.getInfisicalTokenFromKubeSecret(ctx, infisicalSecret)
if err != nil {
return fmt.Errorf("ReconcileInfisicalSecret: unable to get service token from kube secret [err=%s]", err)
}
envSlug := infisicalSecret.Spec.Authentication.ServiceToken.SecretsScope.EnvSlug
secretsPath := infisicalSecret.Spec.Authentication.ServiceToken.SecretsScope.SecretsPath
recursive := infisicalSecret.Spec.Authentication.ServiceToken.SecretsScope.Recursive
plainTextSecretsFromApi, updateDetails, err = util.GetPlainTextSecretsViaServiceToken(infisicalClient, infisicalToken, secretVersionBasedOnETag, envSlug, secretsPath, recursive)
if err != nil {
return fmt.Errorf("\nfailed to get secrets because [err=%v]", err)
}
logger.Info("ReconcileInfisicalSecret: Fetched secrets via [type=SERVICE_TOKEN]")
} else if authDetails.IsMachineIdentityAuth { // * Machine Identity authentication, the SDK will be authenticated at this point
plainTextSecretsFromApi, updateDetails, err = util.GetPlainTextSecretsViaMachineIdentity(infisicalClient, secretVersionBasedOnETag, authDetails.MachineIdentityScope)
if err != nil {
return fmt.Errorf("\nfailed to get secrets because [err=%v]", err)
}
logger.Info(fmt.Sprintf("ReconcileInfisicalSecret: Fetched secrets via machine identity [type=%v]", authDetails.AuthStrategy))
} else {
return errors.New("no authentication method provided. Please configure a authentication method then try again")
}
if !updateDetails.Modified {
logger.Info("ReconcileInfisicalSecret: No secrets modified so reconcile not needed")
continue
}
if managedKubeSecret == nil {
if err := r.createInfisicalManagedKubeSecret(ctx, logger, infisicalSecret, managedSecretReference, plainTextSecretsFromApi, updateDetails.ETag); err != nil {
return fmt.Errorf("failed to create managed secret [err=%s]", err)
}
} else {
if err := r.updateInfisicalManagedKubeSecret(ctx, logger, managedSecretReference, *managedKubeSecret, plainTextSecretsFromApi, updateDetails.ETag); err != nil {
return fmt.Errorf("failed to update managed secret [err=%s]", err)
}
}
}
return nil
}