Merge pull request #4762 from Infisical/ENG-4006

pam: rotation status & error improvements

backend/src/db/migrations/20251028064623_pam-account-rotation-status.ts

@@ -0,0 +1,29 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasColumn(TableName.PamAccount, "rotationStatus"))) {
+    await knex.schema.alterTable(TableName.PamAccount, (t) => {
+      t.string("rotationStatus").nullable();
+    });
+  }
+  if (!(await knex.schema.hasColumn(TableName.PamAccount, "encryptedLastRotationMessage"))) {
+    await knex.schema.alterTable(TableName.PamAccount, (t) => {
+      t.binary("encryptedLastRotationMessage").nullable();
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.PamAccount, "rotationStatus")) {
+    await knex.schema.alterTable(TableName.PamAccount, (t) => {
+      t.dropColumn("rotationStatus");
+    });
+  }
+  if (await knex.schema.hasColumn(TableName.PamAccount, "encryptedLastRotationMessage")) {
+    await knex.schema.alterTable(TableName.PamAccount, (t) => {
+      t.dropColumn("encryptedLastRotationMessage");
+    });
+  }
+}

backend/src/db/schemas/pam-accounts.ts

@@ -21,7 +21,9 @@ export const PamAccountsSchema = z.object({
   updatedAt: z.date(),
   rotationEnabled: z.boolean().default(false),
   rotationIntervalSeconds: z.number().nullable().optional(),
-  lastRotatedAt: z.date().nullable().optional()
+  lastRotatedAt: z.date().nullable().optional(),
+  rotationStatus: z.string().nullable().optional(),
+  encryptedLastRotationMessage: zodBuffer.nullable().optional()
 });
 
 export type TPamAccounts = z.infer<typeof PamAccountsSchema>;

backend/src/ee/routes/v1/pam-resource-routers/index.ts

@@ -1,14 +1,14 @@
+import {
+  CreateMySQLResourceSchema,
+  MySQLResourceSchema,
+  UpdateMySQLResourceSchema
+} from "@app/ee/services/pam-resource/mysql/mysql-resource-schemas";
 import { PamResource } from "@app/ee/services/pam-resource/pam-resource-enums";
 import {
   CreatePostgresResourceSchema,
   SanitizedPostgresResourceSchema,
   UpdatePostgresResourceSchema
 } from "@app/ee/services/pam-resource/postgres/postgres-resource-schemas";
-import {
-  CreateMySQLResourceSchema,
-  MySQLResourceSchema,
-  UpdateMySQLResourceSchema
-} from "@app/ee/services/pam-resource/mysql/mysql-resource-schemas";
 
 import { registerPamResourceEndpoints } from "./pam-resource-endpoints";
 

backend/src/ee/services/pam-account/pam-account-fns.ts

@@ -45,17 +45,47 @@ export const decryptAccountCredentials = async ({
   return JSON.parse(decryptedPlainTextBlob.toString()) as TPamAccountCredentials;
 };
 
-export const decryptAccount = async <T extends { encryptedCredentials: Buffer }>(
+export const decryptAccountMessage = async ({
+  projectId,
+  encryptedMessage,
+  kmsService
+}: {
+  projectId: string;
+  encryptedMessage: Buffer;
+  kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
+}) => {
+  const { decryptor } = await kmsService.createCipherPairWithDataKey({
+    type: KmsDataKey.SecretManager,
+    projectId
+  });
+
+  const decryptedPlainTextBlob = decryptor({
+    cipherTextBlob: encryptedMessage
+  });
+
+  return decryptedPlainTextBlob.toString();
+};
+
+export const decryptAccount = async <
+  T extends { encryptedCredentials: Buffer; encryptedLastRotationMessage?: Buffer | null }
+>(
   account: T,
   projectId: string,
   kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">
-): Promise<T & { credentials: TPamAccountCredentials }> => {
+): Promise<T & { credentials: TPamAccountCredentials; lastRotationMessage: string | null }> => {
   return {
     ...account,
     credentials: await decryptAccountCredentials({
       encryptedCredentials: account.encryptedCredentials,
       projectId,
       kmsService
-    })
+    }),
-  } as T & { credentials: TPamAccountCredentials };
+    lastRotationMessage: account.encryptedLastRotationMessage
+      ? await decryptAccountMessage({
+          encryptedMessage: account.encryptedLastRotationMessage,
+          projectId,
+          kmsService
+        })
+      : null
+  };
 };

backend/src/ee/services/pam-account/pam-account-service.ts

@@ -15,6 +15,7 @@ import { logger } from "@app/lib/logger";
 import { OrgServiceActor } from "@app/lib/types";
 import { ActorType } from "@app/services/auth/auth-type";
 import { TKmsServiceFactory } from "@app/services/kms/kms-service";
+import { KmsDataKey } from "@app/services/kms/kms-types";
 import { TProjectDALFactory } from "@app/services/project/project-dal";
 import { TUserDALFactory } from "@app/services/user/user-dal";
 
@@ -353,6 +354,7 @@ export const pamAccountServiceFactory = ({
       TPamAccounts & {
         resource: Pick<TPamResources, "id" | "name" | "resourceType"> & { rotationCredentialsConfigured: boolean };
         credentials: TPamAccountCredentials;
+        lastRotationMessage: string | null;
       }
     > = [];
 
@@ -376,6 +378,7 @@ export const pamAccountServiceFactory = ({
       ) {
         // Decrypt the account only if the user has permission to read it
         const decryptedAccount = await decryptAccount(account, account.projectId, kmsService);
+
         decryptedAndPermittedAccounts.push({
           ...decryptedAccount,
           resource: {
@@ -575,10 +578,10 @@ export const pamAccountServiceFactory = ({
     for (let i = 0; i < accounts.length; i += ROTATION_CONCURRENCY_LIMIT) {
       const batch = accounts.slice(i, i + ROTATION_CONCURRENCY_LIMIT);
 
-      const rotationPromises = batch.map(async (account) =>
+      const rotationPromises = batch.map(async (account) => {
-        pamAccountDAL.transaction(async (tx) => {
+        let logResourceType = "unknown";
-          let logResourceType = "unknown";
+        try {
-          try {
+          await pamAccountDAL.transaction(async (tx) => {
             const resource = await pamResourceDAL.findById(account.resourceId, tx);
             if (!resource || !resource.encryptedRotationAccountCredentials) return;
             logResourceType = resource.resourceType;
@@ -619,7 +622,9 @@ export const pamAccountServiceFactory = ({
               account.id,
               {
                 encryptedCredentials,
-                lastRotatedAt: new Date()
+                lastRotatedAt: new Date(),
+                rotationStatus: "success",
+                encryptedLastRotationMessage: null
               },
               tx
             );
@@ -640,32 +645,45 @@ export const pamAccountServiceFactory = ({
                 }
               }
             });
-          } catch (error) {
+          });
-            logger.error(error, `Failed to rotate credentials for account [accountId=${account.id}]`);
+        } catch (error) {
+          logger.error(error, `Failed to rotate credentials for account [accountId=${account.id}]`);
 
-            const errorMessage = error instanceof Error ? error.message : "An unknown error occurred";
+          const errorMessage = error instanceof Error ? error.message : "An unknown error occurred";
 
-            await auditLogService.createAuditLog({
+          const { encryptor } = await kmsService.createCipherPairWithDataKey({
-              projectId: account.projectId,
+            type: KmsDataKey.SecretManager,
-              actor: {
+            projectId: account.projectId
-                type: ActorType.PLATFORM,
+          });
-                metadata: {}
+
-              },
+          const { cipherTextBlob: encryptedMessage } = encryptor({
-              event: {
+            plainText: Buffer.from(errorMessage)
-                type: EventType.PAM_ACCOUNT_CREDENTIAL_ROTATION_FAILED,
+          });
-                metadata: {
+
-                  accountId: account.id,
+          await pamAccountDAL.updateById(account.id, {
-                  accountName: account.name,
+            rotationStatus: "failed",
-                  resourceId: account.resourceId,
+            encryptedLastRotationMessage: encryptedMessage
-                  resourceType: logResourceType,
+          });
-                  errorMessage
+
-                }
+          await auditLogService.createAuditLog({
+            projectId: account.projectId,
+            actor: {
+              type: ActorType.PLATFORM,
+              metadata: {}
+            },
+            event: {
+              type: EventType.PAM_ACCOUNT_CREDENTIAL_ROTATION_FAILED,
+              metadata: {
+                accountId: account.id,
+                accountName: account.name,
+                resourceId: account.resourceId,
+                resourceType: logResourceType,
+                errorMessage
               }
-            });
+            }
-            throw error; // Rollback transaction
+          });
-          }
+        }
-        })
+      });
-      );
 
       // eslint-disable-next-line no-await-in-loop
       await Promise.all(rotationPromises);

backend/src/ee/services/pam-resource/pam-resource-schemas.ts

@@ -33,7 +33,9 @@ export const BasePamAccountSchemaWithResource = BasePamAccountSchema.extend({
     resourceType: true
   }).extend({
     rotationCredentialsConfigured: z.boolean()
-  })
+  }),
+  lastRotationMessage: z.string().nullable().optional(),
+  rotationStatus: z.string().nullable().optional()
 });
 
 export const BaseCreatePamAccountSchema = z.object({

backend/src/ee/services/pam-resource/shared/sql/sql-resource-factory.ts

@@ -41,7 +41,10 @@ export interface SqlResourceConnection {
    *
    * @returns Promise to be resolved with the new credentials
    */
-  rotateCredentials: (currentCredentials: TSqlAccountCredentials) => Promise<TSqlAccountCredentials>;
+  rotateCredentials: (
+    currentCredentials: TSqlAccountCredentials,
+    newPassword: string
+  ) => Promise<TSqlAccountCredentials>;
 
   /**
    * Close the connection.
@@ -113,8 +116,7 @@ const makeSqlConnection = (
             });
           }
         },
-        rotateCredentials: async (currentCredentials) => {
+        rotateCredentials: async (currentCredentials, newPassword) => {
-          const newPassword = alphaNumericNanoId(32);
           // Note: The generated random password is not really going to make SQL Injection possible.
           //       The reason we are not using parameters binding is that the "ALTER USER" syntax is DDL,
           //       parameters binding is not supported. But just in case if the this code got copied
@@ -295,6 +297,7 @@ export const sqlResourceFactory: TPamResourceFactory<TSqlResourceConnectionDetai
     rotationAccountCredentials,
     currentCredentials
   ) => {
+    const newPassword = alphaNumericNanoId(32);
     try {
       return await executeWithGateway(
         {
@@ -305,7 +308,7 @@ export const sqlResourceFactory: TPamResourceFactory<TSqlResourceConnectionDetai
           password: rotationAccountCredentials.password
         },
         gatewayV2Service,
-        (client) => client.rotateCredentials(currentCredentials)
+        (client) => client.rotateCredentials(currentCredentials, newPassword)
       );
     } catch (error) {
       if (error instanceof BadRequestError) {
@@ -328,8 +331,10 @@ export const sqlResourceFactory: TPamResourceFactory<TSqlResourceConnectionDetai
         }
       }
 
+      const sanitizedErrorMessage = ((error as Error).message || String(error)).replaceAll(newPassword, "REDACTED");
+
       throw new BadRequestError({
-        message: `Unable to rotate account credentials for ${resourceType}: ${(error as Error).message || String(error)}`
+        message: `Unable to rotate account credentials for ${resourceType}: ${sanitizedErrorMessage}`
       });
     }
   };

frontend/src/hooks/api/pam/types/base-account.ts

@@ -16,6 +16,8 @@ export interface TBasePamAccount {
   rotationEnabled: boolean;
   rotationIntervalSeconds?: number | null;
   lastRotatedAt?: string | null;
+  lastRotationMessage?: string | null;
+  rotationStatus?: string | null;
   createdAt: string;
   updatedAt: string;
 }

frontend/src/pages/pam/PamAccountsPage/components/PamAccountRow.tsx

@@ -5,11 +5,12 @@ import {
   faEdit,
   faEllipsisV,
   faRightToBracket,
+  faRotate,
   faTrash
 } from "@fortawesome/free-solid-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { formatDistance } from "date-fns";
-import { FolderIcon, PackageOpenIcon, RefreshCwIcon } from "lucide-react";
+import { FolderIcon, PackageOpenIcon } from "lucide-react";
 import { twMerge } from "tailwind-merge";
 
 import { createNotification } from "@app/components/notifications";
@@ -101,10 +102,16 @@ export const PamAccountRow = ({
               </Badge>
             )}
             {account.lastRotatedAt && (
-              <Badge variant="info">
+              <Tooltip
-                <RefreshCwIcon />
+                className="max-w-sm text-center"
-                <span>Rotated {formatDistance(new Date(), account.lastRotatedAt)} ago</span>
+                isDisabled={!account.lastRotationMessage}
-              </Badge>
+                content={account.lastRotationMessage}
+              >
+                <Badge variant={account.rotationStatus === "failed" ? "danger" : "success"}>
+                  <FontAwesomeIcon icon={faRotate} />
+                  <span>Rotated {formatDistance(new Date(), account.lastRotatedAt)} ago</span>
+                </Badge>
+              </Tooltip>
             )}
           </div>
         </div>