Merge pull request #4317 from Infisical/daniel/rotation-tests

feat(e2e-tests): secret rotations

.github/workflows/run-backend-tests.yml

@@ -16,6 +16,16 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
+
+
+      - name: Free up disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker system prune -af
+          
       - name: ☁️ Checkout source
         uses: actions/checkout@v3
       - uses: KengoTODA/actions-setup-docker-compose@v1
@@ -34,6 +44,8 @@ jobs:
         working-directory: backend
       - name: Start postgres and redis
         run: touch .env && docker compose -f docker-compose.dev.yml up -d db redis
+      - name: Start Secret Rotation testing databases
+        run: docker compose -f docker-compose.e2e-dbs.yml up -d --wait --wait-timeout 300
       - name: Run unit test
         run: npm run test:unit
         working-directory: backend
@@ -41,6 +53,9 @@ jobs:
         run: npm run test:e2e
         working-directory: backend
         env:
+          E2E_TEST_ORACLE_DB_19_HOST: ${{ secrets.E2E_TEST_ORACLE_DB_19_HOST }}
+          E2E_TEST_ORACLE_DB_19_USERNAME: ${{ secrets.E2E_TEST_ORACLE_DB_19_USERNAME }}
+          E2E_TEST_ORACLE_DB_19_PASSWORD: ${{ secrets.E2E_TEST_ORACLE_DB_19_PASSWORD }}
           REDIS_URL: redis://172.17.0.1:6379
           DB_CONNECTION_URI: postgres://infisical:infisical@172.17.0.1:5432/infisical?sslmode=disable
           AUTH_SECRET: something-random

.infisicalignore

@@ -50,3 +50,4 @@ docs/integrations/app-connections/zabbix.mdx:generic-api-key:91
 docs/integrations/app-connections/bitbucket.mdx:generic-api-key:123
 docs/integrations/app-connections/railway.mdx:generic-api-key:156
 .github/workflows/validate-db-schemas.yml:generic-api-key:21
+k8-operator/config/samples/universalAuthIdentitySecret.yaml:generic-api-key:8

backend/e2e-test/mocks/queue.ts

@@ -1,34 +0,0 @@
-import { TQueueServiceFactory } from "@app/queue";
-
-export const mockQueue = (): TQueueServiceFactory => {
-  const queues: Record<string, unknown> = {};
-  const workers: Record<string, unknown> = {};
-  const job: Record<string, unknown> = {};
-  const events: Record<string, unknown> = {};
-
-  return {
-    queue: async (name, jobData) => {
-      job[name] = jobData;
-    },
-    queuePg: async () => {},
-    schedulePg: async () => {},
-    initialize: async () => {},
-    shutdown: async () => undefined,
-    stopRepeatableJob: async () => true,
-    start: (name, jobFn) => {
-      queues[name] = jobFn;
-      workers[name] = jobFn;
-    },
-    startPg: async () => {},
-    listen: (name, event) => {
-      events[name] = event;
-    },
-    getRepeatableJobs: async () => [],
-    getDelayedJobs: async () => [],
-    clearQueue: async () => {},
-    stopJobById: async () => {},
-    stopJobByIdPg: async () => {},
-    stopRepeatableJobByJobId: async () => true,
-    stopRepeatableJobByKey: async () => true
-  };
-};

backend/e2e-test/routes/v3/secret-rotations.spec.ts

@@ -0,0 +1,726 @@
+/* eslint-disable no-promise-executor-return */
+/* eslint-disable no-await-in-loop */
+import knex from "knex";
+import { v4 as uuidv4 } from "uuid";
+
+import { seedData1 } from "@app/db/seed-data";
+
+enum SecretRotationType {
+  OracleDb = "oracledb",
+  MySQL = "mysql",
+  Postgres = "postgres"
+}
+
+type TGenericSqlCredentials = {
+  host: string;
+  port: number;
+  username: string;
+  password: string;
+  database: string;
+};
+
+type TSecretMapping = {
+  username: string;
+  password: string;
+};
+
+type TDatabaseUserCredentials = {
+  username: string;
+};
+
+const formatSqlUsername = (username: string) => `${username}_${uuidv4().slice(0, 8).replace(/-/g, "").toUpperCase()}`;
+
+const getSecretValue = async (secretKey: string) => {
+  const passwordSecret = await testServer.inject({
+    url: `/api/v3/secrets/raw/${secretKey}`,
+    method: "GET",
+    query: {
+      workspaceId: seedData1.projectV3.id,
+      environment: seedData1.environment.slug
+    },
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+
+  expect(passwordSecret.statusCode).toBe(200);
+  expect(passwordSecret.json().secret).toBeDefined();
+
+  const passwordSecretJson = JSON.parse(passwordSecret.payload);
+
+  return passwordSecretJson.secret.secretValue as string;
+};
+
+const deleteSecretRotation = async (id: string, type: SecretRotationType) => {
+  const res = await testServer.inject({
+    method: "DELETE",
+    query: {
+      deleteSecrets: "true",
+      revokeGeneratedCredentials: "true"
+    },
+    url: `/api/v2/secret-rotations/${type}-credentials/${id}`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+
+  expect(res.statusCode).toBe(200);
+};
+
+const deleteAppConnection = async (id: string, type: SecretRotationType) => {
+  const res = await testServer.inject({
+    method: "DELETE",
+    url: `/api/v1/app-connections/${type}/${id}`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    }
+  });
+
+  expect(res.statusCode).toBe(200);
+};
+
+const createOracleDBAppConnection = async (credentials: TGenericSqlCredentials) => {
+  const createOracleDBAppConnectionReqBody = {
+    credentials: {
+      database: credentials.database,
+      host: credentials.host,
+      username: credentials.username,
+      password: credentials.password,
+      port: credentials.port,
+      sslEnabled: true,
+      sslRejectUnauthorized: true
+    },
+    name: `oracle-db-${uuidv4()}`,
+    description: "Test OracleDB App Connection",
+    gatewayId: null,
+    isPlatformManagedCredentials: false,
+    method: "username-and-password"
+  };
+
+  const res = await testServer.inject({
+    method: "POST",
+    url: `/api/v1/app-connections/oracledb`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: createOracleDBAppConnectionReqBody
+  });
+
+  const json = JSON.parse(res.payload);
+
+  expect(res.statusCode).toBe(200);
+  expect(json.appConnection).toBeDefined();
+
+  return json.appConnection.id as string;
+};
+
+const createMySQLAppConnection = async (credentials: TGenericSqlCredentials) => {
+  const createMySQLAppConnectionReqBody = {
+    name: `mysql-test-${uuidv4()}`,
+    description: "test-mysql",
+    gatewayId: null,
+    method: "username-and-password",
+    credentials: {
+      host: credentials.host,
+      port: credentials.port,
+      database: credentials.database,
+      username: credentials.username,
+      password: credentials.password,
+      sslEnabled: false,
+      sslRejectUnauthorized: true
+    }
+  };
+
+  const res = await testServer.inject({
+    method: "POST",
+    url: `/api/v1/app-connections/mysql`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: createMySQLAppConnectionReqBody
+  });
+
+  const json = JSON.parse(res.payload);
+
+  expect(res.statusCode).toBe(200);
+  expect(json.appConnection).toBeDefined();
+
+  return json.appConnection.id as string;
+};
+
+const createPostgresAppConnection = async (credentials: TGenericSqlCredentials) => {
+  const createPostgresAppConnectionReqBody = {
+    credentials: {
+      host: credentials.host,
+      port: credentials.port,
+      database: credentials.database,
+      username: credentials.username,
+      password: credentials.password,
+      sslEnabled: false,
+      sslRejectUnauthorized: true
+    },
+    name: `postgres-test-${uuidv4()}`,
+    description: "test-postgres",
+    gatewayId: null,
+    method: "username-and-password"
+  };
+
+  const res = await testServer.inject({
+    method: "POST",
+    url: `/api/v1/app-connections/postgres`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: createPostgresAppConnectionReqBody
+  });
+
+  const json = JSON.parse(res.payload);
+
+  expect(res.statusCode).toBe(200);
+  expect(json.appConnection).toBeDefined();
+
+  return json.appConnection.id as string;
+};
+
+const createOracleInfisicalUsers = async (
+  credentials: TGenericSqlCredentials,
+  userCredentials: TDatabaseUserCredentials[]
+) => {
+  const client = knex({
+    client: "oracledb",
+    connection: {
+      database: credentials.database,
+      port: credentials.port,
+      host: credentials.host,
+      user: credentials.username,
+      password: credentials.password,
+      connectionTimeoutMillis: 10000,
+      ssl: {
+        // @ts-expect-error - this is a valid property for the ssl object
+        sslServerDNMatch: true
+      }
+    }
+  });
+
+  for await (const { username } of userCredentials) {
+    // check if user exists, and if it does, don't create it
+    const existingUser = await client.raw(`SELECT * FROM all_users WHERE username = '${username}'`);
+
+    if (!existingUser.length) {
+      await client.raw(`CREATE USER ${username} IDENTIFIED BY "temporary_password"`);
+    }
+    await client.raw(`GRANT ALL PRIVILEGES TO ${username} WITH ADMIN OPTION`);
+  }
+
+  await client.destroy();
+};
+
+const createMySQLInfisicalUsers = async (
+  credentials: TGenericSqlCredentials,
+  userCredentials: TDatabaseUserCredentials[]
+) => {
+  const client = knex({
+    client: "mysql2",
+    connection: {
+      database: credentials.database,
+      port: credentials.port,
+      host: credentials.host,
+      user: credentials.username,
+      password: credentials.password,
+      connectionTimeoutMillis: 10000
+    }
+  });
+
+  // Fix: Ensure root has GRANT OPTION privileges
+  try {
+    await client.raw("GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION;");
+    await client.raw("FLUSH PRIVILEGES;");
+  } catch (error) {
+    // Ignore if already has privileges
+  }
+
+  for await (const { username } of userCredentials) {
+    // check if user exists, and if it does, dont create it
+
+    const existingUser = await client.raw(`SELECT * FROM mysql.user WHERE user = '${username}'`);
+
+    if (!existingUser[0].length) {
+      await client.raw(`CREATE USER '${username}'@'%' IDENTIFIED BY 'temporary_password';`);
+    }
+
+    await client.raw(`GRANT ALL PRIVILEGES ON \`${credentials.database}\`.* TO '${username}'@'%';`);
+    await client.raw("FLUSH PRIVILEGES;");
+  }
+
+  await client.destroy();
+};
+
+const createPostgresInfisicalUsers = async (
+  credentials: TGenericSqlCredentials,
+  userCredentials: TDatabaseUserCredentials[]
+) => {
+  const client = knex({
+    client: "pg",
+    connection: {
+      database: credentials.database,
+      port: credentials.port,
+      host: credentials.host,
+      user: credentials.username,
+      password: credentials.password,
+      connectionTimeoutMillis: 10000
+    }
+  });
+
+  for await (const { username } of userCredentials) {
+    // check if user exists, and if it does, don't create it
+    const existingUser = await client.raw("SELECT * FROM pg_catalog.pg_user WHERE usename = ?", [username]);
+
+    if (!existingUser.rows.length) {
+      await client.raw(`CREATE USER "${username}" WITH PASSWORD 'temporary_password'`);
+    }
+
+    await client.raw("GRANT ALL PRIVILEGES ON DATABASE ?? TO ??", [credentials.database, username]);
+  }
+
+  await client.destroy();
+};
+
+const createOracleDBSecretRotation = async (
+  appConnectionId: string,
+  credentials: TGenericSqlCredentials,
+  userCredentials: TDatabaseUserCredentials[],
+  secretMapping: TSecretMapping
+) => {
+  const now = new Date();
+  const rotationTime = new Date(now.getTime() - 2 * 60 * 1000); // 2 minutes ago
+
+  await createOracleInfisicalUsers(credentials, userCredentials);
+
+  const createOracleDBSecretRotationReqBody = {
+    parameters: userCredentials.reduce(
+      (acc, user, index) => {
+        acc[`username${index + 1}`] = user.username;
+        return acc;
+      },
+      {} as Record<string, string>
+    ),
+    secretsMapping: {
+      username: secretMapping.username,
+      password: secretMapping.password
+    },
+    name: `test-oracle-${uuidv4()}`,
+    description: "Test OracleDB Secret Rotation",
+    secretPath: "/",
+    isAutoRotationEnabled: true,
+    rotationInterval: 5, // 5 seconds for testing
+    rotateAtUtc: {
+      hours: rotationTime.getUTCHours(),
+      minutes: rotationTime.getUTCMinutes()
+    },
+    connectionId: appConnectionId,
+    environment: seedData1.environment.slug,
+    projectId: seedData1.projectV3.id
+  };
+
+  const res = await testServer.inject({
+    method: "POST",
+    url: `/api/v2/secret-rotations/oracledb-credentials`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: createOracleDBSecretRotationReqBody
+  });
+
+  expect(res.statusCode).toBe(200);
+  expect(res.json().secretRotation).toBeDefined();
+
+  return res;
+};
+
+const createMySQLSecretRotation = async (
+  appConnectionId: string,
+  credentials: TGenericSqlCredentials,
+  userCredentials: TDatabaseUserCredentials[],
+  secretMapping: TSecretMapping
+) => {
+  const now = new Date();
+  const rotationTime = new Date(now.getTime() - 2 * 60 * 1000); // 2 minutes ago
+
+  await createMySQLInfisicalUsers(credentials, userCredentials);
+
+  const createMySQLSecretRotationReqBody = {
+    parameters: userCredentials.reduce(
+      (acc, user, index) => {
+        acc[`username${index + 1}`] = user.username;
+        return acc;
+      },
+      {} as Record<string, string>
+    ),
+    secretsMapping: {
+      username: secretMapping.username,
+      password: secretMapping.password
+    },
+    name: `test-mysql-rotation-${uuidv4()}`,
+    description: "Test MySQL Secret Rotation",
+    secretPath: "/",
+    isAutoRotationEnabled: true,
+    rotationInterval: 5,
+    rotateAtUtc: {
+      hours: rotationTime.getUTCHours(),
+      minutes: rotationTime.getUTCMinutes()
+    },
+    connectionId: appConnectionId,
+    environment: seedData1.environment.slug,
+    projectId: seedData1.projectV3.id
+  };
+
+  const res = await testServer.inject({
+    method: "POST",
+    url: `/api/v2/secret-rotations/mysql-credentials`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: createMySQLSecretRotationReqBody
+  });
+
+  expect(res.statusCode).toBe(200);
+  expect(res.json().secretRotation).toBeDefined();
+
+  return res;
+};
+
+const createPostgresSecretRotation = async (
+  appConnectionId: string,
+  credentials: TGenericSqlCredentials,
+  userCredentials: TDatabaseUserCredentials[],
+  secretMapping: TSecretMapping
+) => {
+  const now = new Date();
+  const rotationTime = new Date(now.getTime() - 2 * 60 * 1000); // 2 minutes ago
+
+  await createPostgresInfisicalUsers(credentials, userCredentials);
+
+  const createPostgresSecretRotationReqBody = {
+    parameters: userCredentials.reduce(
+      (acc, user, index) => {
+        acc[`username${index + 1}`] = user.username;
+        return acc;
+      },
+      {} as Record<string, string>
+    ),
+    secretsMapping: {
+      username: secretMapping.username,
+      password: secretMapping.password
+    },
+    name: `test-postgres-rotation-${uuidv4()}`,
+    description: "Test Postgres Secret Rotation",
+    secretPath: "/",
+    isAutoRotationEnabled: true,
+    rotationInterval: 5,
+    rotateAtUtc: {
+      hours: rotationTime.getUTCHours(),
+      minutes: rotationTime.getUTCMinutes()
+    },
+    connectionId: appConnectionId,
+    environment: seedData1.environment.slug,
+    projectId: seedData1.projectV3.id
+  };
+
+  const res = await testServer.inject({
+    method: "POST",
+    url: `/api/v2/secret-rotations/postgres-credentials`,
+    headers: {
+      authorization: `Bearer ${jwtAuthToken}`
+    },
+    body: createPostgresSecretRotationReqBody
+  });
+
+  expect(res.statusCode).toBe(200);
+  expect(res.json().secretRotation).toBeDefined();
+
+  return res;
+};
+
+describe("Secret Rotations", async () => {
+  const testCases = [
+    {
+      type: SecretRotationType.MySQL,
+      name: "MySQL (8.4.6) Secret Rotation",
+      dbCredentials: {
+        database: "mysql-test",
+        host: "127.0.0.1",
+        username: "root",
+        password: "mysql-test",
+        port: 3306
+      },
+      secretMapping: {
+        username: formatSqlUsername("MYSQL_USERNAME"),
+        password: formatSqlUsername("MYSQL_PASSWORD")
+      },
+      userCredentials: [
+        {
+          username: formatSqlUsername("MYSQL_USER_1")
+        },
+        {
+          username: formatSqlUsername("MYSQL_USER_2")
+        }
+      ]
+    },
+    {
+      type: SecretRotationType.MySQL,
+      name: "MySQL (8.0.29) Secret Rotation",
+      dbCredentials: {
+        database: "mysql-test",
+        host: "127.0.0.1",
+        username: "root",
+        password: "mysql-test",
+        port: 3307
+      },
+      secretMapping: {
+        username: formatSqlUsername("MYSQL_USERNAME"),
+        password: formatSqlUsername("MYSQL_PASSWORD")
+      },
+      userCredentials: [
+        {
+          username: formatSqlUsername("MYSQL_USER_1")
+        },
+        {
+          username: formatSqlUsername("MYSQL_USER_2")
+        }
+      ]
+    },
+    {
+      type: SecretRotationType.MySQL,
+      name: "MySQL (5.7.31) Secret Rotation",
+      dbCredentials: {
+        database: "mysql-test",
+        host: "127.0.0.1",
+        username: "root",
+        password: "mysql-test",
+        port: 3308
+      },
+      secretMapping: {
+        username: formatSqlUsername("MYSQL_USERNAME"),
+        password: formatSqlUsername("MYSQL_PASSWORD")
+      },
+      userCredentials: [
+        {
+          username: formatSqlUsername("MYSQL_USER_1")
+        },
+        {
+          username: formatSqlUsername("MYSQL_USER_2")
+        }
+      ]
+    },
+    {
+      type: SecretRotationType.OracleDb,
+      name: "OracleDB (23.8) Secret Rotation",
+      dbCredentials: {
+        database: "FREEPDB1",
+        host: "127.0.0.1",
+        username: "system",
+        password: "pdb-password",
+        port: 1521
+      },
+      secretMapping: {
+        username: formatSqlUsername("ORACLEDB_USERNAME"),
+        password: formatSqlUsername("ORACLEDB_PASSWORD")
+      },
+      userCredentials: [
+        {
+          username: formatSqlUsername("INFISICAL_USER_1")
+        },
+        {
+          username: formatSqlUsername("INFISICAL_USER_2")
+        }
+      ]
+    },
+    {
+      type: SecretRotationType.OracleDb,
+      name: "OracleDB (19.3) Secret Rotation",
+      skippable: true,
+      dbCredentials: {
+        password: process.env.E2E_TEST_ORACLE_DB_19_PASSWORD!,
+        host: process.env.E2E_TEST_ORACLE_DB_19_HOST!,
+        username: process.env.E2E_TEST_ORACLE_DB_19_USERNAME!,
+        port: 1521,
+        database: "ORCLPDB1"
+      },
+      secretMapping: {
+        username: formatSqlUsername("ORACLEDB_USERNAME"),
+        password: formatSqlUsername("ORACLEDB_PASSWORD")
+      },
+      userCredentials: [
+        {
+          username: formatSqlUsername("INFISICAL_USER_1")
+        },
+        {
+          username: formatSqlUsername("INFISICAL_USER_2")
+        }
+      ]
+    },
+    {
+      type: SecretRotationType.Postgres,
+      name: "Postgres (17) Secret Rotation",
+      dbCredentials: {
+        database: "postgres-test",
+        host: "127.0.0.1",
+        username: "postgres-test",
+        password: "postgres-test",
+        port: 5433
+      },
+      secretMapping: {
+        username: formatSqlUsername("POSTGRES_USERNAME"),
+        password: formatSqlUsername("POSTGRES_PASSWORD")
+      },
+      userCredentials: [
+        {
+          username: formatSqlUsername("INFISICAL_USER_1")
+        },
+        {
+          username: formatSqlUsername("INFISICAL_USER_2")
+        }
+      ]
+    },
+    {
+      type: SecretRotationType.Postgres,
+      name: "Postgres (16) Secret Rotation",
+      dbCredentials: {
+        database: "postgres-test",
+        host: "127.0.0.1",
+        username: "postgres-test",
+        password: "postgres-test",
+        port: 5434
+      },
+      secretMapping: {
+        username: formatSqlUsername("POSTGRES_USERNAME"),
+        password: formatSqlUsername("POSTGRES_PASSWORD")
+      },
+      userCredentials: [
+        {
+          username: formatSqlUsername("INFISICAL_USER_1")
+        },
+        {
+          username: formatSqlUsername("INFISICAL_USER_2")
+        }
+      ]
+    },
+    {
+      type: SecretRotationType.Postgres,
+      name: "Postgres (10.12) Secret Rotation",
+      dbCredentials: {
+        database: "postgres-test",
+        host: "127.0.0.1",
+        username: "postgres-test",
+        password: "postgres-test",
+        port: 5435
+      },
+      secretMapping: {
+        username: formatSqlUsername("POSTGRES_USERNAME"),
+        password: formatSqlUsername("POSTGRES_PASSWORD")
+      },
+      userCredentials: [
+        {
+          username: formatSqlUsername("INFISICAL_USER_1")
+        },
+        {
+          username: formatSqlUsername("INFISICAL_USER_2")
+        }
+      ]
+    }
+  ] as {
+    skippable?: boolean;
+    type: SecretRotationType;
+    name: string;
+    dbCredentials: TGenericSqlCredentials;
+    secretMapping: TSecretMapping;
+    userCredentials: TDatabaseUserCredentials[];
+  }[];
+
+  const createAppConnectionMap = {
+    [SecretRotationType.OracleDb]: createOracleDBAppConnection,
+    [SecretRotationType.MySQL]: createMySQLAppConnection,
+    [SecretRotationType.Postgres]: createPostgresAppConnection
+  };
+
+  const createRotationMap = {
+    [SecretRotationType.OracleDb]: createOracleDBSecretRotation,
+    [SecretRotationType.MySQL]: createMySQLSecretRotation,
+    [SecretRotationType.Postgres]: createPostgresSecretRotation
+  };
+
+  const appConnectionIds: { id: string; type: SecretRotationType }[] = [];
+  const secretRotationIds: { id: string; type: SecretRotationType }[] = [];
+
+  afterAll(async () => {
+    for (const { id, type } of secretRotationIds) {
+      await deleteSecretRotation(id, type);
+    }
+
+    for (const { id, type } of appConnectionIds) {
+      await deleteAppConnection(id, type);
+    }
+  });
+
+  testCases.forEach(({ skippable, dbCredentials, secretMapping, userCredentials, type, name }) => {
+    const shouldSkip = () => {
+      if (skippable) {
+        if (type === SecretRotationType.OracleDb) {
+          if (!process.env.E2E_TEST_ORACLE_DB_19_HOST) {
+            return true;
+          }
+        }
+      }
+
+      return false;
+    };
+
+    if (shouldSkip()) {
+      test.skip(`Skipping Secret Rotation for ${type} (${name}) because E2E_TEST_ORACLE_DB_19_HOST is not set`);
+    } else {
+      test.concurrent(
+        `Create secret rotation for ${name}`,
+        async () => {
+          const appConnectionId = await createAppConnectionMap[type](dbCredentials);
+
+          if (appConnectionId) {
+            appConnectionIds.push({ id: appConnectionId, type });
+          }
+
+          const res = await createRotationMap[type](appConnectionId, dbCredentials, userCredentials, secretMapping);
+
+          const resJson = JSON.parse(res.payload);
+
+          if (resJson.secretRotation) {
+            secretRotationIds.push({ id: resJson.secretRotation.id, type });
+          }
+
+          const startSecretValue = await getSecretValue(secretMapping.password);
+          expect(startSecretValue).toBeDefined();
+
+          let attempts = 0;
+          while (attempts < 60) {
+            const currentSecretValue = await getSecretValue(secretMapping.password);
+
+            if (currentSecretValue !== startSecretValue) {
+              break;
+            }
+
+            attempts += 1;
+            await new Promise((resolve) => setTimeout(resolve, 2_500));
+          }
+
+          if (attempts >= 60) {
+            throw new Error("Secret rotation failed to rotate after 60 attempts");
+          }
+
+          const finalSecretValue = await getSecretValue(secretMapping.password);
+          expect(finalSecretValue).not.toBe(startSecretValue);
+        },
+        {
+          timeout: 300_000
+        }
+      );
+    }
+  });
+});

backend/e2e-test/vitest-environment-knex.ts

@@ -18,6 +18,7 @@ import { keyStoreFactory } from "@app/keystore/keystore";
 import { initializeHsmModule } from "@app/ee/services/hsm/hsm-fns";
 import { buildRedisFromConfig } from "@app/lib/config/redis";
 import { superAdminDALFactory } from "@app/services/super-admin/super-admin-dal";
+import { bootstrapCheck } from "@app/server/boot-strap-check";
 
 dotenv.config({ path: path.join(__dirname, "../../.env.test"), debug: true });
 export default {
@@ -63,6 +64,8 @@ export default {
       const queue = queueServiceFactory(envCfg, { dbConnectionUrl: envCfg.DB_CONNECTION_URI });
       const keyStore = keyStoreFactory(envCfg);
 
+      await queue.initialize();
+
       const hsmModule = initializeHsmModule(envCfg);
       hsmModule.initialize();
 
@@ -78,9 +81,13 @@ export default {
         envConfig: envCfg
       });
 
+      await bootstrapCheck({ db });
+
       // @ts-expect-error type
       globalThis.testServer = server;
       // @ts-expect-error type
+      globalThis.testQueue = queue;
+      // @ts-expect-error type
       globalThis.testSuperAdminDAL = superAdminDAL;
       // @ts-expect-error type
       globalThis.jwtAuthToken = crypto.jwt().sign(
@@ -105,6 +112,8 @@ export default {
     // custom setup
     return {
       async teardown() {
+        // @ts-expect-error type
+        await globalThis.testQueue.shutdown();
         // @ts-expect-error type
         await globalThis.testServer.close();
         // @ts-expect-error type
@@ -112,7 +121,9 @@ export default {
         // @ts-expect-error type
         delete globalThis.testSuperAdminDAL;
         // @ts-expect-error type
-        delete globalThis.jwtToken;
+        delete globalThis.jwtAuthToken;
+        // @ts-expect-error type
+        delete globalThis.testQueue;
         // called after all tests with this env have been run
         await db.migrate.rollback(
           {

backend/src/db/migrations/20250725144940_fix-secret-reminders-migration.ts

@@ -2,7 +2,7 @@
 import { Knex } from "knex";
 
 import { chunkArray } from "@app/lib/fn";
-import { logger } from "@app/lib/logger";
+import { initLogger, logger } from "@app/lib/logger";
 
 import { TableName } from "../schemas";
 import { TReminders, TRemindersInsert } from "../schemas/reminders";
@@ -107,5 +107,6 @@ export async function up(knex: Knex): Promise<void> {
 }
 
 export async function down(): Promise<void> {
+  initLogger();
   logger.info("Rollback not implemented for secret reminders fix migration");
 }

backend/src/ee/services/dynamic-secret/dynamic-secret-fns.ts

@@ -9,7 +9,7 @@ import { getDbConnectionHost } from "@app/lib/knex";
 export const verifyHostInputValidity = async (host: string, isGateway = false) => {
   const appCfg = getConfig();
 
-  if (appCfg.isDevelopmentMode) return [host];
+  if (appCfg.isDevelopmentMode || appCfg.isTestMode) return [host];
 
   if (isGateway) return [host];
 

backend/src/ee/services/license/__mocks__/license-fns.ts

@@ -31,7 +31,7 @@ export const getDefaultOnPremFeatures = () => {
     caCrl: false,
     sshHostGroups: false,
     enterpriseSecretSyncs: false,
-    enterpriseAppConnections: false,
+    enterpriseAppConnections: true,
     machineIdentityAuthTemplates: false
   };
 };

backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-fns.ts

@@ -2,6 +2,7 @@ import { AxiosError } from "axios";
 
 import { getConfig } from "@app/lib/config/env";
 import { BadRequestError } from "@app/lib/errors";
+import { logger } from "@app/lib/logger";
 import { KmsDataKey } from "@app/services/kms/kms-types";
 
 import { AUTH0_CLIENT_SECRET_ROTATION_LIST_OPTION } from "./auth0-client-secret";
@@ -13,9 +14,11 @@ import { MYSQL_CREDENTIALS_ROTATION_LIST_OPTION } from "./mysql-credentials";
 import { OKTA_CLIENT_SECRET_ROTATION_LIST_OPTION } from "./okta-client-secret";
 import { ORACLEDB_CREDENTIALS_ROTATION_LIST_OPTION } from "./oracledb-credentials";
 import { POSTGRES_CREDENTIALS_ROTATION_LIST_OPTION } from "./postgres-credentials";
+import { TSecretRotationV2DALFactory } from "./secret-rotation-v2-dal";
 import { SecretRotation, SecretRotationStatus } from "./secret-rotation-v2-enums";
-import { TSecretRotationV2ServiceFactoryDep } from "./secret-rotation-v2-service";
+import { TSecretRotationV2ServiceFactory, TSecretRotationV2ServiceFactoryDep } from "./secret-rotation-v2-service";
 import {
+  TSecretRotationRotateSecretsJobPayload,
   TSecretRotationV2,
   TSecretRotationV2GeneratedCredentials,
   TSecretRotationV2ListItem,
@@ -74,6 +77,10 @@ export const getNextUtcRotationInterval = (rotateAtUtc?: TSecretRotationV2["rota
   const appCfg = getConfig();
 
   if (appCfg.isRotationDevelopmentMode) {
+    if (appCfg.isTestMode) {
+      // if its test mode, it should always rotate
+      return new Date(Date.now() + 365 * 24 * 60 * 60 * 1000); // Current time + 1 year
+    }
     return getNextUTCMinuteInterval(rotateAtUtc);
   }
 
@@ -263,3 +270,51 @@ export const throwOnImmutableParameterUpdate = (
     // do nothing
   }
 };
+
+export const rotateSecretsFns = async ({
+  job,
+  secretRotationV2DAL,
+  secretRotationV2Service
+}: {
+  job: {
+    data: TSecretRotationRotateSecretsJobPayload;
+    id: string;
+    retryCount: number;
+    retryLimit: number;
+  };
+  secretRotationV2DAL: Pick<TSecretRotationV2DALFactory, "findById">;
+  secretRotationV2Service: Pick<TSecretRotationV2ServiceFactory, "rotateGeneratedCredentials">;
+}) => {
+  const { rotationId, queuedAt, isManualRotation } = job.data;
+  const { retryCount, retryLimit } = job;
+
+  const logDetails = `[rotationId=${rotationId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
+
+  try {
+    const secretRotation = await secretRotationV2DAL.findById(rotationId);
+
+    if (!secretRotation) throw new Error(`Secret rotation ${rotationId} not found`);
+
+    if (!secretRotation.isAutoRotationEnabled) {
+      logger.info(`secretRotationV2Queue: Skipping Rotation - Auto-Rotation Disabled Since Queue ${logDetails}`);
+    }
+
+    if (new Date(secretRotation.lastRotatedAt).getTime() >= new Date(queuedAt).getTime()) {
+      // rotated since being queued, skip rotation
+      logger.info(`secretRotationV2Queue: Skipping Rotation - Rotated Since Queue ${logDetails}`);
+      return;
+    }
+
+    await secretRotationV2Service.rotateGeneratedCredentials(secretRotation, {
+      jobId: job.id,
+      shouldSendNotification: true,
+      isFinalAttempt: retryCount === retryLimit,
+      isManualRotation
+    });
+
+    logger.info(`secretRotationV2Queue: Secrets Rotated ${logDetails}`);
+  } catch (error) {
+    logger.error(error, `secretRotationV2Queue: Failed to Rotate Secrets ${logDetails}`);
+    throw error;
+  }
+};

backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts

@@ -1,9 +1,12 @@
+import { v4 as uuidv4 } from "uuid";
+
 import { ProjectMembershipRole } from "@app/db/schemas";
 import { TSecretRotationV2DALFactory } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-dal";
 import { SecretRotation } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-enums";
 import {
   getNextUtcRotationInterval,
-  getSecretRotationRotateSecretJobOptions
+  getSecretRotationRotateSecretJobOptions,
+  rotateSecretsFns
 } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-fns";
 import { SECRET_ROTATION_NAME_MAP } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-maps";
 import { TSecretRotationV2ServiceFactory } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-service";
@@ -63,14 +66,34 @@ export const secretRotationV2QueueServiceFactory = async ({
               rotation.lastRotatedAt
             ).toISOString()}] [rotateAt=${new Date(rotation.nextRotationAt!).toISOString()}]`
           );
-          await queueService.queuePg(
-            QueueJobs.SecretRotationV2RotateSecrets,
-            {
-              rotationId: rotation.id,
-              queuedAt: currentTime
-            },
-            getSecretRotationRotateSecretJobOptions(rotation)
-          );
+
+          const data = {
+            rotationId: rotation.id,
+            queuedAt: currentTime
+          } as TSecretRotationRotateSecretsJobPayload;
+
+          if (appCfg.isTestMode) {
+            logger.warn("secretRotationV2Queue: Manually rotating secrets for test mode");
+            await rotateSecretsFns({
+              job: {
+                id: uuidv4(),
+                data,
+                retryCount: 0,
+                retryLimit: 0
+              },
+              secretRotationV2DAL,
+              secretRotationV2Service
+            });
+          } else {
+            await queueService.queuePg(
+              QueueJobs.SecretRotationV2RotateSecrets,
+              {
+                rotationId: rotation.id,
+                queuedAt: currentTime
+              },
+              getSecretRotationRotateSecretJobOptions(rotation)
+            );
+          }
         }
       } catch (error) {
         logger.error(error, "secretRotationV2Queue: Queue Rotations Error:");
@@ -87,38 +110,14 @@ export const secretRotationV2QueueServiceFactory = async ({
   await queueService.startPg<QueueName.SecretRotationV2>(
     QueueJobs.SecretRotationV2RotateSecrets,
     async ([job]) => {
-      const { rotationId, queuedAt, isManualRotation } = job.data as TSecretRotationRotateSecretsJobPayload;
-      const { retryCount, retryLimit } = job;
-
-      const logDetails = `[rotationId=${rotationId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
-
-      try {
-        const secretRotation = await secretRotationV2DAL.findById(rotationId);
-
-        if (!secretRotation) throw new Error(`Secret rotation ${rotationId} not found`);
-
-        if (!secretRotation.isAutoRotationEnabled) {
-          logger.info(`secretRotationV2Queue: Skipping Rotation - Auto-Rotation Disabled Since Queue ${logDetails}`);
-        }
-
-        if (new Date(secretRotation.lastRotatedAt).getTime() >= new Date(queuedAt).getTime()) {
-          // rotated since being queued, skip rotation
-          logger.info(`secretRotationV2Queue: Skipping Rotation - Rotated Since Queue ${logDetails}`);
-          return;
-        }
-
-        await secretRotationV2Service.rotateGeneratedCredentials(secretRotation, {
-          jobId: job.id,
-          shouldSendNotification: true,
-          isFinalAttempt: retryCount === retryLimit,
-          isManualRotation
-        });
-
-        logger.info(`secretRotationV2Queue: Secrets Rotated ${logDetails}`);
-      } catch (error) {
-        logger.error(error, `secretRotationV2Queue: Failed to Rotate Secrets ${logDetails}`);
-        throw error;
-      }
+      await rotateSecretsFns({
+        job: {
+          ...job,
+          data: job.data as TSecretRotationRotateSecretsJobPayload
+        },
+        secretRotationV2DAL,
+        secretRotationV2Service
+      });
     },
     {
       batchSize: 1,

backend/src/lib/config/env.ts

@@ -348,7 +348,9 @@ const envSchema = z
     isSmtpConfigured: Boolean(data.SMTP_HOST),
     isRedisConfigured: Boolean(data.REDIS_URL || data.REDIS_SENTINEL_HOSTS),
     isDevelopmentMode: data.NODE_ENV === "development",
-    isRotationDevelopmentMode: data.NODE_ENV === "development" && data.ROTATION_DEVELOPMENT_MODE,
+    isTestMode: data.NODE_ENV === "test",
+    isRotationDevelopmentMode:
+      (data.NODE_ENV === "development" && data.ROTATION_DEVELOPMENT_MODE) || data.NODE_ENV === "test",
     isDailyResourceCleanUpDevelopmentMode:
       data.NODE_ENV === "development" && data.DAILY_RESOURCE_CLEAN_UP_DEVELOPMENT_MODE,
     isProductionMode: data.NODE_ENV === "production" || IS_PACKAGED,

backend/vitest.e2e.config.ts

@@ -5,7 +5,10 @@ export default defineConfig({
   test: {
     globals: true,
     env: {
-      NODE_ENV: "test"
+      NODE_ENV: "test",
+      E2E_TEST_ORACLE_DB_19_HOST: process.env.E2E_TEST_ORACLE_DB_19_HOST!,
+      E2E_TEST_ORACLE_DB_19_USERNAME: process.env.E2E_TEST_ORACLE_DB_19_USERNAME!,
+      E2E_TEST_ORACLE_DB_19_PASSWORD: process.env.E2E_TEST_ORACLE_DB_19_PASSWORD!
     },
     environment: "./e2e-test/vitest-environment-knex.ts",
     include: ["./e2e-test/**/*.spec.ts"],

docker-compose.e2e-dbs.yml

@@ -0,0 +1,157 @@
+version: '3.8'
+
+services:
+  # Oracle Databases
+  oracle-db-23.8:
+    image: container-registry.oracle.com/database/free:23.8.0.0
+    container_name: oracle-db-23.8
+    ports:
+      - "1521:1521"
+    environment:
+      - ORACLE_PDB=pdb
+      - ORACLE_PWD=pdb-password
+    volumes:
+      - oracle-data-23.8:/opt/oracle/oradata
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "sqlplus", "-L", "system/pdb-password@//localhost:1521/FREEPDB1", "<<<", "SELECT 1 FROM DUAL;"]
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 30s
+
+  # MySQL Databases
+  mysql-8.4.6:
+    image: mysql:8.4.6
+    container_name: mysql-8.4.6
+    ports:
+      - "3306:3306"
+    environment:
+      - MYSQL_ROOT_PASSWORD=mysql-test
+      - MYSQL_DATABASE=mysql-test
+      - MYSQL_ROOT_HOST=% 
+      - MYSQL_USER=mysql-test
+      - MYSQL_PASSWORD=mysql-test
+    volumes:
+      - mysql-data-8.4.6:/var/lib/mysql
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "mysql-test", "-pmysql-test"]
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 30s
+
+  mysql-8.0.29:
+    image: mysql:8.0.29
+    container_name: mysql-8.0.28
+    ports:
+      - "3307:3306"
+    environment:
+      - MYSQL_ROOT_PASSWORD=mysql-test
+      - MYSQL_DATABASE=mysql-test
+      - MYSQL_ROOT_HOST=%
+      - MYSQL_USER=mysql-test
+      - MYSQL_PASSWORD=mysql-test
+    volumes:
+      - mysql-data-8.0.29:/var/lib/mysql
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "mysql-test", "-pmysql-test"]
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 30s
+
+  mysql-5.7.31:
+    image: mysql:5.7.31
+    container_name: mysql-5.7.31
+    platform: linux/amd64
+    ports:
+      - "3308:3306"
+    environment:
+      - MYSQL_ROOT_PASSWORD=mysql-test
+      - MYSQL_DATABASE=mysql-test
+      - MYSQL_ROOT_HOST=%
+      - MYSQL_USER=mysql-test
+      - MYSQL_PASSWORD=mysql-test
+    volumes:
+      - mysql-data-5.7.31:/var/lib/mysql
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "mysql-test", "-pmysql-test"]
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 30s
+
+
+  # PostgreSQL Databases
+  postgres-17:
+    image: postgres:17
+    platform: linux/amd64
+    container_name: postgres-17
+    ports:
+      - "5433:5432"
+    environment:
+      - POSTGRES_DB=postgres-test
+      - POSTGRES_USER=postgres-test
+      - POSTGRES_PASSWORD=postgres-test
+    volumes:
+      - postgres-data-17:/var/lib/postgresql/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres-test -d postgres-test"]
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 30s
+
+  postgres-16:
+    image: postgres:16
+    platform: linux/amd64
+    container_name: postgres-16
+    ports:
+      - "5434:5432"
+    environment:
+      - POSTGRES_DB=postgres-test
+      - POSTGRES_USER=postgres-test
+      - POSTGRES_PASSWORD=postgres-test
+    volumes:
+      - postgres-data-16:/var/lib/postgresql/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres-test -d postgres-test"]
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 30s
+
+  postgres-10.12:
+    image: postgres:10.12
+    platform: linux/amd64
+    container_name: postgres-10.12
+    ports:
+      - "5435:5432"
+    environment:
+      - POSTGRES_DB=postgres-test
+      - POSTGRES_USER=postgres-test
+      - POSTGRES_PASSWORD=postgres-test
+    volumes:
+      - postgres-data-10.12:/var/lib/postgresql/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres-test -d postgres-test"]
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 30s
+
+volumes:
+  oracle-data-23.8:
+  mysql-data-8.4.6:
+  mysql-data-8.0.29:
+  mysql-data-5.7.31:
+  postgres-data-17:
+  postgres-data-16:
+  postgres-data-10.12: