From 33af2fb2b8809820957ab090d93f00aeee47bf90 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Wed, 6 Aug 2025 03:28:28 +0400 Subject: [PATCH 01/20] feaet(e2e-tests): secret rotation tests --- .github/workflows/run-backend-tests.yml | 2 + backend/e2e-test/mocks/queue.ts | 34 -- .../routes/v3/secret-rotations.spec.ts | 442 ++++++++++++++++++ backend/e2e-test/vitest-environment-knex.ts | 11 + .../dynamic-secret/dynamic-secret-fns.ts | 2 +- .../services/license/__mocks__/license-fns.ts | 2 +- .../secret-rotation-v2-fns.ts | 57 ++- .../secret-rotation-v2-queue.ts | 65 ++- backend/src/lib/config/env.ts | 4 +- backend/vitest.e2e.config.ts | 12 +- docker-compose.e2e-dbs.yml | 43 ++ 11 files changed, 599 insertions(+), 75 deletions(-) delete mode 100644 backend/e2e-test/mocks/queue.ts create mode 100644 backend/e2e-test/routes/v3/secret-rotations.spec.ts create mode 100644 docker-compose.e2e-dbs.yml diff --git a/.github/workflows/run-backend-tests.yml b/.github/workflows/run-backend-tests.yml index f2ba04e76..0ca20311b 100644 --- a/.github/workflows/run-backend-tests.yml +++ b/.github/workflows/run-backend-tests.yml @@ -34,6 +34,8 @@ jobs: working-directory: backend - name: Start postgres and redis run: touch .env && docker compose -f docker-compose.dev.yml up -d db redis + - name: Start Secret Rotation testing databases + run: docker compose -f docker-compose.e2e-dbs.yml up -d - name: Run unit test run: npm run test:unit working-directory: backend diff --git a/backend/e2e-test/mocks/queue.ts b/backend/e2e-test/mocks/queue.ts deleted file mode 100644 index 04a78bcd1..000000000 --- a/backend/e2e-test/mocks/queue.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { TQueueServiceFactory } from "@app/queue"; - -export const mockQueue = (): TQueueServiceFactory => { - const queues: Record = {}; - const workers: Record = {}; - const job: Record = {}; - const events: Record = {}; - - return { - queue: async (name, jobData) => { - job[name] = jobData; - }, - queuePg: async () => {}, - schedulePg: async () => {}, - initialize: async () => {}, - shutdown: async () => undefined, - stopRepeatableJob: async () => true, - start: (name, jobFn) => { - queues[name] = jobFn; - workers[name] = jobFn; - }, - startPg: async () => {}, - listen: (name, event) => { - events[name] = event; - }, - getRepeatableJobs: async () => [], - getDelayedJobs: async () => [], - clearQueue: async () => {}, - stopJobById: async () => {}, - stopJobByIdPg: async () => {}, - stopRepeatableJobByJobId: async () => true, - stopRepeatableJobByKey: async () => true - }; -}; diff --git a/backend/e2e-test/routes/v3/secret-rotations.spec.ts b/backend/e2e-test/routes/v3/secret-rotations.spec.ts new file mode 100644 index 000000000..e202ad089 --- /dev/null +++ b/backend/e2e-test/routes/v3/secret-rotations.spec.ts @@ -0,0 +1,442 @@ +/* eslint-disable no-promise-executor-return */ +/* eslint-disable no-await-in-loop */ +import knex from "knex"; +import { v4 as uuidv4 } from "uuid"; + +import { seedData1 } from "@app/db/seed-data"; + +enum SecretRotationType { + OracleDb = "oracledb", + MySQL = "mysql" +} + +type TGenericSqlCredentials = { + host: string; + port: number; + username: string; + password: string; + database: string; +}; + +type TSecretMapping = { + username: string; + password: string; +}; + +type TDatabaseUserCredentials = { + username: string; +}; + +const formatSqlUsername = (username: string) => `${username}_${uuidv4().slice(0, 8).replace(/-/g, "").toUpperCase()}`; + +const getSecretValue = async (secretKey: string) => { + const passwordSecret = await testServer.inject({ + url: `/api/v3/secrets/raw/${secretKey}`, + method: "GET", + query: { + workspaceId: seedData1.projectV3.id, + environment: seedData1.environment.slug + }, + headers: { + authorization: `Bearer ${jwtAuthToken}` + } + }); + + expect(passwordSecret.statusCode).toBe(200); + expect(passwordSecret.json().secret).toBeDefined(); + + const passwordSecretJson = JSON.parse(passwordSecret.payload); + + return passwordSecretJson.secret.secretValue as string; +}; + +const deleteSecretRotation = async (id: string, type: SecretRotationType) => { + const res = await testServer.inject({ + method: "DELETE", + query: { + deleteSecrets: "true", + revokeGeneratedCredentials: "true" + }, + url: `/api/v2/secret-rotations/${type}-credentials/${id}`, + headers: { + authorization: `Bearer ${jwtAuthToken}` + } + }); + + expect(res.statusCode).toBe(200); +}; + +const deleteAppConnection = async (id: string, type: SecretRotationType) => { + const res = await testServer.inject({ + method: "DELETE", + url: `/api/v1/app-connections/${type}/${id}`, + headers: { + authorization: `Bearer ${jwtAuthToken}` + } + }); + + expect(res.statusCode).toBe(200); +}; + +const createOracleDBAppConnection = async (credentials: TGenericSqlCredentials) => { + const createOracleDBAppConnectionReqBody = { + credentials: { + database: credentials.database, + host: credentials.host, + username: credentials.username, + password: credentials.password, + port: credentials.port, + sslEnabled: true, + sslRejectUnauthorized: true + }, + name: `oracle-db-${uuidv4()}`, + description: "Test OracleDB App Connection", + gatewayId: null, + isPlatformManagedCredentials: false, + method: "username-and-password" + }; + + const res = await testServer.inject({ + method: "POST", + url: `/api/v1/app-connections/oracledb`, + headers: { + authorization: `Bearer ${jwtAuthToken}` + }, + body: createOracleDBAppConnectionReqBody + }); + + const json = JSON.parse(res.payload); + + expect(res.statusCode).toBe(200); + expect(json.appConnection).toBeDefined(); + + return json.appConnection.id as string; +}; + +const createMySQLAppConnection = async (credentials: TGenericSqlCredentials) => { + const createMySQLAppConnectionReqBody = { + name: `mysql-test-${uuidv4()}`, + description: "test-mysql", + gatewayId: null, + method: "username-and-password", + credentials: { + host: credentials.host, + port: credentials.port, + database: credentials.database, + username: credentials.username, + password: credentials.password, + sslEnabled: false, + sslRejectUnauthorized: true + } + }; + + const res = await testServer.inject({ + method: "POST", + url: `/api/v1/app-connections/mysql`, + headers: { + authorization: `Bearer ${jwtAuthToken}` + }, + body: createMySQLAppConnectionReqBody + }); + + const json = JSON.parse(res.payload); + + expect(res.statusCode).toBe(200); + expect(json.appConnection).toBeDefined(); + + return json.appConnection.id as string; +}; + +const createOracleInfisicalUsers = async ( + credentials: TGenericSqlCredentials, + userCredentials: TDatabaseUserCredentials[] +) => { + const client = knex({ + client: "oracledb", + connection: { + database: credentials.database, + port: credentials.port, + host: credentials.host, + user: credentials.username, + password: credentials.password, + connectionTimeoutMillis: 10000, + ssl: { + // @ts-expect-error - this is a valid property for the ssl object + sslServerDNMatch: true + } + } + }); + + for await (const { username } of userCredentials) { + // check if user exists, and if it does, don't create it + const existingUser = await client.raw(`SELECT * FROM all_users WHERE username = '${username.toUpperCase()}'`); + + if (!existingUser.length) { + await client.raw(`CREATE USER ${username} IDENTIFIED BY "temporary_password"`); + } + await client.raw(`GRANT ALL PRIVILEGES TO ${username} WITH ADMIN OPTION`); + } + + await client.destroy(); +}; + +const createMySQLInfisicalUsers = async ( + credentials: TGenericSqlCredentials, + userCredentials: TDatabaseUserCredentials[] +) => { + const client = knex({ + client: "mysql2", + connection: { + database: credentials.database, + port: credentials.port, + host: credentials.host, + user: credentials.username, + password: credentials.password, + connectionTimeoutMillis: 10000 + } + }); + + // Fix: Ensure root has GRANT OPTION privileges + try { + await client.raw("GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION;"); + await client.raw("FLUSH PRIVILEGES;"); + } catch (error) { + // Ignore if already has privileges + } + + for await (const { username } of userCredentials) { + // check if user exists, and if it does, dont create it + + const existingUser = await client.raw(`SELECT * FROM mysql.user WHERE user = '${username}'`); + + if (!existingUser[0].length) { + await client.raw(`CREATE USER '${username}'@'%' IDENTIFIED BY 'temporary_password';`); + } + + await client.raw(`GRANT ALL PRIVILEGES ON \`${credentials.database}\`.* TO '${username}'@'%';`); + await client.raw("FLUSH PRIVILEGES;"); + } + + await client.destroy(); +}; + +const createOracleDBSecretRotation = async ( + appConnectionId: string, + credentials: TGenericSqlCredentials, + userCredentials: TDatabaseUserCredentials[], + secretMapping: TSecretMapping +) => { + const now = new Date(); + const rotationTime = new Date(now.getTime() - 2 * 60 * 1000); // 2 minutes ago + + await createOracleInfisicalUsers(credentials, userCredentials); + + const createOracleDBSecretRotationReqBody = { + parameters: userCredentials.reduce( + (acc, user, index) => { + acc[`username${index + 1}`] = user.username; + return acc; + }, + {} as Record + ), + secretsMapping: { + username: secretMapping.username, + password: secretMapping.password + }, + name: `test-oracle-${uuidv4()}`, + description: "Test OracleDB Secret Rotation", + secretPath: "/", + isAutoRotationEnabled: true, + rotationInterval: 5, // 5 seconds for testing + rotateAtUtc: { + hours: rotationTime.getUTCHours(), + minutes: rotationTime.getUTCMinutes() + }, + connectionId: appConnectionId, + environment: seedData1.environment.slug, + projectId: seedData1.projectV3.id + }; + + const res = await testServer.inject({ + method: "POST", + url: `/api/v2/secret-rotations/oracledb-credentials`, + headers: { + authorization: `Bearer ${jwtAuthToken}` + }, + body: createOracleDBSecretRotationReqBody + }); + + expect(res.statusCode).toBe(200); + expect(res.json().secretRotation).toBeDefined(); + + return res; +}; + +const createMySQLSecretRotation = async ( + appConnectionId: string, + credentials: TGenericSqlCredentials, + userCredentials: TDatabaseUserCredentials[], + secretMapping: TSecretMapping +) => { + const now = new Date(); + const rotationTime = new Date(now.getTime() - 2 * 60 * 1000); // 2 minutes ago + + await createMySQLInfisicalUsers(credentials, userCredentials); + + const createMySQLSecretRotationReqBody = { + parameters: userCredentials.reduce( + (acc, user, index) => { + acc[`username${index + 1}`] = user.username; + return acc; + }, + {} as Record + ), + secretsMapping: { + username: secretMapping.username, + password: secretMapping.password + }, + name: `test-mysql-rotation-${uuidv4()}`, + description: "Test MySQL Secret Rotation", + secretPath: "/", + isAutoRotationEnabled: true, + rotationInterval: 5, + rotateAtUtc: { + hours: rotationTime.getUTCHours(), + minutes: rotationTime.getUTCMinutes() + }, + connectionId: appConnectionId, + environment: seedData1.environment.slug, + projectId: seedData1.projectV3.id + }; + + const res = await testServer.inject({ + method: "POST", + url: `/api/v2/secret-rotations/mysql-credentials`, + headers: { + authorization: `Bearer ${jwtAuthToken}` + }, + body: createMySQLSecretRotationReqBody + }); + + expect(res.statusCode).toBe(200); + expect(res.json().secretRotation).toBeDefined(); + + return res; +}; + +describe("Secret Rotations", async () => { + const testCases = [ + { + type: SecretRotationType.MySQL, + name: "MySQL (8.4.6) Secret Rotation", + dbCredentials: { + database: "mysql-test", + host: "127.0.0.1", + username: "root", + password: "mysql-test", + port: 3306 + }, + secretMapping: { + username: formatSqlUsername("MYSQL_USERNAME"), + password: formatSqlUsername("MYSQL_PASSWORD") + }, + userCredentials: [ + { + username: formatSqlUsername("MYSQL_USER_1") + }, + { + username: formatSqlUsername("MYSQL_USER_2") + } + ] + }, + { + type: SecretRotationType.OracleDb, + name: "OracleDB (23.8) Secret Rotation", + dbCredentials: { + database: "FREEPDB1", + host: "127.0.0.1", + username: "system", + password: "pdb-password", + port: 1521 + }, + secretMapping: { + username: formatSqlUsername("ORACLEDB_USERNAME"), + password: formatSqlUsername("ORACLEDB_PASSWORD") + }, + userCredentials: [ + { + username: formatSqlUsername("INFISICAL_USER_1") + }, + { + username: formatSqlUsername("INFISICAL_USER_2") + } + ] + } + ] as { + type: SecretRotationType; + name: string; + dbCredentials: TGenericSqlCredentials; + secretMapping: TSecretMapping; + userCredentials: TDatabaseUserCredentials[]; + }[]; + + const createAppConnectionMap = { + [SecretRotationType.OracleDb]: createOracleDBAppConnection, + [SecretRotationType.MySQL]: createMySQLAppConnection + }; + + const createRotationMap = { + [SecretRotationType.OracleDb]: createOracleDBSecretRotation, + [SecretRotationType.MySQL]: createMySQLSecretRotation + }; + + const appConnectionIds: { id: string; type: SecretRotationType }[] = []; + const secretRotationIds: { id: string; type: SecretRotationType }[] = []; + + afterAll(async () => { + for (const { id, type } of secretRotationIds) { + await deleteSecretRotation(id, type); + } + + for (const { id, type } of appConnectionIds) { + await deleteAppConnection(id, type); + } + }); + + test.concurrent.each(testCases)( + "Create secret rotation for $name", + async ({ dbCredentials, secretMapping, userCredentials, type }) => { + const appConnectionId = await createAppConnectionMap[type](dbCredentials); + + if (appConnectionId) { + appConnectionIds.push({ id: appConnectionId, type }); + } + + const res = await createRotationMap[type](appConnectionId, dbCredentials, userCredentials, secretMapping); + + const resJson = JSON.parse(res.payload); + + if (resJson.secretRotation) { + secretRotationIds.push({ id: resJson.secretRotation.id, type }); + } + + const startSecretValue = await getSecretValue(secretMapping.password); + expect(startSecretValue).toBeDefined(); + + let attempts = 0; + while (attempts < 60) { + const currentSecretValue = await getSecretValue(secretMapping.password); + + if (currentSecretValue !== startSecretValue) { + break; + } + + attempts += 1; + await new Promise((resolve) => setTimeout(resolve, 2_500)); + } + }, + { + timeout: 300_000 + } + ); +}); diff --git a/backend/e2e-test/vitest-environment-knex.ts b/backend/e2e-test/vitest-environment-knex.ts index 60e70d379..7d227720f 100644 --- a/backend/e2e-test/vitest-environment-knex.ts +++ b/backend/e2e-test/vitest-environment-knex.ts @@ -18,6 +18,7 @@ import { keyStoreFactory } from "@app/keystore/keystore"; import { initializeHsmModule } from "@app/ee/services/hsm/hsm-fns"; import { buildRedisFromConfig } from "@app/lib/config/redis"; import { superAdminDALFactory } from "@app/services/super-admin/super-admin-dal"; +import { bootstrapCheck } from "@app/server/boot-strap-check"; dotenv.config({ path: path.join(__dirname, "../../.env.test"), debug: true }); export default { @@ -63,6 +64,8 @@ export default { const queue = queueServiceFactory(envCfg, { dbConnectionUrl: envCfg.DB_CONNECTION_URI }); const keyStore = keyStoreFactory(envCfg); + await queue.initialize(); + const hsmModule = initializeHsmModule(envCfg); hsmModule.initialize(); @@ -78,9 +81,13 @@ export default { envConfig: envCfg }); + await bootstrapCheck({ db }); + // @ts-expect-error type globalThis.testServer = server; // @ts-expect-error type + globalThis.testQueue = queue; + // @ts-expect-error type globalThis.testSuperAdminDAL = superAdminDAL; // @ts-expect-error type globalThis.jwtAuthToken = crypto.jwt().sign( @@ -105,6 +112,8 @@ export default { // custom setup return { async teardown() { + // @ts-expect-error type + await globalThis.testQueue.shutdown(); // @ts-expect-error type await globalThis.testServer.close(); // @ts-expect-error type @@ -113,6 +122,8 @@ export default { delete globalThis.testSuperAdminDAL; // @ts-expect-error type delete globalThis.jwtToken; + // @ts-expect-error type + delete globalThis.testQueue; // called after all tests with this env have been run await db.migrate.rollback( { diff --git a/backend/src/ee/services/dynamic-secret/dynamic-secret-fns.ts b/backend/src/ee/services/dynamic-secret/dynamic-secret-fns.ts index 3b405a418..e9fc2b6e5 100644 --- a/backend/src/ee/services/dynamic-secret/dynamic-secret-fns.ts +++ b/backend/src/ee/services/dynamic-secret/dynamic-secret-fns.ts @@ -9,7 +9,7 @@ import { getDbConnectionHost } from "@app/lib/knex"; export const verifyHostInputValidity = async (host: string, isGateway = false) => { const appCfg = getConfig(); - if (appCfg.isDevelopmentMode) return [host]; + if (appCfg.isDevelopmentMode || appCfg.isTestMode) return [host]; if (isGateway) return [host]; diff --git a/backend/src/ee/services/license/__mocks__/license-fns.ts b/backend/src/ee/services/license/__mocks__/license-fns.ts index 5259d4616..e2b67b7b5 100644 --- a/backend/src/ee/services/license/__mocks__/license-fns.ts +++ b/backend/src/ee/services/license/__mocks__/license-fns.ts @@ -31,7 +31,7 @@ export const getDefaultOnPremFeatures = () => { caCrl: false, sshHostGroups: false, enterpriseSecretSyncs: false, - enterpriseAppConnections: false + enterpriseAppConnections: true }; }; diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-fns.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-fns.ts index 7c0239add..1e5086f3c 100644 --- a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-fns.ts +++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-fns.ts @@ -14,14 +14,17 @@ import { OKTA_CLIENT_SECRET_ROTATION_LIST_OPTION } from "./okta-client-secret"; import { ORACLEDB_CREDENTIALS_ROTATION_LIST_OPTION } from "./oracledb-credentials"; import { POSTGRES_CREDENTIALS_ROTATION_LIST_OPTION } from "./postgres-credentials"; import { SecretRotation, SecretRotationStatus } from "./secret-rotation-v2-enums"; -import { TSecretRotationV2ServiceFactoryDep } from "./secret-rotation-v2-service"; +import { TSecretRotationV2ServiceFactory, TSecretRotationV2ServiceFactoryDep } from "./secret-rotation-v2-service"; import { + TSecretRotationRotateSecretsJobPayload, TSecretRotationV2, TSecretRotationV2GeneratedCredentials, TSecretRotationV2ListItem, TSecretRotationV2Raw, TUpdateSecretRotationV2DTO } from "./secret-rotation-v2-types"; +import { logger } from "@app/lib/logger"; +import { TSecretRotationV2DALFactory } from "./secret-rotation-v2-dal"; const SECRET_ROTATION_LIST_OPTIONS: Record = { [SecretRotation.PostgresCredentials]: POSTGRES_CREDENTIALS_ROTATION_LIST_OPTION, @@ -74,6 +77,10 @@ export const getNextUtcRotationInterval = (rotateAtUtc?: TSecretRotationV2["rota const appCfg = getConfig(); if (appCfg.isRotationDevelopmentMode) { + if (appCfg.isTestMode) { + // if its test mode, it should always rotate + return new Date(Date.now() + 365 * 24 * 60 * 60 * 1000); // Current time + 1 year + } return getNextUTCMinuteInterval(rotateAtUtc); } @@ -263,3 +270,51 @@ export const throwOnImmutableParameterUpdate = ( // do nothing } }; + +export const rotateSecretsFns = async ({ + job, + secretRotationV2DAL, + secretRotationV2Service +}: { + job: { + data: TSecretRotationRotateSecretsJobPayload; + id: string; + retryCount: number; + retryLimit: number; + }; + secretRotationV2DAL: Pick; + secretRotationV2Service: Pick; +}) => { + const { rotationId, queuedAt, isManualRotation } = job.data; + const { retryCount, retryLimit } = job; + + const logDetails = `[rotationId=${rotationId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`; + + try { + const secretRotation = await secretRotationV2DAL.findById(rotationId); + + if (!secretRotation) throw new Error(`Secret rotation ${rotationId} not found`); + + if (!secretRotation.isAutoRotationEnabled) { + logger.info(`secretRotationV2Queue: Skipping Rotation - Auto-Rotation Disabled Since Queue ${logDetails}`); + } + + if (new Date(secretRotation.lastRotatedAt).getTime() >= new Date(queuedAt).getTime()) { + // rotated since being queued, skip rotation + logger.info(`secretRotationV2Queue: Skipping Rotation - Rotated Since Queue ${logDetails}`); + return; + } + + await secretRotationV2Service.rotateGeneratedCredentials(secretRotation, { + jobId: job.id, + shouldSendNotification: true, + isFinalAttempt: retryCount === retryLimit, + isManualRotation + }); + + logger.info(`secretRotationV2Queue: Secrets Rotated ${logDetails}`); + } catch (error) { + logger.error(error, `secretRotationV2Queue: Failed to Rotate Secrets ${logDetails}`); + throw error; + } +}; diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts index 765ca3ab3..5a5f043d2 100644 --- a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts +++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts @@ -1,9 +1,12 @@ +import { v4 as uuidv4 } from "uuid"; + import { ProjectMembershipRole } from "@app/db/schemas"; import { TSecretRotationV2DALFactory } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-dal"; import { SecretRotation } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-enums"; import { getNextUtcRotationInterval, - getSecretRotationRotateSecretJobOptions + getSecretRotationRotateSecretJobOptions, + rotateSecretsFns } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-fns"; import { SECRET_ROTATION_NAME_MAP } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-maps"; import { TSecretRotationV2ServiceFactory } from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-service"; @@ -63,6 +66,26 @@ export const secretRotationV2QueueServiceFactory = async ({ rotation.lastRotatedAt ).toISOString()}] [rotateAt=${new Date(rotation.nextRotationAt!).toISOString()}]` ); + + const data = { + rotationId: rotation.id, + queuedAt: currentTime + } as TSecretRotationRotateSecretsJobPayload; + + if (appCfg.isTestMode) { + logger.warn("secretRotationV2Queue: Manually rotating secrets for test mode"); + await rotateSecretsFns({ + job: { + id: uuidv4(), + data, + retryCount: 0, + retryLimit: 0 + }, + secretRotationV2DAL, + secretRotationV2Service + }); + } + await queueService.queuePg( QueueJobs.SecretRotationV2RotateSecrets, { @@ -87,38 +110,14 @@ export const secretRotationV2QueueServiceFactory = async ({ await queueService.startPg( QueueJobs.SecretRotationV2RotateSecrets, async ([job]) => { - const { rotationId, queuedAt, isManualRotation } = job.data as TSecretRotationRotateSecretsJobPayload; - const { retryCount, retryLimit } = job; - - const logDetails = `[rotationId=${rotationId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`; - - try { - const secretRotation = await secretRotationV2DAL.findById(rotationId); - - if (!secretRotation) throw new Error(`Secret rotation ${rotationId} not found`); - - if (!secretRotation.isAutoRotationEnabled) { - logger.info(`secretRotationV2Queue: Skipping Rotation - Auto-Rotation Disabled Since Queue ${logDetails}`); - } - - if (new Date(secretRotation.lastRotatedAt).getTime() >= new Date(queuedAt).getTime()) { - // rotated since being queued, skip rotation - logger.info(`secretRotationV2Queue: Skipping Rotation - Rotated Since Queue ${logDetails}`); - return; - } - - await secretRotationV2Service.rotateGeneratedCredentials(secretRotation, { - jobId: job.id, - shouldSendNotification: true, - isFinalAttempt: retryCount === retryLimit, - isManualRotation - }); - - logger.info(`secretRotationV2Queue: Secrets Rotated ${logDetails}`); - } catch (error) { - logger.error(error, `secretRotationV2Queue: Failed to Rotate Secrets ${logDetails}`); - throw error; - } + await rotateSecretsFns({ + job: { + ...job, + data: job.data as TSecretRotationRotateSecretsJobPayload + }, + secretRotationV2DAL, + secretRotationV2Service + }); }, { batchSize: 1, diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 9ff7339c0..ce40574e0 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -345,7 +345,9 @@ const envSchema = z isSmtpConfigured: Boolean(data.SMTP_HOST), isRedisConfigured: Boolean(data.REDIS_URL || data.REDIS_SENTINEL_HOSTS), isDevelopmentMode: data.NODE_ENV === "development", - isRotationDevelopmentMode: data.NODE_ENV === "development" && data.ROTATION_DEVELOPMENT_MODE, + isTestMode: data.NODE_ENV === "test", + isRotationDevelopmentMode: + (data.NODE_ENV === "development" && data.ROTATION_DEVELOPMENT_MODE) || data.NODE_ENV === "test", isProductionMode: data.NODE_ENV === "production" || IS_PACKAGED, isRedisSentinelMode: Boolean(data.REDIS_SENTINEL_HOSTS), REDIS_SENTINEL_HOSTS: data.REDIS_SENTINEL_HOSTS?.trim() diff --git a/backend/vitest.e2e.config.ts b/backend/vitest.e2e.config.ts index 684a4dc42..3e59790ce 100644 --- a/backend/vitest.e2e.config.ts +++ b/backend/vitest.e2e.config.ts @@ -8,14 +8,18 @@ export default defineConfig({ NODE_ENV: "test" }, environment: "./e2e-test/vitest-environment-knex.ts", - include: ["./e2e-test/**/*.spec.ts"], + include: ["./e2e-test/**/secret-rotations.spec.ts"], + + pool: "forks", poolOptions: { - threads: { - singleThread: true, - useAtomics: true, + forks: { + singleFork: true, isolate: false } }, + isolate: false, + fileParallelism: false, + alias: { "./license-fns": path.resolve(__dirname, "./src/ee/services/license/__mocks__/license-fns") } diff --git a/docker-compose.e2e-dbs.yml b/docker-compose.e2e-dbs.yml new file mode 100644 index 000000000..1e3777e10 --- /dev/null +++ b/docker-compose.e2e-dbs.yml @@ -0,0 +1,43 @@ +version: '3.8' + +services: + oracle-db-23.8: + image: container-registry.oracle.com/database/free:23.8.0.0 + container_name: oracle-db-23.8 + ports: + - "1521:1521" + environment: + - ORACLE_PDB=pdb + - ORACLE_PWD=pdb-password + volumes: + - oracle-data-23.8:/opt/oracle/oradata + restart: unless-stopped + healthcheck: + test: ["CMD", "sqlplus", "-L", "system/pdb-password@//localhost:1521/FREEPDB1", "<<<", "SELECT 1 FROM DUAL;"] + interval: 30s + timeout: 10s + retries: 5 + + mysql-8.4.6: + image: mysql:8.4.6 + container_name: mysql-8.4.6 + ports: + - "3306:3306" + environment: + - MYSQL_ROOT_PASSWORD=mysql-test + - MYSQL_DATABASE=mysql-test + - MYSQL_ROOT_HOST=% + - MYSQL_USER=mysql-test + - MYSQL_PASSWORD=mysql-test + volumes: + - mysql-data-8.4.6:/var/lib/mysql + restart: unless-stopped + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "mysql-test", "-pmysql-test"] + interval: 30s + timeout: 10s + retries: 5 + +volumes: + oracle-data-23.8: + mysql-data-8.4.6: \ No newline at end of file From d7f3892b73b1e2a84f1aee8bc2f2418ba0642f2a Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Wed, 6 Aug 2025 03:29:13 +0400 Subject: [PATCH 02/20] Update vitest.e2e.config.ts --- backend/vitest.e2e.config.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/backend/vitest.e2e.config.ts b/backend/vitest.e2e.config.ts index 3e59790ce..684a4dc42 100644 --- a/backend/vitest.e2e.config.ts +++ b/backend/vitest.e2e.config.ts @@ -8,18 +8,14 @@ export default defineConfig({ NODE_ENV: "test" }, environment: "./e2e-test/vitest-environment-knex.ts", - include: ["./e2e-test/**/secret-rotations.spec.ts"], - - pool: "forks", + include: ["./e2e-test/**/*.spec.ts"], poolOptions: { - forks: { - singleFork: true, + threads: { + singleThread: true, + useAtomics: true, isolate: false } }, - isolate: false, - fileParallelism: false, - alias: { "./license-fns": path.resolve(__dirname, "./src/ee/services/license/__mocks__/license-fns") } From 5c63955fdec3aa589630cc1a74e5e9f201a00c46 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Wed, 6 Aug 2025 03:39:08 +0400 Subject: [PATCH 03/20] requested changes --- backend/e2e-test/routes/v3/secret-rotations.spec.ts | 7 +++++++ backend/e2e-test/vitest-environment-knex.ts | 2 +- .../services/secret-rotation-v2/secret-rotation-v2-fns.ts | 4 ++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/backend/e2e-test/routes/v3/secret-rotations.spec.ts b/backend/e2e-test/routes/v3/secret-rotations.spec.ts index e202ad089..f43f47c02 100644 --- a/backend/e2e-test/routes/v3/secret-rotations.spec.ts +++ b/backend/e2e-test/routes/v3/secret-rotations.spec.ts @@ -434,6 +434,13 @@ describe("Secret Rotations", async () => { attempts += 1; await new Promise((resolve) => setTimeout(resolve, 2_500)); } + + if (attempts >= 60) { + throw new Error("Secret rotation failed to rotate after 60 attempts"); + } + + const finalSecretValue = await getSecretValue(secretMapping.password); + expect(finalSecretValue).not.toBe(startSecretValue); }, { timeout: 300_000 diff --git a/backend/e2e-test/vitest-environment-knex.ts b/backend/e2e-test/vitest-environment-knex.ts index 7d227720f..ff5f42286 100644 --- a/backend/e2e-test/vitest-environment-knex.ts +++ b/backend/e2e-test/vitest-environment-knex.ts @@ -121,7 +121,7 @@ export default { // @ts-expect-error type delete globalThis.testSuperAdminDAL; // @ts-expect-error type - delete globalThis.jwtToken; + delete globalThis.jwtAuthToken; // @ts-expect-error type delete globalThis.testQueue; // called after all tests with this env have been run diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-fns.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-fns.ts index 1e5086f3c..4d8cea6a3 100644 --- a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-fns.ts +++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-fns.ts @@ -2,6 +2,7 @@ import { AxiosError } from "axios"; import { getConfig } from "@app/lib/config/env"; import { BadRequestError } from "@app/lib/errors"; +import { logger } from "@app/lib/logger"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { AUTH0_CLIENT_SECRET_ROTATION_LIST_OPTION } from "./auth0-client-secret"; @@ -13,6 +14,7 @@ import { MYSQL_CREDENTIALS_ROTATION_LIST_OPTION } from "./mysql-credentials"; import { OKTA_CLIENT_SECRET_ROTATION_LIST_OPTION } from "./okta-client-secret"; import { ORACLEDB_CREDENTIALS_ROTATION_LIST_OPTION } from "./oracledb-credentials"; import { POSTGRES_CREDENTIALS_ROTATION_LIST_OPTION } from "./postgres-credentials"; +import { TSecretRotationV2DALFactory } from "./secret-rotation-v2-dal"; import { SecretRotation, SecretRotationStatus } from "./secret-rotation-v2-enums"; import { TSecretRotationV2ServiceFactory, TSecretRotationV2ServiceFactoryDep } from "./secret-rotation-v2-service"; import { @@ -23,8 +25,6 @@ import { TSecretRotationV2Raw, TUpdateSecretRotationV2DTO } from "./secret-rotation-v2-types"; -import { logger } from "@app/lib/logger"; -import { TSecretRotationV2DALFactory } from "./secret-rotation-v2-dal"; const SECRET_ROTATION_LIST_OPTIONS: Record = { [SecretRotation.PostgresCredentials]: POSTGRES_CREDENTIALS_ROTATION_LIST_OPTION, From 1b32de5c5bf0396ab7f24063a5d1b16280a9e683 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Wed, 6 Aug 2025 03:46:37 +0400 Subject: [PATCH 04/20] Update license-fns.ts --- backend/src/ee/services/license/__mocks__/license-fns.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/ee/services/license/__mocks__/license-fns.ts b/backend/src/ee/services/license/__mocks__/license-fns.ts index 2ae3f92ba..729696baa 100644 --- a/backend/src/ee/services/license/__mocks__/license-fns.ts +++ b/backend/src/ee/services/license/__mocks__/license-fns.ts @@ -31,7 +31,7 @@ export const getDefaultOnPremFeatures = () => { caCrl: false, sshHostGroups: false, enterpriseSecretSyncs: false, - enterpriseAppConnections: true + enterpriseAppConnections: true, machineIdentityAuthTemplates: false }; }; From 88fcbcadd488f6f5b745ecf8b885a0c1957dcb29 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Thu, 7 Aug 2025 00:41:51 +0400 Subject: [PATCH 05/20] feat(e2e-tests): secret rotations --- .github/workflows/run-backend-tests.yml | 3 + .../routes/v3/secret-rotations.spec.ts | 265 +++++++++++++++++- docker-compose.e2e-dbs.yml | 130 ++++++++- 3 files changed, 393 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-backend-tests.yml b/.github/workflows/run-backend-tests.yml index 0ca20311b..e1c88a312 100644 --- a/.github/workflows/run-backend-tests.yml +++ b/.github/workflows/run-backend-tests.yml @@ -34,6 +34,9 @@ jobs: working-directory: backend - name: Start postgres and redis run: touch .env && docker compose -f docker-compose.dev.yml up -d db redis + - name: Login to Oracle Container Registry + run: echo "${{ secrets.ORACLE_DOCKER_REGISTRY_PASSWORD }}" | docker login container-registry.oracle.com -u "${{ secrets.ORACLE_DOCKER_REGISTRY_USERNAME }}" --password-stdin + - name: Start Secret Rotation testing databases run: docker compose -f docker-compose.e2e-dbs.yml up -d - name: Run unit test diff --git a/backend/e2e-test/routes/v3/secret-rotations.spec.ts b/backend/e2e-test/routes/v3/secret-rotations.spec.ts index f43f47c02..5237510bf 100644 --- a/backend/e2e-test/routes/v3/secret-rotations.spec.ts +++ b/backend/e2e-test/routes/v3/secret-rotations.spec.ts @@ -7,7 +7,8 @@ import { seedData1 } from "@app/db/seed-data"; enum SecretRotationType { OracleDb = "oracledb", - MySQL = "mysql" + MySQL = "mysql", + Postgres = "postgres" } type TGenericSqlCredentials = { @@ -147,6 +148,40 @@ const createMySQLAppConnection = async (credentials: TGenericSqlCredentials) => return json.appConnection.id as string; }; +const createPostgresAppConnection = async (credentials: TGenericSqlCredentials) => { + const createPostgresAppConnectionReqBody = { + credentials: { + host: credentials.host, + port: credentials.port, + database: credentials.database, + username: credentials.username, + password: credentials.password, + sslEnabled: false, + sslRejectUnauthorized: true + }, + name: `postgres-test-${uuidv4()}`, + description: "test-postgres", + gatewayId: null, + method: "username-and-password" + }; + + const res = await testServer.inject({ + method: "POST", + url: `/api/v1/app-connections/postgres`, + headers: { + authorization: `Bearer ${jwtAuthToken}` + }, + body: createPostgresAppConnectionReqBody + }); + + const json = JSON.parse(res.payload); + + expect(res.statusCode).toBe(200); + expect(json.appConnection).toBeDefined(); + + return json.appConnection.id as string; +}; + const createOracleInfisicalUsers = async ( credentials: TGenericSqlCredentials, userCredentials: TDatabaseUserCredentials[] @@ -169,7 +204,7 @@ const createOracleInfisicalUsers = async ( for await (const { username } of userCredentials) { // check if user exists, and if it does, don't create it - const existingUser = await client.raw(`SELECT * FROM all_users WHERE username = '${username.toUpperCase()}'`); + const existingUser = await client.raw(`SELECT * FROM all_users WHERE username = '${username}'`); if (!existingUser.length) { await client.raw(`CREATE USER ${username} IDENTIFIED BY "temporary_password"`); @@ -220,6 +255,36 @@ const createMySQLInfisicalUsers = async ( await client.destroy(); }; +const createPostgresInfisicalUsers = async ( + credentials: TGenericSqlCredentials, + userCredentials: TDatabaseUserCredentials[] +) => { + const client = knex({ + client: "pg", + connection: { + database: credentials.database, + port: credentials.port, + host: credentials.host, + user: credentials.username, + password: credentials.password, + connectionTimeoutMillis: 10000 + } + }); + + for await (const { username } of userCredentials) { + // check if user exists, and if it does, don't create it + const existingUser = await client.raw("SELECT * FROM pg_catalog.pg_user WHERE usename = ?", [username]); + + if (!existingUser.rows.length) { + await client.raw(`CREATE USER "${username}" WITH PASSWORD 'temporary_password'`); + } + + await client.raw("GRANT ALL PRIVILEGES ON DATABASE ?? TO ??", [credentials.database, username]); + } + + await client.destroy(); +}; + const createOracleDBSecretRotation = async ( appConnectionId: string, credentials: TGenericSqlCredentials, @@ -324,6 +389,58 @@ const createMySQLSecretRotation = async ( return res; }; +const createPostgresSecretRotation = async ( + appConnectionId: string, + credentials: TGenericSqlCredentials, + userCredentials: TDatabaseUserCredentials[], + secretMapping: TSecretMapping +) => { + const now = new Date(); + const rotationTime = new Date(now.getTime() - 2 * 60 * 1000); // 2 minutes ago + + await createPostgresInfisicalUsers(credentials, userCredentials); + + const createPostgresSecretRotationReqBody = { + parameters: userCredentials.reduce( + (acc, user, index) => { + acc[`username${index + 1}`] = user.username; + return acc; + }, + {} as Record + ), + secretsMapping: { + username: secretMapping.username, + password: secretMapping.password + }, + name: `test-postgres-rotation-${uuidv4()}`, + description: "Test Postgres Secret Rotation", + secretPath: "/", + isAutoRotationEnabled: true, + rotationInterval: 5, + rotateAtUtc: { + hours: rotationTime.getUTCHours(), + minutes: rotationTime.getUTCMinutes() + }, + connectionId: appConnectionId, + environment: seedData1.environment.slug, + projectId: seedData1.projectV3.id + }; + + const res = await testServer.inject({ + method: "POST", + url: `/api/v2/secret-rotations/postgres-credentials`, + headers: { + authorization: `Bearer ${jwtAuthToken}` + }, + body: createPostgresSecretRotationReqBody + }); + + expect(res.statusCode).toBe(200); + expect(res.json().secretRotation).toBeDefined(); + + return res; +}; + describe("Secret Rotations", async () => { const testCases = [ { @@ -349,6 +466,52 @@ describe("Secret Rotations", async () => { } ] }, + { + type: SecretRotationType.MySQL, + name: "MySQL (8.0.29) Secret Rotation", + dbCredentials: { + database: "mysql-test", + host: "127.0.0.1", + username: "root", + password: "mysql-test", + port: 3307 + }, + secretMapping: { + username: formatSqlUsername("MYSQL_USERNAME"), + password: formatSqlUsername("MYSQL_PASSWORD") + }, + userCredentials: [ + { + username: formatSqlUsername("MYSQL_USER_1") + }, + { + username: formatSqlUsername("MYSQL_USER_2") + } + ] + }, + { + type: SecretRotationType.MySQL, + name: "MySQL (5.7.31) Secret Rotation", + dbCredentials: { + database: "mysql-test", + host: "127.0.0.1", + username: "root", + password: "mysql-test", + port: 3308 + }, + secretMapping: { + username: formatSqlUsername("MYSQL_USERNAME"), + password: formatSqlUsername("MYSQL_PASSWORD") + }, + userCredentials: [ + { + username: formatSqlUsername("MYSQL_USER_1") + }, + { + username: formatSqlUsername("MYSQL_USER_2") + } + ] + }, { type: SecretRotationType.OracleDb, name: "OracleDB (23.8) Secret Rotation", @@ -371,6 +534,98 @@ describe("Secret Rotations", async () => { username: formatSqlUsername("INFISICAL_USER_2") } ] + }, + { + type: SecretRotationType.OracleDb, + name: "OracleDB (19.3) Secret Rotation", + dbCredentials: { + database: "ORCLPDB1", + host: "127.0.0.1", + username: "system", + password: "OrCAKF112aaSfAdfdA2Ac3@@!", + port: 1522 + }, + secretMapping: { + username: formatSqlUsername("ORACLEDB_USERNAME"), + password: formatSqlUsername("ORACLEDB_PASSWORD") + }, + userCredentials: [ + { + username: formatSqlUsername("INFISICAL_USER_1") + }, + { + username: formatSqlUsername("INFISICAL_USER_2") + } + ] + }, + { + type: SecretRotationType.Postgres, + name: "Postgres (17) Secret Rotation", + dbCredentials: { + database: "postgres-test", + host: "127.0.0.1", + username: "postgres-test", + password: "postgres-test", + port: 5433 + }, + secretMapping: { + username: formatSqlUsername("POSTGRES_USERNAME"), + password: formatSqlUsername("POSTGRES_PASSWORD") + }, + userCredentials: [ + { + username: formatSqlUsername("INFISICAL_USER_1") + }, + { + username: formatSqlUsername("INFISICAL_USER_2") + } + ] + }, + { + type: SecretRotationType.Postgres, + name: "Postgres (16) Secret Rotation", + dbCredentials: { + database: "postgres-test", + host: "127.0.0.1", + username: "postgres-test", + password: "postgres-test", + port: 5434 + }, + secretMapping: { + username: formatSqlUsername("POSTGRES_USERNAME"), + password: formatSqlUsername("POSTGRES_PASSWORD") + }, + userCredentials: [ + { + username: formatSqlUsername("INFISICAL_USER_1") + }, + { + username: formatSqlUsername("INFISICAL_USER_2") + } + ] + }, + { + type: SecretRotationType.Postgres, + name: "Postgres (10.12) Secret Rotation", + dbCredentials: { + database: "postgres-test", + host: "127.0.0.1", + username: "postgres-test", + password: "postgres-test", + port: 5435 + }, + secretMapping: { + username: formatSqlUsername("POSTGRES_USERNAME"), + password: formatSqlUsername("POSTGRES_PASSWORD") + }, + userCredentials: [ + { + username: formatSqlUsername("INFISICAL_USER_1") + }, + { + username: formatSqlUsername("INFISICAL_USER_2") + } + ] } ] as { type: SecretRotationType; @@ -382,12 +637,14 @@ describe("Secret Rotations", async () => { const createAppConnectionMap = { [SecretRotationType.OracleDb]: createOracleDBAppConnection, - [SecretRotationType.MySQL]: createMySQLAppConnection + [SecretRotationType.MySQL]: createMySQLAppConnection, + [SecretRotationType.Postgres]: createPostgresAppConnection }; const createRotationMap = { [SecretRotationType.OracleDb]: createOracleDBSecretRotation, - [SecretRotationType.MySQL]: createMySQLSecretRotation + [SecretRotationType.MySQL]: createMySQLSecretRotation, + [SecretRotationType.Postgres]: createPostgresSecretRotation }; const appConnectionIds: { id: string; type: SecretRotationType }[] = []; diff --git a/docker-compose.e2e-dbs.yml b/docker-compose.e2e-dbs.yml index 1e3777e10..af7416712 100644 --- a/docker-compose.e2e-dbs.yml +++ b/docker-compose.e2e-dbs.yml @@ -1,6 +1,7 @@ version: '3.8' services: + # Oracle Databases oracle-db-23.8: image: container-registry.oracle.com/database/free:23.8.0.0 container_name: oracle-db-23.8 @@ -18,6 +19,26 @@ services: timeout: 10s retries: 5 + oracle-db-19.19: + # Official Oracle 19.19.0.0 - requires docker login container-registry.oracle.com + image: container-registry.oracle.com/database/enterprise:19.19.0.0 + container_name: oracle-db-19.19 + ports: + - "1522:1521" + environment: + - ORACLE_SID=ORCLCDB + - ORACLE_PDB=ORCLPDB1 + - ORACLE_PWD=OrCAKF112aaSfAdfdA2Ac3@@! + - ORACLE_EDITION=enterprise + - ORACLE_CHARACTERSET=AL32UTF8 + volumes: + - oracle-data-19.19:/opt/oracle/oradata + shm_size: 2gb + restart: unless-stopped + healthcheck: + disable: true + + # MySQL Databases mysql-8.4.6: image: mysql:8.4.6 container_name: mysql-8.4.6 @@ -38,6 +59,113 @@ services: timeout: 10s retries: 5 + mysql-8.0.29: + image: mysql:8.0.29 + container_name: mysql-8.0.28 + ports: + - "3307:3306" + environment: + - MYSQL_ROOT_PASSWORD=mysql-test + - MYSQL_DATABASE=mysql-test + - MYSQL_ROOT_HOST=% + - MYSQL_USER=mysql-test + - MYSQL_PASSWORD=mysql-test + volumes: + - mysql-data-8.0.29:/var/lib/mysql + restart: unless-stopped + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "mysql-test", "-pmysql-test"] + interval: 30s + timeout: 10s + retries: 5 + + mysql-5.7.31: + image: mysql:5.7.31 + container_name: mysql-5.7.31 + platform: linux/amd64 + ports: + - "3308:3306" + environment: + - MYSQL_ROOT_PASSWORD=mysql-test + - MYSQL_DATABASE=mysql-test + - MYSQL_ROOT_HOST=% + - MYSQL_USER=mysql-test + - MYSQL_PASSWORD=mysql-test + volumes: + - mysql-data-5.7.31:/var/lib/mysql + restart: unless-stopped + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "mysql-test", "-pmysql-test"] + interval: 30s + timeout: 10s + retries: 5 + + + + # PostgreSQL Databases + postgres-17: + image: postgres:17 + platform: linux/amd64 + container_name: postgres-17 + ports: + - "5433:5432" + environment: + - POSTGRES_DB=postgres-test + - POSTGRES_USER=postgres-test + - POSTGRES_PASSWORD=postgres-test + volumes: + - postgres-data-17:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres-test -d postgres-test"] + interval: 30s + timeout: 10s + retries: 5 + + postgres-16: + image: postgres:16 + platform: linux/amd64 + container_name: postgres-16 + ports: + - "5434:5432" + environment: + - POSTGRES_DB=postgres-test + - POSTGRES_USER=postgres-test + - POSTGRES_PASSWORD=postgres-test + volumes: + - postgres-data-16:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres-test -d postgres-test"] + interval: 30s + timeout: 10s + retries: 5 + + postgres-10.12: + image: postgres:10.12 + platform: linux/amd64 + container_name: postgres-10.12 + ports: + - "5435:5432" + environment: + - POSTGRES_DB=postgres-test + - POSTGRES_USER=postgres-test + - POSTGRES_PASSWORD=postgres-test + volumes: + - postgres-data-10.12:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres-test -d postgres-test"] + interval: 30s + timeout: 10s + retries: 5 + volumes: oracle-data-23.8: - mysql-data-8.4.6: \ No newline at end of file + oracle-data-19.19: + mysql-data-8.4.6: + mysql-data-8.0.29: + mysql-data-5.7.31: + postgres-data-17: + postgres-data-16: + postgres-data-10.12: \ No newline at end of file From 389e2e1fb7710f3098f0e6bedbee6371e8a797fc Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Thu, 7 Aug 2025 00:42:37 +0400 Subject: [PATCH 06/20] Update 20250725144940_fix-secret-reminders-migration.ts --- .../20250725144940_fix-secret-reminders-migration.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/db/migrations/20250725144940_fix-secret-reminders-migration.ts b/backend/src/db/migrations/20250725144940_fix-secret-reminders-migration.ts index b720c97ae..8c6dc8728 100644 --- a/backend/src/db/migrations/20250725144940_fix-secret-reminders-migration.ts +++ b/backend/src/db/migrations/20250725144940_fix-secret-reminders-migration.ts @@ -2,7 +2,7 @@ import { Knex } from "knex"; import { chunkArray } from "@app/lib/fn"; -import { logger } from "@app/lib/logger"; +import { initLogger, logger } from "@app/lib/logger"; import { TableName } from "../schemas"; import { TReminders, TRemindersInsert } from "../schemas/reminders"; @@ -107,5 +107,6 @@ export async function up(knex: Knex): Promise { } export async function down(): Promise { + initLogger(); logger.info("Rollback not implemented for secret reminders fix migration"); } From 9193418f8be63698c2a39e71c7404a1750ab16fc Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Fri, 8 Aug 2025 05:14:05 +0400 Subject: [PATCH 07/20] Update run-backend-tests.yml --- .github/workflows/run-backend-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-backend-tests.yml b/.github/workflows/run-backend-tests.yml index e1c88a312..b429c5d7b 100644 --- a/.github/workflows/run-backend-tests.yml +++ b/.github/workflows/run-backend-tests.yml @@ -38,7 +38,7 @@ jobs: run: echo "${{ secrets.ORACLE_DOCKER_REGISTRY_PASSWORD }}" | docker login container-registry.oracle.com -u "${{ secrets.ORACLE_DOCKER_REGISTRY_USERNAME }}" --password-stdin - name: Start Secret Rotation testing databases - run: docker compose -f docker-compose.e2e-dbs.yml up -d + run: docker compose -f docker-compose.e2e-dbs.yml up -d --wait --wait-timeout 300 - name: Run unit test run: npm run test:unit working-directory: backend From 9d92ffce959019044791e396227be12bfb949b4f Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Fri, 8 Aug 2025 05:21:49 +0400 Subject: [PATCH 08/20] Update docker-compose.e2e-dbs.yml --- docker-compose.e2e-dbs.yml | 42 +++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/docker-compose.e2e-dbs.yml b/docker-compose.e2e-dbs.yml index af7416712..ac6f3fb4f 100644 --- a/docker-compose.e2e-dbs.yml +++ b/docker-compose.e2e-dbs.yml @@ -15,9 +15,10 @@ services: restart: unless-stopped healthcheck: test: ["CMD", "sqlplus", "-L", "system/pdb-password@//localhost:1521/FREEPDB1", "<<<", "SELECT 1 FROM DUAL;"] - interval: 30s + interval: 10s timeout: 10s - retries: 5 + retries: 30 + start_period: 30s oracle-db-19.19: # Official Oracle 19.19.0.0 - requires docker login container-registry.oracle.com @@ -36,7 +37,11 @@ services: shm_size: 2gb restart: unless-stopped healthcheck: - disable: true + test: ["CMD", "sqlplus", "-L", "system/OrCAKF112aaSfAdfdA2Ac3@@!@//localhost:1521/ORCLCDB", "<<<", "SELECT 1 FROM DUAL;"] + interval: 10s + timeout: 10s + retries: 30 + start_period: 30s # MySQL Databases mysql-8.4.6: @@ -55,9 +60,10 @@ services: restart: unless-stopped healthcheck: test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "mysql-test", "-pmysql-test"] - interval: 30s + interval: 10s timeout: 10s - retries: 5 + retries: 30 + start_period: 30s mysql-8.0.29: image: mysql:8.0.29 @@ -75,9 +81,10 @@ services: restart: unless-stopped healthcheck: test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "mysql-test", "-pmysql-test"] - interval: 30s + interval: 10s timeout: 10s - retries: 5 + retries: 30 + start_period: 30s mysql-5.7.31: image: mysql:5.7.31 @@ -96,10 +103,10 @@ services: restart: unless-stopped healthcheck: test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "mysql-test", "-pmysql-test"] - interval: 30s + interval: 10s timeout: 10s - retries: 5 - + retries: 30 + start_period: 30s # PostgreSQL Databases @@ -118,9 +125,10 @@ services: restart: unless-stopped healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres-test -d postgres-test"] - interval: 30s + interval: 10s timeout: 10s - retries: 5 + retries: 30 + start_period: 30s postgres-16: image: postgres:16 @@ -137,9 +145,10 @@ services: restart: unless-stopped healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres-test -d postgres-test"] - interval: 30s + interval: 10s timeout: 10s - retries: 5 + retries: 30 + start_period: 30s postgres-10.12: image: postgres:10.12 @@ -156,9 +165,10 @@ services: restart: unless-stopped healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres-test -d postgres-test"] - interval: 30s + interval: 10s timeout: 10s - retries: 5 + retries: 30 + start_period: 30s volumes: oracle-data-23.8: From fbfc51ee9371a39983d5febdd246f8bab32731c3 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Fri, 8 Aug 2025 05:52:15 +0400 Subject: [PATCH 09/20] Update docker-compose.e2e-dbs.yml --- docker-compose.e2e-dbs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.e2e-dbs.yml b/docker-compose.e2e-dbs.yml index ac6f3fb4f..48c9f16c8 100644 --- a/docker-compose.e2e-dbs.yml +++ b/docker-compose.e2e-dbs.yml @@ -24,6 +24,7 @@ services: # Official Oracle 19.19.0.0 - requires docker login container-registry.oracle.com image: container-registry.oracle.com/database/enterprise:19.19.0.0 container_name: oracle-db-19.19 + platform: linux/amd64 ports: - "1522:1521" environment: From b6902160ce03a333126b892c49267a9cd8e94266 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Fri, 8 Aug 2025 05:59:32 +0400 Subject: [PATCH 10/20] Update docker-compose.e2e-dbs.yml --- docker-compose.e2e-dbs.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.e2e-dbs.yml b/docker-compose.e2e-dbs.yml index 48c9f16c8..d6400e862 100644 --- a/docker-compose.e2e-dbs.yml +++ b/docker-compose.e2e-dbs.yml @@ -20,10 +20,10 @@ services: retries: 30 start_period: 30s - oracle-db-19.19: + oracle-db-19.3: # Official Oracle 19.19.0.0 - requires docker login container-registry.oracle.com - image: container-registry.oracle.com/database/enterprise:19.19.0.0 - container_name: oracle-db-19.19 + image: container-registry.oracle.com/database/enterprise:19.3.0.0 + container_name: oracle-db-19.3 platform: linux/amd64 ports: - "1522:1521" @@ -34,7 +34,7 @@ services: - ORACLE_EDITION=enterprise - ORACLE_CHARACTERSET=AL32UTF8 volumes: - - oracle-data-19.19:/opt/oracle/oradata + - oracle-data-19.3:/opt/oracle/oradata shm_size: 2gb restart: unless-stopped healthcheck: @@ -173,7 +173,7 @@ services: volumes: oracle-data-23.8: - oracle-data-19.19: + oracle-data-19.3: mysql-data-8.4.6: mysql-data-8.0.29: mysql-data-5.7.31: From 35d1eabf49890b3a335c9971cd7076c3393fb2c2 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Fri, 8 Aug 2025 06:07:04 +0400 Subject: [PATCH 11/20] Update run-backend-tests.yml --- .github/workflows/run-backend-tests.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/run-backend-tests.yml b/.github/workflows/run-backend-tests.yml index b429c5d7b..413ed0ebf 100644 --- a/.github/workflows/run-backend-tests.yml +++ b/.github/workflows/run-backend-tests.yml @@ -16,6 +16,16 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: + + + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + docker system prune -af + - name: ☁️ Checkout source uses: actions/checkout@v3 - uses: KengoTODA/actions-setup-docker-compose@v1 From 0b11dcd627701c302233f92649855cf60de577be Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Fri, 8 Aug 2025 18:41:04 +0400 Subject: [PATCH 12/20] Update secret-rotations.spec.ts --- .../routes/v3/secret-rotations.spec.ts | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/backend/e2e-test/routes/v3/secret-rotations.spec.ts b/backend/e2e-test/routes/v3/secret-rotations.spec.ts index 5237510bf..7106cd78b 100644 --- a/backend/e2e-test/routes/v3/secret-rotations.spec.ts +++ b/backend/e2e-test/routes/v3/secret-rotations.spec.ts @@ -535,29 +535,29 @@ describe("Secret Rotations", async () => { } ] }, - { - type: SecretRotationType.OracleDb, - name: "OracleDB (19.3) Secret Rotation", - dbCredentials: { - database: "ORCLPDB1", - host: "127.0.0.1", - username: "system", - password: "OrCAKF112aaSfAdfdA2Ac3@@!", - port: 1522 - }, - secretMapping: { - username: formatSqlUsername("ORACLEDB_USERNAME"), - password: formatSqlUsername("ORACLEDB_PASSWORD") - }, - userCredentials: [ - { - username: formatSqlUsername("INFISICAL_USER_1") - }, - { - username: formatSqlUsername("INFISICAL_USER_2") - } - ] - }, + // { + // type: SecretRotationType.OracleDb, + // name: "OracleDB (19.3) Secret Rotation", + // dbCredentials: { + // database: "ORCLPDB1", + // host: "127.0.0.1", + // username: "system", + // password: "OrCAKF112aaSfAdfdA2Ac3@@!", + // port: 1522 + // }, + // secretMapping: { + // username: formatSqlUsername("ORACLEDB_USERNAME"), + // password: formatSqlUsername("ORACLEDB_PASSWORD") + // }, + // userCredentials: [ + // { + // username: formatSqlUsername("INFISICAL_USER_1") + // }, + // { + // username: formatSqlUsername("INFISICAL_USER_2") + // } + // ] + // }, { type: SecretRotationType.Postgres, name: "Postgres (17) Secret Rotation", From e10aec3170098ee28dc4509e3002c9ac453e01bb Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Fri, 8 Aug 2025 18:42:03 +0400 Subject: [PATCH 13/20] Update docker-compose.e2e-dbs.yml --- docker-compose.e2e-dbs.yml | 49 +++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/docker-compose.e2e-dbs.yml b/docker-compose.e2e-dbs.yml index d6400e862..68b7bc9fe 100644 --- a/docker-compose.e2e-dbs.yml +++ b/docker-compose.e2e-dbs.yml @@ -20,29 +20,30 @@ services: retries: 30 start_period: 30s - oracle-db-19.3: - # Official Oracle 19.19.0.0 - requires docker login container-registry.oracle.com - image: container-registry.oracle.com/database/enterprise:19.3.0.0 - container_name: oracle-db-19.3 - platform: linux/amd64 - ports: - - "1522:1521" - environment: - - ORACLE_SID=ORCLCDB - - ORACLE_PDB=ORCLPDB1 - - ORACLE_PWD=OrCAKF112aaSfAdfdA2Ac3@@! - - ORACLE_EDITION=enterprise - - ORACLE_CHARACTERSET=AL32UTF8 - volumes: - - oracle-data-19.3:/opt/oracle/oradata - shm_size: 2gb - restart: unless-stopped - healthcheck: - test: ["CMD", "sqlplus", "-L", "system/OrCAKF112aaSfAdfdA2Ac3@@!@//localhost:1521/ORCLCDB", "<<<", "SELECT 1 FROM DUAL;"] - interval: 10s - timeout: 10s - retries: 30 - start_period: 30s + # Oracle DB 19.3 is MASSIVE and takes up a lot of compute in github actions... + # oracle-db-19.3: + # # Official Oracle 19.19.0.0 - requires docker login container-registry.oracle.com + # image: container-registry.oracle.com/database/enterprise:19.3.0.0 + # container_name: oracle-db-19.3 + # platform: linux/amd64 + # ports: + # - "1522:1521" + # environment: + # - ORACLE_SID=ORCLCDB + # - ORACLE_PDB=ORCLPDB1 + # - ORACLE_PWD=OrCAKF112aaSfAdfdA2Ac3@@! + # - ORACLE_EDITION=enterprise + # - ORACLE_CHARACTERSET=AL32UTF8 + # volumes: + # - oracle-data-19.3:/opt/oracle/oradata + # shm_size: 2gb + # restart: unless-stopped + # healthcheck: + # test: ["CMD", "sqlplus", "-L", "system/OrCAKF112aaSfAdfdA2Ac3@@!@//localhost:1521/ORCLCDB", "<<<", "SELECT 1 FROM DUAL;"] + # interval: 10s + # timeout: 10s + # retries: 30 + # start_period: 30s # MySQL Databases mysql-8.4.6: @@ -173,7 +174,7 @@ services: volumes: oracle-data-23.8: - oracle-data-19.3: + # oracle-data-19.3: mysql-data-8.4.6: mysql-data-8.0.29: mysql-data-5.7.31: From 23237dd05507f450c5e7bdfeab12d09aae572578 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Fri, 8 Aug 2025 19:52:38 +0400 Subject: [PATCH 14/20] Update secret-rotation-v2-queue.ts --- .../ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts index 5a5f043d2..cd06bd7d0 100644 --- a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts +++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts @@ -84,6 +84,8 @@ export const secretRotationV2QueueServiceFactory = async ({ secretRotationV2DAL, secretRotationV2Service }); + + return; } await queueService.queuePg( From 368e00ea71a812520503b8e86e4679ce8948c5e2 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Fri, 8 Aug 2025 20:16:41 +0400 Subject: [PATCH 15/20] Update secret-rotation-v2-queue.ts --- .../secret-rotation-v2-queue.ts | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts index cd06bd7d0..50c81f444 100644 --- a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts +++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts @@ -84,18 +84,16 @@ export const secretRotationV2QueueServiceFactory = async ({ secretRotationV2DAL, secretRotationV2Service }); - - return; + } else { + await queueService.queuePg( + QueueJobs.SecretRotationV2RotateSecrets, + { + rotationId: rotation.id, + queuedAt: currentTime + }, + getSecretRotationRotateSecretJobOptions(rotation) + ); } - - await queueService.queuePg( - QueueJobs.SecretRotationV2RotateSecrets, - { - rotationId: rotation.id, - queuedAt: currentTime - }, - getSecretRotationRotateSecretJobOptions(rotation) - ); } } catch (error) { logger.error(error, "secretRotationV2Queue: Queue Rotations Error:"); From e13fc93bac7667dd455260906923374671cf9b6c Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Tue, 12 Aug 2025 00:30:32 +0400 Subject: [PATCH 16/20] fix(e2e-tests): oracle 19c rotation fix --- .../routes/v3/secret-rotations.spec.ts | 46 +++++++++---------- backend/vitest.e2e.config.ts | 7 ++- docker-compose.e2e-dbs.yml | 26 ----------- 3 files changed, 28 insertions(+), 51 deletions(-) diff --git a/backend/e2e-test/routes/v3/secret-rotations.spec.ts b/backend/e2e-test/routes/v3/secret-rotations.spec.ts index 7106cd78b..f053eae70 100644 --- a/backend/e2e-test/routes/v3/secret-rotations.spec.ts +++ b/backend/e2e-test/routes/v3/secret-rotations.spec.ts @@ -535,29 +535,29 @@ describe("Secret Rotations", async () => { } ] }, - // { - // type: SecretRotationType.OracleDb, - // name: "OracleDB (19.3) Secret Rotation", - // dbCredentials: { - // database: "ORCLPDB1", - // host: "127.0.0.1", - // username: "system", - // password: "OrCAKF112aaSfAdfdA2Ac3@@!", - // port: 1522 - // }, - // secretMapping: { - // username: formatSqlUsername("ORACLEDB_USERNAME"), - // password: formatSqlUsername("ORACLEDB_PASSWORD") - // }, - // userCredentials: [ - // { - // username: formatSqlUsername("INFISICAL_USER_1") - // }, - // { - // username: formatSqlUsername("INFISICAL_USER_2") - // } - // ] - // }, + { + type: SecretRotationType.OracleDb, + name: "OracleDB (19.3) Secret Rotation", + dbCredentials: { + password: process.env.E2E_TEST_ORACLE_DB_PASSWORD!, + host: process.env.E2E_TEST_ORACLE_DB_HOST!, + username: process.env.E2E_TEST_ORACLE_DB_USERNAME!, + port: 1521, + database: "ORCLPDB1" + }, + secretMapping: { + username: formatSqlUsername("ORACLEDB_USERNAME"), + password: formatSqlUsername("ORACLEDB_PASSWORD") + }, + userCredentials: [ + { + username: formatSqlUsername("INFISICAL_USER_1") + }, + { + username: formatSqlUsername("INFISICAL_USER_2") + } + ] + }, { type: SecretRotationType.Postgres, name: "Postgres (17) Secret Rotation", diff --git a/backend/vitest.e2e.config.ts b/backend/vitest.e2e.config.ts index 684a4dc42..4c4680cb0 100644 --- a/backend/vitest.e2e.config.ts +++ b/backend/vitest.e2e.config.ts @@ -5,10 +5,13 @@ export default defineConfig({ test: { globals: true, env: { - NODE_ENV: "test" + NODE_ENV: "test", + E2E_TEST_ORACLE_DB_HOST: process.env.E2E_TEST_ORACLE_DB_HOST!, + E2E_TEST_ORACLE_DB_USERNAME: process.env.E2E_TEST_ORACLE_DB_USERNAME!, + E2E_TEST_ORACLE_DB_PASSWORD: process.env.E2E_TEST_ORACLE_DB_PASSWORD! }, environment: "./e2e-test/vitest-environment-knex.ts", - include: ["./e2e-test/**/*.spec.ts"], + include: ["./e2e-test/**/secret-rotations.spec.ts"], poolOptions: { threads: { singleThread: true, diff --git a/docker-compose.e2e-dbs.yml b/docker-compose.e2e-dbs.yml index 68b7bc9fe..41f542207 100644 --- a/docker-compose.e2e-dbs.yml +++ b/docker-compose.e2e-dbs.yml @@ -20,31 +20,6 @@ services: retries: 30 start_period: 30s - # Oracle DB 19.3 is MASSIVE and takes up a lot of compute in github actions... - # oracle-db-19.3: - # # Official Oracle 19.19.0.0 - requires docker login container-registry.oracle.com - # image: container-registry.oracle.com/database/enterprise:19.3.0.0 - # container_name: oracle-db-19.3 - # platform: linux/amd64 - # ports: - # - "1522:1521" - # environment: - # - ORACLE_SID=ORCLCDB - # - ORACLE_PDB=ORCLPDB1 - # - ORACLE_PWD=OrCAKF112aaSfAdfdA2Ac3@@! - # - ORACLE_EDITION=enterprise - # - ORACLE_CHARACTERSET=AL32UTF8 - # volumes: - # - oracle-data-19.3:/opt/oracle/oradata - # shm_size: 2gb - # restart: unless-stopped - # healthcheck: - # test: ["CMD", "sqlplus", "-L", "system/OrCAKF112aaSfAdfdA2Ac3@@!@//localhost:1521/ORCLCDB", "<<<", "SELECT 1 FROM DUAL;"] - # interval: 10s - # timeout: 10s - # retries: 30 - # start_period: 30s - # MySQL Databases mysql-8.4.6: image: mysql:8.4.6 @@ -174,7 +149,6 @@ services: volumes: oracle-data-23.8: - # oracle-data-19.3: mysql-data-8.4.6: mysql-data-8.0.29: mysql-data-5.7.31: From e173ff3828a26df4c573e80d01a8afb019fa89cf Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Tue, 12 Aug 2025 00:56:11 +0400 Subject: [PATCH 17/20] final fixes --- .github/workflows/run-backend-tests.yml | 6 +- .../routes/v3/secret-rotations.spec.ts | 96 +++++++++++-------- backend/vitest.e2e.config.ts | 2 +- 3 files changed, 62 insertions(+), 42 deletions(-) diff --git a/.github/workflows/run-backend-tests.yml b/.github/workflows/run-backend-tests.yml index 413ed0ebf..afa87a6cb 100644 --- a/.github/workflows/run-backend-tests.yml +++ b/.github/workflows/run-backend-tests.yml @@ -44,9 +44,6 @@ jobs: working-directory: backend - name: Start postgres and redis run: touch .env && docker compose -f docker-compose.dev.yml up -d db redis - - name: Login to Oracle Container Registry - run: echo "${{ secrets.ORACLE_DOCKER_REGISTRY_PASSWORD }}" | docker login container-registry.oracle.com -u "${{ secrets.ORACLE_DOCKER_REGISTRY_USERNAME }}" --password-stdin - - name: Start Secret Rotation testing databases run: docker compose -f docker-compose.e2e-dbs.yml up -d --wait --wait-timeout 300 - name: Run unit test @@ -56,6 +53,9 @@ jobs: run: npm run test:e2e working-directory: backend env: + E2E_TEST_ORACLE_DB_HOST: ${{ secrets.E2E_TEST_ORACLE_DB_HOST }} + E2E_TEST_ORACLE_DB_USERNAME: ${{ secrets.E2E_TEST_ORACLE_DB_USERNAME }} + E2E_TEST_ORACLE_DB_PASSWORD: ${{ secrets.E2E_TEST_ORACLE_DB_PASSWORD }} REDIS_URL: redis://172.17.0.1:6379 DB_CONNECTION_URI: postgres://infisical:infisical@172.17.0.1:5432/infisical?sslmode=disable AUTH_SECRET: something-random diff --git a/backend/e2e-test/routes/v3/secret-rotations.spec.ts b/backend/e2e-test/routes/v3/secret-rotations.spec.ts index f053eae70..0dc9a4b13 100644 --- a/backend/e2e-test/routes/v3/secret-rotations.spec.ts +++ b/backend/e2e-test/routes/v3/secret-rotations.spec.ts @@ -538,6 +538,7 @@ describe("Secret Rotations", async () => { { type: SecretRotationType.OracleDb, name: "OracleDB (19.3) Secret Rotation", + skippable: true, dbCredentials: { password: process.env.E2E_TEST_ORACLE_DB_PASSWORD!, host: process.env.E2E_TEST_ORACLE_DB_HOST!, @@ -628,6 +629,7 @@ describe("Secret Rotations", async () => { ] } ] as { + skippable?: boolean; type: SecretRotationType; name: string; dbCredentials: TGenericSqlCredentials; @@ -660,47 +662,65 @@ describe("Secret Rotations", async () => { } }); - test.concurrent.each(testCases)( - "Create secret rotation for $name", - async ({ dbCredentials, secretMapping, userCredentials, type }) => { - const appConnectionId = await createAppConnectionMap[type](dbCredentials); - - if (appConnectionId) { - appConnectionIds.push({ id: appConnectionId, type }); - } - - const res = await createRotationMap[type](appConnectionId, dbCredentials, userCredentials, secretMapping); - - const resJson = JSON.parse(res.payload); - - if (resJson.secretRotation) { - secretRotationIds.push({ id: resJson.secretRotation.id, type }); - } - - const startSecretValue = await getSecretValue(secretMapping.password); - expect(startSecretValue).toBeDefined(); - - let attempts = 0; - while (attempts < 60) { - const currentSecretValue = await getSecretValue(secretMapping.password); - - if (currentSecretValue !== startSecretValue) { - break; + testCases.forEach(({ skippable, dbCredentials, secretMapping, userCredentials, type, name }) => { + const shouldSkip = () => { + if (skippable) { + if (type === SecretRotationType.OracleDb) { + if (!process.env.E2E_TEST_ORACLE_DB_HOST) { + return true; + } } - - attempts += 1; - await new Promise((resolve) => setTimeout(resolve, 2_500)); } - if (attempts >= 60) { - throw new Error("Secret rotation failed to rotate after 60 attempts"); - } + return false; + }; - const finalSecretValue = await getSecretValue(secretMapping.password); - expect(finalSecretValue).not.toBe(startSecretValue); - }, - { - timeout: 300_000 + if (shouldSkip()) { + test.skip(`Skipping Secret Rotation for ${type} (${name}) because E2E_TEST_ORACLE_DB_HOST is not set`); + } else { + test.concurrent( + `Create secret rotation for ${name}`, + async () => { + const appConnectionId = await createAppConnectionMap[type](dbCredentials); + + if (appConnectionId) { + appConnectionIds.push({ id: appConnectionId, type }); + } + + const res = await createRotationMap[type](appConnectionId, dbCredentials, userCredentials, secretMapping); + + const resJson = JSON.parse(res.payload); + + if (resJson.secretRotation) { + secretRotationIds.push({ id: resJson.secretRotation.id, type }); + } + + const startSecretValue = await getSecretValue(secretMapping.password); + expect(startSecretValue).toBeDefined(); + + let attempts = 0; + while (attempts < 60) { + const currentSecretValue = await getSecretValue(secretMapping.password); + + if (currentSecretValue !== startSecretValue) { + break; + } + + attempts += 1; + await new Promise((resolve) => setTimeout(resolve, 2_500)); + } + + if (attempts >= 60) { + throw new Error("Secret rotation failed to rotate after 60 attempts"); + } + + const finalSecretValue = await getSecretValue(secretMapping.password); + expect(finalSecretValue).not.toBe(startSecretValue); + }, + { + timeout: 300_000 + } + ); } - ); + }); }); diff --git a/backend/vitest.e2e.config.ts b/backend/vitest.e2e.config.ts index 4c4680cb0..7584e6c9c 100644 --- a/backend/vitest.e2e.config.ts +++ b/backend/vitest.e2e.config.ts @@ -11,7 +11,7 @@ export default defineConfig({ E2E_TEST_ORACLE_DB_PASSWORD: process.env.E2E_TEST_ORACLE_DB_PASSWORD! }, environment: "./e2e-test/vitest-environment-knex.ts", - include: ["./e2e-test/**/secret-rotations.spec.ts"], + include: ["./e2e-test/**/*.spec.ts"], poolOptions: { threads: { singleThread: true, From 372537f0b626ba253a2d9e08ab1403644dfe9cc0 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Tue, 12 Aug 2025 01:06:45 +0400 Subject: [PATCH 18/20] updated env vars --- backend/e2e-test/routes/v3/secret-rotations.spec.ts | 10 +++++----- backend/vitest.e2e.config.ts | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/backend/e2e-test/routes/v3/secret-rotations.spec.ts b/backend/e2e-test/routes/v3/secret-rotations.spec.ts index 0dc9a4b13..de539e5db 100644 --- a/backend/e2e-test/routes/v3/secret-rotations.spec.ts +++ b/backend/e2e-test/routes/v3/secret-rotations.spec.ts @@ -540,9 +540,9 @@ describe("Secret Rotations", async () => { name: "OracleDB (19.3) Secret Rotation", skippable: true, dbCredentials: { - password: process.env.E2E_TEST_ORACLE_DB_PASSWORD!, - host: process.env.E2E_TEST_ORACLE_DB_HOST!, - username: process.env.E2E_TEST_ORACLE_DB_USERNAME!, + password: process.env.E2E_TEST_ORACLE_DB_19_PASSWORD!, + host: process.env.E2E_TEST_ORACLE_DB_19_HOST!, + username: process.env.E2E_TEST_ORACLE_DB_19_USERNAME!, port: 1521, database: "ORCLPDB1" }, @@ -666,7 +666,7 @@ describe("Secret Rotations", async () => { const shouldSkip = () => { if (skippable) { if (type === SecretRotationType.OracleDb) { - if (!process.env.E2E_TEST_ORACLE_DB_HOST) { + if (!process.env.E2E_TEST_ORACLE_DB_19_HOST) { return true; } } @@ -676,7 +676,7 @@ describe("Secret Rotations", async () => { }; if (shouldSkip()) { - test.skip(`Skipping Secret Rotation for ${type} (${name}) because E2E_TEST_ORACLE_DB_HOST is not set`); + test.skip(`Skipping Secret Rotation for ${type} (${name}) because E2E_TEST_ORACLE_DB_19_HOST is not set`); } else { test.concurrent( `Create secret rotation for ${name}`, diff --git a/backend/vitest.e2e.config.ts b/backend/vitest.e2e.config.ts index 7584e6c9c..bb7ae8087 100644 --- a/backend/vitest.e2e.config.ts +++ b/backend/vitest.e2e.config.ts @@ -6,9 +6,9 @@ export default defineConfig({ globals: true, env: { NODE_ENV: "test", - E2E_TEST_ORACLE_DB_HOST: process.env.E2E_TEST_ORACLE_DB_HOST!, - E2E_TEST_ORACLE_DB_USERNAME: process.env.E2E_TEST_ORACLE_DB_USERNAME!, - E2E_TEST_ORACLE_DB_PASSWORD: process.env.E2E_TEST_ORACLE_DB_PASSWORD! + E2E_TEST_ORACLE_DB_19_HOST: process.env.E2E_TEST_ORACLE_DB_19_HOST!, + E2E_TEST_ORACLE_DB_19_USERNAME: process.env.E2E_TEST_ORACLE_DB_19_USERNAME!, + E2E_TEST_ORACLE_DB_19_PASSWORD: process.env.E2E_TEST_ORACLE_DB_19_PASSWORD! }, environment: "./e2e-test/vitest-environment-knex.ts", include: ["./e2e-test/**/*.spec.ts"], From 78bfd0922a038fd285c82731ec1317f07a30748e Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Tue, 12 Aug 2025 01:20:29 +0400 Subject: [PATCH 19/20] Update run-backend-tests.yml --- .github/workflows/run-backend-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-backend-tests.yml b/.github/workflows/run-backend-tests.yml index afa87a6cb..0a71819f1 100644 --- a/.github/workflows/run-backend-tests.yml +++ b/.github/workflows/run-backend-tests.yml @@ -53,9 +53,9 @@ jobs: run: npm run test:e2e working-directory: backend env: - E2E_TEST_ORACLE_DB_HOST: ${{ secrets.E2E_TEST_ORACLE_DB_HOST }} - E2E_TEST_ORACLE_DB_USERNAME: ${{ secrets.E2E_TEST_ORACLE_DB_USERNAME }} - E2E_TEST_ORACLE_DB_PASSWORD: ${{ secrets.E2E_TEST_ORACLE_DB_PASSWORD }} + E2E_TEST_ORACLE_DB_19_HOST: ${{ secrets.E2E_TEST_ORACLE_DB_19_HOST }} + E2E_TEST_ORACLE_DB_19_USERNAME: ${{ secrets.E2E_TEST_ORACLE_DB_19_USERNAME }} + E2E_TEST_ORACLE_DB_19_PASSWORD: ${{ secrets.E2E_TEST_ORACLE_DB_19_PASSWORD }} REDIS_URL: redis://172.17.0.1:6379 DB_CONNECTION_URI: postgres://infisical:infisical@172.17.0.1:5432/infisical?sslmode=disable AUTH_SECRET: something-random From c61dd1ee6e56f21c5beee9c458482a95be046ffc Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Tue, 12 Aug 2025 02:31:48 +0400 Subject: [PATCH 20/20] Update .infisicalignore --- .infisicalignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.infisicalignore b/.infisicalignore index fd4415178..66e2fb635 100644 --- a/.infisicalignore +++ b/.infisicalignore @@ -50,3 +50,4 @@ docs/integrations/app-connections/zabbix.mdx:generic-api-key:91 docs/integrations/app-connections/bitbucket.mdx:generic-api-key:123 docs/integrations/app-connections/railway.mdx:generic-api-key:156 .github/workflows/validate-db-schemas.yml:generic-api-key:21 +k8-operator/config/samples/universalAuthIdentitySecret.yaml:generic-api-key:8