Merge remote-tracking branch 'origin/main' into feat/gateway-v2

backend/src/ee/routes/v1/dynamic-secret-router.ts

@@ -84,7 +84,9 @@ export const registerDynamicSecretRouter = async (server: FastifyZodProvider) =>
       }),
       response: {
         200: z.object({
-          dynamicSecret: SanitizedDynamicSecretSchema
+          dynamicSecret: SanitizedDynamicSecretSchema.extend({
+            inputs: z.unknown()
+          })
         })
       }
     },
@@ -151,7 +153,9 @@ export const registerDynamicSecretRouter = async (server: FastifyZodProvider) =>
       }),
       response: {
         200: z.object({
-          dynamicSecret: SanitizedDynamicSecretSchema
+          dynamicSecret: SanitizedDynamicSecretSchema.extend({
+            inputs: z.unknown()
+          })
         })
       }
     },

backend/src/ee/routes/v1/license-router.ts

@@ -43,6 +43,12 @@ export const registerLicenseRouter = async (server: FastifyZodProvider) => {
     },
     schema: {
       params: z.object({ organizationId: z.string().trim() }),
+      querystring: z.object({
+        refreshCache: z
+          .enum(["true", "false"])
+          .default("false")
+          .transform((value) => value === "true")
+      }),
       response: {
         200: z.object({ plan: z.any() })
       }
@@ -54,7 +60,8 @@ export const registerLicenseRouter = async (server: FastifyZodProvider) => {
         actor: req.permission.type,
         actorOrgId: req.permission.orgId,
         actorAuthMethod: req.permission.authMethod,
-        orgId: req.params.organizationId
+        orgId: req.params.organizationId,
+        refreshCache: req.query.refreshCache
       });
       return { plan };
     }

backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts

@@ -46,7 +46,10 @@ export const dynamicSecretLeaseDALFactory = (db: TDbClient) => {
 
   const countLeasesForDynamicSecret = async (dynamicSecretId: string, tx?: Knex) => {
     try {
-      const doc = await (tx || db)(TableName.DynamicSecretLease).count("*").where({ dynamicSecretId }).first();
+      const doc = await (tx || db.replicaNode())(TableName.DynamicSecretLease)
+        .count("*")
+        .where({ dynamicSecretId })
+        .first();
       return parseInt(doc || "0", 10);
     } catch (error) {
       throw new DatabaseError({ error, name: "DynamicSecretCountLeases" });
@@ -55,7 +58,7 @@ export const dynamicSecretLeaseDALFactory = (db: TDbClient) => {
 
   const findById = async (id: string, tx?: Knex) => {
     try {
-      const doc = await (tx || db)(TableName.DynamicSecretLease)
+      const doc = await (tx || db.replicaNode())(TableName.DynamicSecretLease)
         .where({ [`${TableName.DynamicSecretLease}.id` as "id"]: id })
         .first()
         .join(

backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts

@@ -190,7 +190,7 @@ export const dynamicSecretServiceFactory = ({
       return cfg;
     });
 
-    return dynamicSecretCfg;
+    return { ...dynamicSecretCfg, inputs };
   };
 
   const updateByName: TDynamicSecretServiceFactory["updateByName"] = async ({
@@ -355,7 +355,7 @@ export const dynamicSecretServiceFactory = ({
       return cfg;
     });
 
-    return updatedDynamicCfg;
+    return { ...updatedDynamicCfg, inputs: updatedInput };
   };
 
   const deleteByName: TDynamicSecretServiceFactory["deleteByName"] = async ({

backend/src/ee/services/dynamic-secret/providers/models.ts

@@ -165,6 +165,7 @@ export const DynamicSecretSqlDBSchema = z.object({
   revocationStatement: z.string().trim(),
   renewStatement: z.string().trim().optional(),
   ca: z.string().optional(),
+  sslEnabled: z.boolean().optional(),
   gatewayId: z.string().nullable().optional()
 });
 

backend/src/ee/services/dynamic-secret/providers/sql-database.ts

@@ -1,4 +1,5 @@
 import handlebars from "handlebars";
+import RE2 from "re2";
 import knex from "knex";
 import { z } from "zod";
 
@@ -156,19 +157,40 @@ export const SqlDatabaseProvider = ({
     return { ...providerInputs, hostIp };
   };
 
-  const $getClient = async (providerInputs: z.infer<typeof DynamicSecretSqlDBSchema> & { hostIp: string }) => {
+  const $getClient = async (
+    providerInputs: z.infer<typeof DynamicSecretSqlDBSchema> & { hostIp: string; originalHost: string }
+  ) => {
     const ssl = providerInputs.ca
       ? { rejectUnauthorized: false, ca: providerInputs.ca, servername: providerInputs.host }
       : undefined;
+
     const isMsSQLClient = providerInputs.client === SqlProviders.MsSQL;
 
+    /*
+      We route through the gateway by setting connection.host = "localhost".
+      Azure SQL identifies the logical server from the TDS login name when the host
+      isn’t the Azure FQDN. Therefore, when using the gateway, ensure username is
+      "user@<azure-server-name>" so Azure opens the correct logical server.
+      Direct connections to the Azure FQDN usually don’t require this suffix.
+    */
+    const isAzureSql = isMsSQLClient && new RE2(/\.database\.windows\.net$/i).test(providerInputs.originalHost);
+    const azureServerLabel =
+      isAzureSql && providerInputs.gatewayId ? providerInputs.originalHost?.split(".")[0] : undefined;
+    const effectiveUser =
+      isAzureSql && !providerInputs.username.includes("@") && azureServerLabel
+        ? `${providerInputs.username}@${azureServerLabel}`
+        : providerInputs.username;
+
     const db = knex({
       client: providerInputs.client,
       connection: {
         database: providerInputs.database,
         port: providerInputs.port,
-        host: providerInputs.client === SqlProviders.Postgres ? providerInputs.hostIp : providerInputs.host,
-        user: providerInputs.username,
+        host:
+          providerInputs.client === SqlProviders.Postgres && !providerInputs.gatewayId
+            ? providerInputs.hostIp
+            : providerInputs.host,
+        user: effectiveUser,
         password: providerInputs.password,
         ssl,
         // @ts-expect-error this is because of knexjs type signature issue. This is directly passed to driver
@@ -176,6 +198,7 @@ export const SqlDatabaseProvider = ({
         // https://github.com/tediousjs/tedious/blob/ebb023ed90969a7ec0e4b036533ad52739d921f7/test/config.ci.ts#L19
         options: isMsSQLClient
           ? {
+              ...(providerInputs.sslEnabled !== undefined ? { encrypt: providerInputs.sslEnabled } : {}),
               trustServerCertificate: !providerInputs.ca,
               cryptoCredentialsDetails: providerInputs.ca ? { ca: providerInputs.ca } : {}
             }
@@ -238,7 +261,13 @@ export const SqlDatabaseProvider = ({
     const providerInputs = await validateProviderInputs(inputs);
     let isConnected = false;
     const gatewayCallback = async (host = providerInputs.host, port = providerInputs.port) => {
-      const db = await $getClient({ ...providerInputs, port, host, hostIp: providerInputs.hostIp });
+      const db = await $getClient({
+        ...providerInputs,
+        port,
+        host,
+        hostIp: providerInputs.hostIp,
+        originalHost: providerInputs.host
+      });
       // oracle needs from keyword
       const testStatement = providerInputs.client === SqlProviders.Oracle ? "SELECT 1 FROM DUAL" : "SELECT 1";
 
@@ -279,7 +308,12 @@ export const SqlDatabaseProvider = ({
 
     const password = generatePassword(providerInputs.client, providerInputs.passwordRequirements);
     const gatewayCallback = async (host = providerInputs.host, port = providerInputs.port) => {
-      const db = await $getClient({ ...providerInputs, port, host });
+      const db = await $getClient({
+        ...providerInputs,
+        port,
+        host,
+        originalHost: providerInputs.host
+      });
       try {
         const expiration = new Date(expireAt).toISOString();
 
@@ -322,7 +356,12 @@ export const SqlDatabaseProvider = ({
     const username = entityId;
     const { database } = providerInputs;
     const gatewayCallback = async (host = providerInputs.host, port = providerInputs.port) => {
-      const db = await $getClient({ ...providerInputs, port, host });
+      const db = await $getClient({
+        ...providerInputs,
+        port,
+        host,
+        originalHost: providerInputs.host
+      });
       try {
         const revokeStatement = handlebars.compile(providerInputs.revocationStatement)({ username, database });
         const queries = revokeStatement.toString().split(";").filter(Boolean);
@@ -357,7 +396,12 @@ export const SqlDatabaseProvider = ({
     if (!providerInputs.renewStatement) return { entityId };
 
     const gatewayCallback = async (host = providerInputs.host, port = providerInputs.port) => {
-      const db = await $getClient({ ...providerInputs, port, host });
+      const db = await $getClient({
+        ...providerInputs,
+        port,
+        host,
+        originalHost: providerInputs.host
+      });
       const expiration = new Date(expireAt).toISOString();
       const { database } = providerInputs;
 

backend/src/ee/services/gateway/gateway-dal.ts

@@ -13,7 +13,7 @@ export const gatewayDALFactory = (db: TDbClient) => {
     { offset, limit, sort, tx }: TFindOpt<TGateways> = {}
   ) => {
     try {
-      const query = (tx || db)(TableName.Gateway)
+      const query = (tx || db.replicaNode())(TableName.Gateway)
         // eslint-disable-next-line @typescript-eslint/no-misused-promises
         .where(buildFindFilter(filter, TableName.Gateway, ["orgId"]))
         .join(TableName.Identity, `${TableName.Identity}.id`, `${TableName.Gateway}.identityId`)

backend/src/ee/services/group/user-group-membership-dal.ts

@@ -23,7 +23,7 @@ export const userGroupMembershipDALFactory = (db: TDbClient) => {
         .whereIn(`${TableName.ProjectMembership}.projectId`, projectIds)
         .pluck(`${TableName.ProjectMembership}.projectId`);
 
-      const userGroupMemberships: string[] = await (tx || db)(TableName.UserGroupMembership)
+      const userGroupMemberships: string[] = await (tx || db.replicaNode())(TableName.UserGroupMembership)
         .where(`${TableName.UserGroupMembership}.userId`, userId)
         .whereNot(`${TableName.UserGroupMembership}.groupId`, groupId)
         .join(
@@ -79,7 +79,7 @@ export const userGroupMembershipDALFactory = (db: TDbClient) => {
         .pluck(`${TableName.GroupProjectMembership}.groupId`);
 
       // main query
-      const members = await (tx || db)(TableName.UserGroupMembership)
+      const members = await (tx || db.replicaNode())(TableName.UserGroupMembership)
         .where(`${TableName.UserGroupMembership}.groupId`, groupId)
         .where(`${TableName.UserGroupMembership}.isPending`, false)
         .join(TableName.Users, `${TableName.UserGroupMembership}.userId`, `${TableName.Users}.id`)

backend/src/ee/services/ldap-config/ldap-config-service.ts

@@ -127,6 +127,20 @@ export const ldapConfigServiceFactory = ({
         message:
           "Failed to create LDAP configuration due to plan restriction. Upgrade plan to create LDAP configuration."
       });
+
+    const org = await orgDAL.findOrgById(orgId);
+
+    if (!org) {
+      throw new NotFoundError({ message: `Could not find organization with ID "${orgId}"` });
+    }
+
+    if (org.googleSsoAuthEnforced && isActive) {
+      throw new BadRequestError({
+        message:
+          "You cannot enable LDAP SSO while Google OAuth is enforced. Disable Google OAuth enforcement to enable LDAP SSO."
+      });
+    }
+
     const { encryptor } = await kmsService.createCipherPairWithDataKey({
       type: KmsDataKey.Organization,
       orgId
@@ -233,6 +247,19 @@ export const ldapConfigServiceFactory = ({
           "Failed to update LDAP configuration due to plan restriction. Upgrade plan to update LDAP configuration."
       });
 
+    const org = await orgDAL.findOrgById(orgId);
+
+    if (!org) {
+      throw new NotFoundError({ message: `Could not find organization with ID "${orgId}"` });
+    }
+
+    if (org.googleSsoAuthEnforced && isActive) {
+      throw new BadRequestError({
+        message:
+          "You cannot enable LDAP SSO while Google OAuth is enforced. Disable Google OAuth enforcement to enable LDAP SSO."
+      });
+    }
+
     const updateQuery: TLdapConfigsUpdate = {
       isActive,
       url,

backend/src/ee/services/license/license-dal.ts

@@ -28,7 +28,7 @@ export const licenseDALFactory = (db: TDbClient) => {
   const countOrgUsersAndIdentities = async (orgId: string | null, tx?: Knex) => {
     try {
       // count org users
-      const userDoc = await (tx || db)(TableName.OrgMembership)
+      const userDoc = await (tx || db.replicaNode())(TableName.OrgMembership)
         .where({ status: OrgMembershipStatus.Accepted })
         .andWhere((bd) => {
           if (orgId) {
@@ -42,7 +42,7 @@ export const licenseDALFactory = (db: TDbClient) => {
       const userCount = Number(userDoc?.[0].count);
 
       // count org identities
-      const identityDoc = await (tx || db)(TableName.IdentityOrgMembership)
+      const identityDoc = await (tx || db.replicaNode())(TableName.IdentityOrgMembership)
         .where((bd) => {
           if (orgId) {
             void bd.where({ orgId });

backend/src/ee/services/license/license-service.ts

@@ -99,6 +99,17 @@ export const licenseServiceFactory = ({
       const workspacesUsed = await projectDAL.countOfOrgProjects(null);
       currentPlan.workspacesUsed = workspacesUsed;
 
+      const usedIdentitySeats = await licenseDAL.countOrgUsersAndIdentities(null);
+      if (usedIdentitySeats !== currentPlan.identitiesUsed) {
+        const usedSeats = await licenseDAL.countOfOrgMembers(null);
+        await licenseServerOnPremApi.request.patch(`/api/license/v1/license`, {
+          usedSeats,
+          usedIdentitySeats
+        });
+        currentPlan.identitiesUsed = usedIdentitySeats;
+        currentPlan.membersUsed = usedSeats;
+      }
+
       onPremFeatures = currentPlan;
       logger.info("Successfully synchronized license key features");
     } catch (error) {
@@ -226,10 +237,13 @@ export const licenseServiceFactory = ({
   };
 
   const refreshPlan = async (orgId: string) => {
+    await keyStore.deleteItem(FEATURE_CACHE_KEY(orgId));
     if (instanceType === InstanceType.Cloud) {
-      await keyStore.deleteItem(FEATURE_CACHE_KEY(orgId));
       await getPlan(orgId);
     }
+    if (instanceType === InstanceType.EnterpriseOnPrem) {
+      await syncLicenseKeyOnPremFeatures(true);
+    }
   };
 
   const generateOrgCustomerId = async (orgName: string, email?: string | null) => {
@@ -296,8 +310,19 @@ export const licenseServiceFactory = ({
     return data;
   };
 
-  const getOrgPlan = async ({ orgId, actor, actorId, actorOrgId, actorAuthMethod, projectId }: TOrgPlanDTO) => {
+  const getOrgPlan = async ({
+    orgId,
+    actor,
+    actorId,
+    actorOrgId,
+    actorAuthMethod,
+    projectId,
+    refreshCache
+  }: TOrgPlanDTO) => {
     await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId);
+    if (refreshCache) {
+      await refreshPlan(orgId);
+    }
     const plan = await getPlan(orgId, projectId);
     return plan;
   };

backend/src/ee/services/license/license-types.ts

@@ -87,6 +87,7 @@ export type TOrgPlansTableDTO = {
 
 export type TOrgPlanDTO = {
   projectId?: string;
+  refreshCache?: boolean;
 } & TOrgPermission;
 
 export type TStartOrgTrialDTO = {

backend/src/ee/services/oidc/oidc-config-service.ts

@@ -499,6 +499,13 @@ export const oidcConfigServiceFactory = ({
     );
     ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.Sso);
 
+    if (org.googleSsoAuthEnforced && isActive) {
+      throw new BadRequestError({
+        message:
+          "You cannot enable OIDC SSO while Google OAuth is enforced. Disable Google OAuth enforcement to enable OIDC SSO."
+      });
+    }
+
     const { encryptor } = await kmsService.createCipherPairWithDataKey({
       type: KmsDataKey.Organization,
       orgId: org.id
@@ -586,6 +593,13 @@ export const oidcConfigServiceFactory = ({
     );
     ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.Sso);
 
+    if (org.googleSsoAuthEnforced && isActive) {
+      throw new BadRequestError({
+        message:
+          "You cannot enable OIDC SSO while Google OAuth is enforced. Disable Google OAuth enforcement to enable OIDC SSO."
+      });
+    }
+
     const { encryptor } = await kmsService.createCipherPairWithDataKey({
       type: KmsDataKey.Organization,
       orgId: org.id

backend/src/ee/services/saml-config/saml-config-service.ts

@@ -82,6 +82,19 @@ export const samlConfigServiceFactory = ({
           "Failed to create SAML SSO configuration due to plan restriction. Upgrade plan to create SSO configuration."
       });
 
+    const org = await orgDAL.findOrgById(orgId);
+
+    if (!org) {
+      throw new NotFoundError({ message: `Could not find organization with ID "${orgId}"` });
+    }
+
+    if (org.googleSsoAuthEnforced && isActive) {
+      throw new BadRequestError({
+        message:
+          "You cannot enable SAML SSO while Google OAuth is enforced. Disable Google OAuth enforcement to enable SAML SSO."
+      });
+    }
+
     const { encryptor } = await kmsService.createCipherPairWithDataKey({
       type: KmsDataKey.Organization,
       orgId
@@ -120,6 +133,19 @@ export const samlConfigServiceFactory = ({
           "Failed to update SAML SSO configuration due to plan restriction. Upgrade plan to update SSO configuration."
       });
 
+    const org = await orgDAL.findOrgById(orgId);
+
+    if (!org) {
+      throw new NotFoundError({ message: `Could not find organization with ID "${orgId}"` });
+    }
+
+    if (org.googleSsoAuthEnforced && isActive) {
+      throw new BadRequestError({
+        message:
+          "Cannot enable SAML SSO while Google OAuth is enforced. Disable Google OAuth enforcement to enable SAML SSO."
+      });
+    }
+
     const updateQuery: TSamlConfigsUpdate = { authProvider, isActive, lastUsed: null };
     const { encryptor } = await kmsService.createCipherPairWithDataKey({
       type: KmsDataKey.Organization,

backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts

@@ -345,7 +345,7 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => {
 
   const findProjectRequestCount = async (projectId: string, userId: string, policyId?: string, tx?: Knex) => {
     try {
-      const docs = await (tx || db)
+      const docs = await (tx || db.replicaNode())
         .with(
           "temp",
           (tx || db.replicaNode())(TableName.SecretApprovalRequest)
@@ -494,7 +494,7 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => {
         .distinctOn(`${TableName.SecretApprovalRequest}.id`)
         .as("inner");
 
-      const query = (tx || db)
+      const query = (tx || db.replicaNode())
         .select("*")
         .select(db.raw("count(*) OVER() as total_count"))
         .from(innerQuery)

backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts

@@ -377,7 +377,7 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => {
   // special query for migration to v2 secret
   const findByProjectId = async (projectId: string, tx?: Knex) => {
     try {
-      const docs = await (tx || db)(TableName.SecretApprovalRequestSecret)
+      const docs = await (tx || db.replicaNode())(TableName.SecretApprovalRequestSecret)
         .join(
           TableName.SecretApprovalRequest,
           `${TableName.SecretApprovalRequest}.id`,

backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts

@@ -787,6 +787,7 @@ export const secretApprovalRequestServiceFactory = ({
           },
           tx
         );
+        await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId, tx);
         return {
           secrets: { created: newSecrets, updated: updatedSecrets, deleted: deletedSecret },
           approval: updatedSecretApproval
@@ -976,6 +977,7 @@ export const secretApprovalRequestServiceFactory = ({
           },
           tx
         );
+        await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId, tx);
         return {
           secrets: { created: newSecrets, updated: updatedSecrets, deleted: deletedSecret },
           approval: updatedSecretApproval
@@ -983,7 +985,6 @@ export const secretApprovalRequestServiceFactory = ({
       });
     }
 
-    await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
     await snapshotService.performSnapshot(folderId);
     const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [folderId]);
     if (!folder) {

backend/src/ee/services/secret-replication/secret-replication-service.ts

@@ -509,9 +509,9 @@ export const secretReplicationServiceFactory = ({
                     tx
                   );
                 }
+                await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId, tx);
               });
 
-              await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
               await secretQueueService.syncSecrets({
                 projectId,
                 orgId,

backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts

@@ -361,9 +361,8 @@ export const secretRotationQueueFactory = ({
             },
             tx
           );
+          await secretV2BridgeDAL.invalidateSecretCacheByProjectId(secretRotation.projectId, tx);
         });
-
-        await secretV2BridgeDAL.invalidateSecretCacheByProjectId(secretRotation.projectId);
       } else {
         if (!botKey)
           throw new NotFoundError({

backend/src/ee/services/secret-snapshot/snapshot-dal.ts

@@ -265,7 +265,7 @@ export const snapshotDALFactory = (db: TDbClient) => {
   // then joins with respective secrets and folder
   const findRecursivelySnapshots = async (snapshotId: string, tx?: Knex) => {
     try {
-      const data = await (tx || db)
+      const data = await (tx || db.replicaNode())
         .withRecursive("parent", (qb) => {
           void qb
             .from(TableName.Snapshot)
@@ -419,7 +419,7 @@ export const snapshotDALFactory = (db: TDbClient) => {
   // then joins with respective secrets and folder
   const findRecursivelySnapshotsV2Bridge = async (snapshotId: string, tx?: Knex) => {
     try {
-      const data = await (tx || db)
+      const data = await (tx || db.replicaNode())
         .withRecursive("parent", (qb) => {
           void qb
             .from(TableName.Snapshot)
@@ -581,7 +581,11 @@ export const snapshotDALFactory = (db: TDbClient) => {
       const docs = await (tx || db.replicaNode())(TableName.Snapshot)
         .where(`${TableName.Snapshot}.folderId`, folderId)
         .join<TSecretSnapshots>(
-          (tx || db)(TableName.Snapshot).groupBy("folderId").max("createdAt").select("folderId").as("latestVersion"),
+          (tx || db.replicaNode())(TableName.Snapshot)
+            .groupBy("folderId")
+            .max("createdAt")
+            .select("folderId")
+            .as("latestVersion"),
           (bd) => {
             bd.on(`${TableName.Snapshot}.folderId`, "latestVersion.folderId").andOn(
               `${TableName.Snapshot}.createdAt`,
@@ -766,7 +770,7 @@ export const snapshotDALFactory = (db: TDbClient) => {
         )
         .orderBy(`${TableName.Snapshot}.createdAt`, "desc")
         .where(`${TableName.Snapshot}.folderId`, folderId);
-      const data = await (tx || db)
+      const data = await (tx || db.replicaNode())
         .with("w", query)
         .select("*")
         .from<Awaited<typeof query>[number]>("w")