From f1b6cd9974cc1f33c7ec50303545c85ffd21ae14 Mon Sep 17 00:00:00 2001 From: Carlos Monastyrski Date: Thu, 17 Jul 2025 23:46:16 -0300 Subject: [PATCH 01/18] Add card declined alert --- frontend/src/hooks/api/subscriptions/types.ts | 1 + .../components/NavBar/Navbar.tsx | 65 ++++++++++++++++++- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/frontend/src/hooks/api/subscriptions/types.ts b/frontend/src/hooks/api/subscriptions/types.ts index 87a02231f..0893d3bd1 100644 --- a/frontend/src/hooks/api/subscriptions/types.ts +++ b/frontend/src/hooks/api/subscriptions/types.ts @@ -53,4 +53,5 @@ export type SubscriptionPlan = { secretScanning: boolean; enterpriseSecretSyncs: boolean; enterpriseAppConnections: boolean; + cardDeclined?: boolean; }; diff --git a/frontend/src/layouts/OrganizationLayout/components/NavBar/Navbar.tsx b/frontend/src/layouts/OrganizationLayout/components/NavBar/Navbar.tsx index a46578726..8af8a24da 100644 --- a/frontend/src/layouts/OrganizationLayout/components/NavBar/Navbar.tsx +++ b/frontend/src/layouts/OrganizationLayout/components/NavBar/Navbar.tsx @@ -8,6 +8,7 @@ import { faCaretDown, faCheck, faEnvelope, + faExclamationTriangle, faInfo, faInfoCircle, faSignOut, @@ -109,6 +110,9 @@ export const Navbar = () => { const { subscription } = useSubscription(); const { currentOrg } = useOrganization(); const [showAdminsModal, setShowAdminsModal] = useState(false); + const [showCardDeclinedModal, setShowCardDeclinedModal] = useState( + subscription?.cardDeclined || false + ); const { data: orgs } = useGetOrganizations(); const navigate = useNavigate(); @@ -195,8 +199,23 @@ export const Navbar = () => {
{currentOrg?.name}
-
- {getPlan(subscription)} +
+
+ {getPlan(subscription)} +
+ {subscription.cardDeclined && ( + +
+ +
+ + )}
@@ -394,6 +413,48 @@ export const Navbar = () => { + + + + Your payment method has been declined + + } + > +
+
+
+

+ Your payment method was declined and your subscription may be at risk. Please + update your payment information to continue using premium features. +

+
+
+
+ + + + +
+
+
+
+ +
From 8666f328e2f2b78b08222a3bf2e5b51c3a38b398 Mon Sep 17 00:00:00 2001 From: Carlos Monastyrski Date: Wed, 30 Jul 2025 11:58:35 -0300 Subject: [PATCH 02/18] Add declined payment reason and force refresh on billing page load --- backend/src/ee/routes/v1/license-router.ts | 9 +++++++- .../ee/services/license/license-service.ts | 13 ++++++++++- .../src/ee/services/license/license-types.ts | 1 + .../SubscriptionContext.tsx | 4 ++-- .../src/hooks/api/subscriptions/queries.tsx | 4 ++-- frontend/src/hooks/api/subscriptions/types.ts | 1 + .../components/NavBar/Navbar.tsx | 22 ++++++++++++------- .../BillingCloudTab/PreviewSection.tsx | 12 +++++++++- 8 files changed, 51 insertions(+), 15 deletions(-) diff --git a/backend/src/ee/routes/v1/license-router.ts b/backend/src/ee/routes/v1/license-router.ts index 0a59fa7b5..17923975d 100644 --- a/backend/src/ee/routes/v1/license-router.ts +++ b/backend/src/ee/routes/v1/license-router.ts @@ -43,6 +43,12 @@ export const registerLicenseRouter = async (server: FastifyZodProvider) => { }, schema: { params: z.object({ organizationId: z.string().trim() }), + querystring: z.object({ + refreshCache: z + .enum(["true", "false"]) + .default("false") + .transform((value) => value === "true") + }), response: { 200: z.object({ plan: z.any() }) } @@ -54,7 +60,8 @@ export const registerLicenseRouter = async (server: FastifyZodProvider) => { actor: req.permission.type, actorOrgId: req.permission.orgId, actorAuthMethod: req.permission.authMethod, - orgId: req.params.organizationId + orgId: req.params.organizationId, + refreshCache: req.query.refreshCache }); return { plan }; } diff --git a/backend/src/ee/services/license/license-service.ts b/backend/src/ee/services/license/license-service.ts index 7e784d9ad..bb208ab2e 100644 --- a/backend/src/ee/services/license/license-service.ts +++ b/backend/src/ee/services/license/license-service.ts @@ -295,8 +295,19 @@ export const licenseServiceFactory = ({ return data; }; - const getOrgPlan = async ({ orgId, actor, actorId, actorOrgId, actorAuthMethod, projectId }: TOrgPlanDTO) => { + const getOrgPlan = async ({ + orgId, + actor, + actorId, + actorOrgId, + actorAuthMethod, + projectId, + refreshCache + }: TOrgPlanDTO) => { await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId); + if (refreshCache) { + await refreshPlan(orgId); + } const plan = await getPlan(orgId, projectId); return plan; }; diff --git a/backend/src/ee/services/license/license-types.ts b/backend/src/ee/services/license/license-types.ts index a3412f574..ea94f6172 100644 --- a/backend/src/ee/services/license/license-types.ts +++ b/backend/src/ee/services/license/license-types.ts @@ -84,6 +84,7 @@ export type TOrgPlansTableDTO = { export type TOrgPlanDTO = { projectId?: string; + refreshCache?: boolean; } & TOrgPermission; export type TStartOrgTrialDTO = { diff --git a/frontend/src/context/SubscriptionContext/SubscriptionContext.tsx b/frontend/src/context/SubscriptionContext/SubscriptionContext.tsx index de52e5f18..95da6bb01 100644 --- a/frontend/src/context/SubscriptionContext/SubscriptionContext.tsx +++ b/frontend/src/context/SubscriptionContext/SubscriptionContext.tsx @@ -3,7 +3,7 @@ import { useRouteContext } from "@tanstack/react-router"; import { fetchOrgSubscription, subscriptionQueryKeys } from "@app/hooks/api/subscriptions/queries"; -export const useSubscription = () => { +export const useSubscription = (refreshCache?: boolean) => { const organizationId = useRouteContext({ from: "/_authenticate/_inject-org-details", select: (el) => el.organizationId @@ -11,7 +11,7 @@ export const useSubscription = () => { const { data: subscription } = useSuspenseQuery({ queryKey: subscriptionQueryKeys.getOrgSubsription(organizationId), - queryFn: () => fetchOrgSubscription(organizationId), + queryFn: () => fetchOrgSubscription(organizationId, refreshCache), staleTime: Infinity }); diff --git a/frontend/src/hooks/api/subscriptions/queries.tsx b/frontend/src/hooks/api/subscriptions/queries.tsx index 99b1f4486..f545565eb 100644 --- a/frontend/src/hooks/api/subscriptions/queries.tsx +++ b/frontend/src/hooks/api/subscriptions/queries.tsx @@ -10,9 +10,9 @@ export const subscriptionQueryKeys = { getOrgSubsription: (orgID: string) => ["plan", { orgID }] as const }; -export const fetchOrgSubscription = async (orgID: string) => { +export const fetchOrgSubscription = async (orgID: string, refreshCache: boolean = false) => { const { data } = await apiRequest.get<{ plan: SubscriptionPlan }>( - `/api/v1/organizations/${orgID}/plan` + `/api/v1/organizations/${orgID}/plan${refreshCache ? "?refreshCache=true" : ""}` ); return data.plan; diff --git a/frontend/src/hooks/api/subscriptions/types.ts b/frontend/src/hooks/api/subscriptions/types.ts index 0893d3bd1..6042df67d 100644 --- a/frontend/src/hooks/api/subscriptions/types.ts +++ b/frontend/src/hooks/api/subscriptions/types.ts @@ -54,4 +54,5 @@ export type SubscriptionPlan = { enterpriseSecretSyncs: boolean; enterpriseAppConnections: boolean; cardDeclined?: boolean; + cardDeclinedReason?: string; }; diff --git a/frontend/src/layouts/OrganizationLayout/components/NavBar/Navbar.tsx b/frontend/src/layouts/OrganizationLayout/components/NavBar/Navbar.tsx index 8af8a24da..96da887b1 100644 --- a/frontend/src/layouts/OrganizationLayout/components/NavBar/Navbar.tsx +++ b/frontend/src/layouts/OrganizationLayout/components/NavBar/Navbar.tsx @@ -1,4 +1,4 @@ -import { useState } from "react"; +import { useEffect, useState } from "react"; import { faGithub, faSlack } from "@fortawesome/free-brands-svg-icons"; import { faCircleQuestion, faUserCircle } from "@fortawesome/free-regular-svg-icons"; import { @@ -110,9 +110,14 @@ export const Navbar = () => { const { subscription } = useSubscription(); const { currentOrg } = useOrganization(); const [showAdminsModal, setShowAdminsModal] = useState(false); - const [showCardDeclinedModal, setShowCardDeclinedModal] = useState( - subscription?.cardDeclined || false - ); + const [showCardDeclinedModal, setShowCardDeclinedModal] = useState(false); + + useEffect(() => { + if (subscription?.cardDeclined && !sessionStorage.getItem("paymentFailed")) { + sessionStorage.setItem("paymentFailed", "true"); + setShowCardDeclinedModal(true); + } + }, [subscription]); const { data: orgs } = useGetOrganizations(); const navigate = useNavigate(); @@ -205,7 +210,7 @@ export const Navbar = () => {
{subscription.cardDeclined && (
@@ -418,7 +423,7 @@ export const Navbar = () => { title={
- Your payment method has been declined + Your payment could not be processed.
} > @@ -426,8 +431,9 @@ export const Navbar = () => {

- Your payment method was declined and your subscription may be at risk. Please - update your payment information to continue using premium features. + We were unable to process your last payment + {subscription.cardDeclinedReason ? `: ${subscription.cardDeclinedReason}` : ""}. + Please update your payment information to continue using premium features.

diff --git a/frontend/src/pages/organization/BillingPage/components/BillingCloudTab/PreviewSection.tsx b/frontend/src/pages/organization/BillingPage/components/BillingCloudTab/PreviewSection.tsx index eaa90450e..6ab0429ee 100644 --- a/frontend/src/pages/organization/BillingPage/components/BillingCloudTab/PreviewSection.tsx +++ b/frontend/src/pages/organization/BillingPage/components/BillingCloudTab/PreviewSection.tsx @@ -1,5 +1,7 @@ +import { useEffect } from "react"; import { faArrowUpRightFromSquare } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { useQueryClient } from "@tanstack/react-query"; import { OrgPermissionCan } from "@app/components/permissions"; import { Button } from "@app/components/v2"; @@ -15,13 +17,15 @@ import { useGetOrgPlanBillingInfo, useGetOrgTrialUrl } from "@app/hooks/api"; +import { subscriptionQueryKeys } from "@app/hooks/api/subscriptions/queries"; import { usePopUp } from "@app/hooks/usePopUp"; import { ManagePlansModal } from "./ManagePlansModal"; export const PreviewSection = () => { const { currentOrg } = useOrganization(); - const { subscription } = useSubscription(); + const { subscription } = useSubscription(true); + const queryClient = useQueryClient(); const { data, isPending } = useGetOrgPlanBillingInfo(currentOrg?.id ?? ""); const getOrgTrialUrl = useGetOrgTrialUrl(); const createCustomerPortalSession = useCreateCustomerPortalSession(); @@ -37,6 +41,12 @@ export const PreviewSection = () => { return formattedTotal; }; + useEffect(() => { + queryClient.invalidateQueries({ + queryKey: subscriptionQueryKeys.getOrgSubsription(currentOrg?.id ?? "") + }); + }, []); + const formatDate = (date: number) => { const createdDate = new Date(date * 1000); const day: number = createdDate.getDate(); From 301b9a46b383fbd88ccca8954787b2d8b16e3659 Mon Sep 17 00:00:00 2001 From: Scott Wilson Date: Mon, 8 Sep 2025 16:16:48 -0700 Subject: [PATCH 03/18] improvement(google-oauth): prevent google oauth enforcement if sso configured, and prevent sso enablement if google oauth enforcement is enabled on both back/frontend and improve labeling --- .../ldap-config/ldap-config-service.ts | 27 +++++++ .../ee/services/oidc/oidc-config-service.ts | 14 ++++ .../saml-config/saml-config-service.ts | 26 +++++++ backend/src/keystore/keystore.ts | 3 +- backend/src/server/routes/index.ts | 1 + backend/src/services/org/org-service.ts | 39 +++++++++++ .../permissions/OrgPermissionCan.tsx | 15 +++- .../OrgSsoTab/OrgGeneralAuthSection.tsx | 70 ++++++++++++++----- .../components/OrgSsoTab/OrgLDAPSection.tsx | 35 +++++++--- .../components/OrgSsoTab/OrgOIDCSection.tsx | 31 ++++++-- .../components/OrgSsoTab/OrgSSOSection.tsx | 31 ++++++-- .../components/OrgSsoTab/OrgSsoTab.tsx | 3 + 12 files changed, 252 insertions(+), 43 deletions(-) diff --git a/backend/src/ee/services/ldap-config/ldap-config-service.ts b/backend/src/ee/services/ldap-config/ldap-config-service.ts index a72d50760..4592cf003 100644 --- a/backend/src/ee/services/ldap-config/ldap-config-service.ts +++ b/backend/src/ee/services/ldap-config/ldap-config-service.ts @@ -127,6 +127,20 @@ export const ldapConfigServiceFactory = ({ message: "Failed to create LDAP configuration due to plan restriction. Upgrade plan to create LDAP configuration." }); + + const org = await orgDAL.findOrgById(orgId); + + if (!org) { + throw new NotFoundError({ message: `Could not find organization with ID "${orgId}"` }); + } + + if (org.googleSsoAuthEnforced && isActive) { + throw new BadRequestError({ + message: + "You cannot enable LDAP SSO while Google OAuth is enforced. Disable Google OAuth enforcement to enable LDAP SSO." + }); + } + const { encryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.Organization, orgId @@ -233,6 +247,19 @@ export const ldapConfigServiceFactory = ({ "Failed to update LDAP configuration due to plan restriction. Upgrade plan to update LDAP configuration." }); + const org = await orgDAL.findOrgById(orgId); + + if (!org) { + throw new NotFoundError({ message: `Could not find organization with ID "${orgId}"` }); + } + + if (org.googleSsoAuthEnforced && isActive) { + throw new BadRequestError({ + message: + "You cannot enable LDAP SSO while Google OAuth is enforced. Disable Google OAuth enforcement to enable LDAP SSO." + }); + } + const updateQuery: TLdapConfigsUpdate = { isActive, url, diff --git a/backend/src/ee/services/oidc/oidc-config-service.ts b/backend/src/ee/services/oidc/oidc-config-service.ts index 8f479b12c..fb1e70a85 100644 --- a/backend/src/ee/services/oidc/oidc-config-service.ts +++ b/backend/src/ee/services/oidc/oidc-config-service.ts @@ -499,6 +499,13 @@ export const oidcConfigServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.Sso); + if (org.googleSsoAuthEnforced && isActive) { + throw new BadRequestError({ + message: + "You cannot enable OIDC SSO while Google OAuth is enforced. Disable Google OAuth enforcement to enable OIDC SSO." + }); + } + const { encryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.Organization, orgId: org.id @@ -586,6 +593,13 @@ export const oidcConfigServiceFactory = ({ ); ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.Sso); + if (org.googleSsoAuthEnforced && isActive) { + throw new BadRequestError({ + message: + "You cannot enable OIDC SSO while Google OAuth is enforced. Disable Google OAuth enforcement to enable OIDC SSO." + }); + } + const { encryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.Organization, orgId: org.id diff --git a/backend/src/ee/services/saml-config/saml-config-service.ts b/backend/src/ee/services/saml-config/saml-config-service.ts index 6b8bbe304..1cbe6a930 100644 --- a/backend/src/ee/services/saml-config/saml-config-service.ts +++ b/backend/src/ee/services/saml-config/saml-config-service.ts @@ -82,6 +82,19 @@ export const samlConfigServiceFactory = ({ "Failed to create SAML SSO configuration due to plan restriction. Upgrade plan to create SSO configuration." }); + const org = await orgDAL.findOrgById(orgId); + + if (!org) { + throw new NotFoundError({ message: `Could not find organization with ID "${orgId}"` }); + } + + if (org.googleSsoAuthEnforced && isActive) { + throw new BadRequestError({ + message: + "You cannot enable SAML SSO while Google OAuth is enforced. Disable Google OAuth enforcement to enable SAML SSO." + }); + } + const { encryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.Organization, orgId @@ -120,6 +133,19 @@ export const samlConfigServiceFactory = ({ "Failed to update SAML SSO configuration due to plan restriction. Upgrade plan to update SSO configuration." }); + const org = await orgDAL.findOrgById(orgId); + + if (!org) { + throw new NotFoundError({ message: `Could not find organization with ID "${orgId}"` }); + } + + if (org.googleSsoAuthEnforced && isActive) { + throw new BadRequestError({ + message: + "Cannot enable SAML SSO while Google OAuth is enforced. Disable Google OAuth enforcement to enable SAML SSO." + }); + } + const updateQuery: TSamlConfigsUpdate = { authProvider, isActive, lastUsed: null }; const { encryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.Organization, diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index 45589e529..a6135e0ca 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -1,9 +1,10 @@ +import { Cluster, Redis } from "ioredis"; + import { buildRedisFromConfig, TRedisConfigKeys } from "@app/lib/config/redis"; import { pgAdvisoryLockHashText } from "@app/lib/crypto/hashtext"; import { applyJitter } from "@app/lib/dates"; import { delay as delayMs } from "@app/lib/delay"; import { ExecutionResult, Redlock, Settings } from "@app/lib/red-lock"; -import { Redis, Cluster } from "ioredis"; export const PgSqlLock = { BootUpMigration: 2023, diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 64c9b3389..4803a6cd2 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -807,6 +807,7 @@ export const registerRoutes = async ( groupDAL, orgBotDAL, oidcConfigDAL, + ldapConfigDAL, loginService, projectBotService, reminderService diff --git a/backend/src/services/org/org-service.ts b/backend/src/services/org/org-service.ts index 978bc8dae..f99737231 100644 --- a/backend/src/services/org/org-service.ts +++ b/backend/src/services/org/org-service.ts @@ -14,6 +14,7 @@ import { TUsers } from "@app/db/schemas"; import { TGroupDALFactory } from "@app/ee/services/group/group-dal"; +import { TLdapConfigDALFactory } from "@app/ee/services/ldap-config/ldap-config-dal"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { TOidcConfigDALFactory } from "@app/ee/services/oidc/oidc-config-dal"; import { @@ -125,6 +126,7 @@ type TOrgServiceFactoryDep = { incidentContactDAL: TIncidentContactsDALFactory; samlConfigDAL: Pick; oidcConfigDAL: Pick; + ldapConfigDAL: Pick; smtpService: TSmtpService; tokenService: TAuthTokenServiceFactory; permissionService: TPermissionServiceFactory; @@ -165,6 +167,7 @@ export const orgServiceFactory = ({ projectRoleDAL, samlConfigDAL, oidcConfigDAL, + ldapConfigDAL, projectUserMembershipRoleDAL, identityMetadataDAL, projectBotService, @@ -483,6 +486,42 @@ export const orgServiceFactory = ({ }); } + const samlCfg = await samlConfigDAL.findOne({ + orgId, + isActive: true + }); + + if (samlCfg) { + throw new BadRequestError({ + message: + "Cannot enable Google OAuth enforcement while SAML SSO is configured. Disable SAML SSO to enforce Google OAuth." + }); + } + + const oidcCfg = await oidcConfigDAL.findOne({ + orgId, + isActive: true + }); + + if (oidcCfg) { + throw new BadRequestError({ + message: + "Cannot enable Google OAuth enforcement while OIDC SSO is configured. Disable OIDC SSO to enforce Google OAuth." + }); + } + + const ldapCfg = await ldapConfigDAL.findOne({ + orgId, + isActive: true + }); + + if (ldapCfg) { + throw new BadRequestError({ + message: + "Cannot enable Google OAuth enforcement while LDAP SSO is configured. Disable LDAP SSO to enforce Google OAuth." + }); + } + if (!currentOrg.googleSsoAuthLastUsed) { throw new BadRequestError({ message: diff --git a/frontend/src/components/permissions/OrgPermissionCan.tsx b/frontend/src/components/permissions/OrgPermissionCan.tsx index d2e698e88..8e0bf08ad 100644 --- a/frontend/src/components/permissions/OrgPermissionCan.tsx +++ b/frontend/src/components/permissions/OrgPermissionCan.tsx @@ -1,6 +1,7 @@ import { FunctionComponent, ReactNode } from "react"; import { BoundCanProps, Can } from "@casl/react"; +import { TooltipProps } from "@app/components/v2/Tooltip/Tooltip"; import { TOrgPermission, useOrgPermission } from "@app/context/OrgPermissionContext"; import { AccessRestrictedBanner, Tooltip } from "../v2"; @@ -20,6 +21,7 @@ type Props = { renderTooltip?: boolean; allowedLabel?: string; renderGuardBanner?: boolean; + tooltipProps?: Omit; } & BoundCanProps; export const OrgPermissionCan: FunctionComponent = ({ @@ -29,6 +31,7 @@ export const OrgPermissionCan: FunctionComponent = ({ renderTooltip, allowedLabel, renderGuardBanner, + tooltipProps, ...props }) => { const { permission } = useOrgPermission(); @@ -43,11 +46,19 @@ export const OrgPermissionCan: FunctionComponent = ({ : children; if (!isAllowed && passThrough) { - return {finalChild}; + return ( + + {finalChild} + + ); } if (isAllowed && renderTooltip && allowedLabel) { - return {finalChild}; + return ( + + {finalChild} + + ); } if (!isAllowed && renderGuardBanner) { diff --git a/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgGeneralAuthSection.tsx b/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgGeneralAuthSection.tsx index b9ca11748..c3d8cea98 100644 --- a/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgGeneralAuthSection.tsx +++ b/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgGeneralAuthSection.tsx @@ -24,11 +24,17 @@ enum EnforceAuthType { export const OrgGeneralAuthSection = ({ isSamlConfigured, isOidcConfigured, - isGoogleConfigured + isGoogleConfigured, + isSamlActive, + isOidcActive, + isLdapActive }: { isSamlConfigured: boolean; isOidcConfigured: boolean; isGoogleConfigured: boolean; + isSamlActive: boolean; + isOidcActive: boolean; + isLdapActive: boolean; }) => { const { currentOrg } = useOrganization(); const { subscription } = useSubscription(); @@ -126,6 +132,15 @@ export const OrgGeneralAuthSection = ({ } }; + const isGoogleOAuthEnforced = currentOrg.googleSsoAuthEnforced; + + const getActiveSsoLabel = () => { + if (isSamlActive) return "SAML"; + if (isOidcActive) return "OIDC"; + if (isLdapActive) return "LDAP"; + return ""; + }; + return (
@@ -135,7 +150,7 @@ export const OrgGeneralAuthSection = ({

-
+
Enforce SAML SSO @@ -160,7 +175,7 @@ export const OrgGeneralAuthSection = ({

-
+
Enforce OIDC SSO @@ -188,26 +203,47 @@ export const OrgGeneralAuthSection = ({
- Enforce Google SSO + Enforce Google OAuth
- + {(isAllowed) => ( - - handleEnforceOrgAuthToggle(value, EnforceAuthType.GOOGLE) - } - isChecked={currentOrg?.googleSsoAuthEnforced ?? false} - isDisabled={!isAllowed || currentOrg?.authEnforced} - /> +
+ + handleEnforceOrgAuthToggle(value, EnforceAuthType.GOOGLE) + } + isChecked={currentOrg?.googleSsoAuthEnforced ?? false} + isDisabled={ + !isAllowed || + currentOrg?.authEnforced || + isOidcActive || + isSamlActive || + isLdapActive + } + /> +
)}

- Enforce users to authenticate via Google OAuth SSO to access this organization. + Enforce users to authenticate via Google OAuth to access this organization.
When this is enabled your organization members will only be able to login with Google - SSO (not Google SAML). + OAuth (not Google SAML).

@@ -267,8 +303,8 @@ export const OrgGeneralAuthSection = ({

- Allow organization admins to bypass SAML enforcement when SSO is unavailable, - misconfigured, or inaccessible. + Allow organization admins to bypass SSO login enforcement when your SSO provider is + unavailable, misconfigured, or inaccessible.

diff --git a/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgLDAPSection.tsx b/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgLDAPSection.tsx index e66987bae..45392a660 100644 --- a/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgLDAPSection.tsx +++ b/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgLDAPSection.tsx @@ -94,6 +94,8 @@ export const OrgLDAPSection = (): JSX.Element => { handlePopUpOpen("ldapGroupMap"); }; + const isGoogleOAuthEnabled = currentOrg.googleSsoAuthEnforced; + return (
@@ -116,16 +118,31 @@ export const OrgLDAPSection = (): JSX.Element => {

Enable LDAP

- + {(isAllowed) => ( - handleLDAPToggle(value)} - isChecked={data ? data.isActive : false} - isDisabled={!isAllowed} - > - Enable - +
+ handleLDAPToggle(value)} + isChecked={data ? data.isActive : false} + isDisabled={!isAllowed || isGoogleOAuthEnabled} + > + Enable + +
)}
diff --git a/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgOIDCSection.tsx b/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgOIDCSection.tsx index ce10c3907..3ba1a8275 100644 --- a/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgOIDCSection.tsx +++ b/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgOIDCSection.tsx @@ -83,6 +83,8 @@ export const OrgOIDCSection = (): JSX.Element => { } }; + const isGoogleOAuthEnabled = currentOrg.googleSsoAuthEnforced; + return (
@@ -106,14 +108,29 @@ export const OrgOIDCSection = (): JSX.Element => {

Enable OIDC

{!isPending && ( - + {(isAllowed) => ( - handleOIDCToggle(value)} - isChecked={data ? data.isActive : false} - isDisabled={!isAllowed} - /> +
+ handleOIDCToggle(value)} + isChecked={data ? data.isActive : false} + isDisabled={!isAllowed || isGoogleOAuthEnabled} + /> +
)} )} diff --git a/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgSSOSection.tsx b/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgSSOSection.tsx index 33843f50f..53e6cece7 100644 --- a/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgSSOSection.tsx +++ b/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgSSOSection.tsx @@ -78,6 +78,8 @@ export const OrgSSOSection = (): JSX.Element => { } }; + const isGoogleOAuthEnabled = currentOrg.googleSsoAuthEnforced; + return (
@@ -99,14 +101,29 @@ export const OrgSSOSection = (): JSX.Element => {

Enable SAML

{!isPending && ( - + {(isAllowed) => ( - handleSamlSSOToggle(value)} - isChecked={data ? data.isActive : false} - isDisabled={!isAllowed} - /> +
+ handleSamlSSOToggle(value)} + isChecked={data ? data.isActive : false} + isDisabled={!isAllowed || isGoogleOAuthEnabled} + /> +
)} )} diff --git a/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgSsoTab.tsx b/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgSsoTab.tsx index 9964fdf4d..e4e3a9c23 100644 --- a/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgSsoTab.tsx +++ b/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgSsoTab.tsx @@ -184,6 +184,9 @@ export const OrgSsoTab = withPermission( isSamlConfigured={isSamlConfigured} isOidcConfigured={isOidcConfigured} isGoogleConfigured={isGoogleConfigured} + isSamlActive={Boolean(samlConfig?.isActive)} + isOidcActive={Boolean(oidcConfig?.isActive)} + isLdapActive={Boolean(ldapConfig?.isActive)} /> )} From 9ad8bc697714aaff400d0aed8a165089eb7917eb Mon Sep 17 00:00:00 2001 From: Scott Wilson Date: Mon, 8 Sep 2025 16:26:14 -0700 Subject: [PATCH 04/18] improvement: correct switch enable switch id --- .../SettingsPage/components/OrgSsoTab/OrgLDAPSection.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgLDAPSection.tsx b/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgLDAPSection.tsx index 45392a660..a17483fdd 100644 --- a/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgLDAPSection.tsx +++ b/frontend/src/pages/organization/SettingsPage/components/OrgSsoTab/OrgLDAPSection.tsx @@ -135,7 +135,7 @@ export const OrgLDAPSection = (): JSX.Element => { {(isAllowed) => (
handleLDAPToggle(value)} isChecked={data ? data.isActive : false} isDisabled={!isAllowed || isGoogleOAuthEnabled} From 48f17f4d8db4768c44551401244336257361cbe1 Mon Sep 17 00:00:00 2001 From: x032205 Date: Mon, 8 Sep 2025 20:31:15 -0400 Subject: [PATCH 05/18] feat(ua-login): improve lock error message & disable lock when lockout is disabled --- .../identity-ua/identity-ua-service.ts | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/backend/src/services/identity-ua/identity-ua-service.ts b/backend/src/services/identity-ua/identity-ua-service.ts index 597747683..8aec16371 100644 --- a/backend/src/services/identity-ua/identity-ua-service.ts +++ b/backend/src/services/identity-ua/identity-ua-service.ts @@ -84,18 +84,20 @@ export const identityUaServiceFactory = ({ const LOCKOUT_KEY = `lockout:identity:${identityUa.identityId}:${IdentityAuthMethod.UNIVERSAL_AUTH}:${clientId}`; - let lock: Awaited>; - try { - lock = await keyStore.acquireLock([KeyStorePrefixes.IdentityLockoutLock(LOCKOUT_KEY)], 500, { - retryCount: 3, - retryDelay: 300, - retryJitter: 100 - }); - } catch (e) { - logger.info( - `identity login failed to acquire lock [identityId=${identityUa.identityId}] [authMethod=${IdentityAuthMethod.UNIVERSAL_AUTH}]` - ); - throw new RateLimitError({ message: "Rate limit exceeded" }); + let lock: Awaited> | undefined; + if (identityUa.lockoutEnabled) { + try { + lock = await keyStore.acquireLock([KeyStorePrefixes.IdentityLockoutLock(LOCKOUT_KEY)], 500, { + retryCount: 3, + retryDelay: 300, + retryJitter: 100 + }); + } catch (e) { + logger.info( + `identity login failed to acquire lock [identityId=${identityUa.identityId}] [authMethod=${IdentityAuthMethod.UNIVERSAL_AUTH}]` + ); + throw new RateLimitError({ message: "Failed to acquire lock: rate limit exceeded" }); + } } try { @@ -257,7 +259,7 @@ export const identityUaServiceFactory = ({ ...accessTokenTTLParams }; } finally { - await lock.release(); + if (lock) await lock.release(); } }; From b113eabeb70ba4a56511931d99e0b0a89dbccab3 Mon Sep 17 00:00:00 2001 From: Scott Wilson Date: Tue, 9 Sep 2025 10:27:41 -0700 Subject: [PATCH 06/18] improvement: address feedback --- backend/src/services/org/org-service.ts | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/backend/src/services/org/org-service.ts b/backend/src/services/org/org-service.ts index f99737231..ffcf4459e 100644 --- a/backend/src/services/org/org-service.ts +++ b/backend/src/services/org/org-service.ts @@ -9,8 +9,10 @@ import { ProjectMembershipRole, ProjectVersion, TableName, + TOidcConfigs, TProjectMemberships, TProjectUserMembershipRolesInsert, + TSamlConfigs, TUsers } from "@app/db/schemas"; import { TGroupDALFactory } from "@app/ee/services/group/group-dal"; @@ -449,16 +451,20 @@ export const orgServiceFactory = ({ }); } - if (authEnforced) { - const samlCfg = await samlConfigDAL.findOne({ + let samlCfg: TSamlConfigs | undefined; + let oidcCfg: TOidcConfigs | undefined; + if (authEnforced || googleSsoAuthEnforced) { + samlCfg = await samlConfigDAL.findOne({ orgId, isActive: true }); - const oidcCfg = await oidcConfigDAL.findOne({ + oidcCfg = await oidcConfigDAL.findOne({ orgId, isActive: true }); + } + if (authEnforced) { if (!samlCfg && !oidcCfg) throw new NotFoundError({ message: `SAML or OIDC configuration for organization with ID '${orgId}' not found` @@ -486,11 +492,6 @@ export const orgServiceFactory = ({ }); } - const samlCfg = await samlConfigDAL.findOne({ - orgId, - isActive: true - }); - if (samlCfg) { throw new BadRequestError({ message: @@ -498,11 +499,6 @@ export const orgServiceFactory = ({ }); } - const oidcCfg = await oidcConfigDAL.findOne({ - orgId, - isActive: true - }); - if (oidcCfg) { throw new BadRequestError({ message: From d7fae950bfb36d30fe2cfe6f66cac169f0856ddc Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Wed, 10 Sep 2025 15:02:22 +0530 Subject: [PATCH 07/18] fix: resolved using host when it's postgres with gateway --- .../src/ee/services/dynamic-secret/providers/sql-database.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/src/ee/services/dynamic-secret/providers/sql-database.ts b/backend/src/ee/services/dynamic-secret/providers/sql-database.ts index 57ce710b1..e68e41c6b 100644 --- a/backend/src/ee/services/dynamic-secret/providers/sql-database.ts +++ b/backend/src/ee/services/dynamic-secret/providers/sql-database.ts @@ -161,7 +161,10 @@ export const SqlDatabaseProvider = ({ gatewayService }: TSqlDatabaseProviderDTO) connection: { database: providerInputs.database, port: providerInputs.port, - host: providerInputs.client === SqlProviders.Postgres ? providerInputs.hostIp : providerInputs.host, + host: + providerInputs.client === SqlProviders.Postgres && !providerInputs.gatewayId + ? providerInputs.hostIp + : providerInputs.host, user: providerInputs.username, password: providerInputs.password, ssl, From d6f8189a9677397916c852aff15624824988b438 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 9 Sep 2025 14:07:19 +0530 Subject: [PATCH 08/18] feat: added cache versioning to sql for geo invalidation --- backend/e2e-test/mocks/keystore.ts | 9 ++ backend/e2e-test/vitest-environment-knex.ts | 4 +- backend/src/@types/knex.d.ts | 8 ++ .../20250908193226_sql-cache_int.ts | 17 ++++ backend/src/db/schemas/index.ts | 1 + backend/src/db/schemas/key-value-store.ts | 18 ++++ backend/src/db/schemas/models.ts | 3 +- .../secret-approval-request-service.ts | 2 +- .../secret-replication-service.ts | 2 +- .../secret-rotation-queue.ts | 3 +- backend/src/keystore/key-value-store-dal.ts | 91 +++++++++++++++++++ backend/src/keystore/keystore.ts | 73 +++++++++------ backend/src/keystore/memory.ts | 9 ++ backend/src/lib/config/env.ts | 1 + backend/src/lib/knex/index.ts | 8 +- backend/src/main.ts | 4 +- backend/src/server/routes/index.ts | 7 +- .../folder-commit/folder-commit-service.ts | 2 +- .../resource-cleanup-queue.ts | 10 +- .../secret-v2-bridge/secret-v2-bridge-dal.ts | 7 +- .../secret-v2-bridge-service.ts | 41 +++++---- 21 files changed, 258 insertions(+), 62 deletions(-) create mode 100644 backend/src/db/migrations/20250908193226_sql-cache_int.ts create mode 100644 backend/src/db/schemas/key-value-store.ts create mode 100644 backend/src/keystore/key-value-store-dal.ts diff --git a/backend/e2e-test/mocks/keystore.ts b/backend/e2e-test/mocks/keystore.ts index 91b64ff0d..0cebe6ef3 100644 --- a/backend/e2e-test/mocks/keystore.ts +++ b/backend/e2e-test/mocks/keystore.ts @@ -56,6 +56,15 @@ export const mockKeyStore = (): TKeyStoreFactory => { incrementBy: async () => { return 1; }, + pgGetIntItem: async (key) => { + const value = store[key]; + if (typeof value === "number") { + return Number(value); + } + }, + pgIncrementBy: async () => { + return 1; + }, getItems: async (keys) => { const values = keys.map((key) => { const value = store[key]; diff --git a/backend/e2e-test/vitest-environment-knex.ts b/backend/e2e-test/vitest-environment-knex.ts index ff5f42286..085b8fe30 100644 --- a/backend/e2e-test/vitest-environment-knex.ts +++ b/backend/e2e-test/vitest-environment-knex.ts @@ -15,6 +15,7 @@ import { mockSmtpServer } from "./mocks/smtp"; import { initDbConnection } from "@app/db"; import { queueServiceFactory } from "@app/queue"; import { keyStoreFactory } from "@app/keystore/keystore"; +import { keyValueStoreDALFactory } from "@app/keystore/key-value-store-dal"; import { initializeHsmModule } from "@app/ee/services/hsm/hsm-fns"; import { buildRedisFromConfig } from "@app/lib/config/redis"; import { superAdminDALFactory } from "@app/services/super-admin/super-admin-dal"; @@ -62,7 +63,8 @@ export default { const smtp = mockSmtpServer(); const queue = queueServiceFactory(envCfg, { dbConnectionUrl: envCfg.DB_CONNECTION_URI }); - const keyStore = keyStoreFactory(envCfg); + const keyValueStoreDAL = keyValueStoreDALFactory(db); + const keyStore = keyStoreFactory(envCfg, keyValueStoreDAL); await queue.initialize(); diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index 525ad1619..d14084885 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -191,6 +191,9 @@ import { TInternalKms, TInternalKmsInsert, TInternalKmsUpdate, + TKeyValueStore, + TKeyValueStoreInsert, + TKeyValueStoreUpdate, TKmipClientCertificates, TKmipClientCertificatesInsert, TKmipClientCertificatesUpdate, @@ -1264,5 +1267,10 @@ declare module "knex/types/tables" { TUserNotificationsInsert, TUserNotificationsUpdate >; + [TableName.KeyValueStore]: KnexOriginal.CompositeTableType< + TKeyValueStore, + TKeyValueStoreInsert, + TKeyValueStoreUpdate + >; } } diff --git a/backend/src/db/migrations/20250908193226_sql-cache_int.ts b/backend/src/db/migrations/20250908193226_sql-cache_int.ts new file mode 100644 index 000000000..dc4060d02 --- /dev/null +++ b/backend/src/db/migrations/20250908193226_sql-cache_int.ts @@ -0,0 +1,17 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasTable(TableName.KeyValueStore))) { + await knex.schema.createTable(TableName.KeyValueStore, (t) => { + t.text("key").primary(); + t.bigint("integerValue"); + t.datetime("expiresAt"); + }); + } +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists(TableName.KeyValueStore); +} diff --git a/backend/src/db/schemas/index.ts b/backend/src/db/schemas/index.ts index 1642c3555..e63ac4901 100644 --- a/backend/src/db/schemas/index.ts +++ b/backend/src/db/schemas/index.ts @@ -61,6 +61,7 @@ export * from "./integration-auths"; export * from "./integrations"; export * from "./internal-certificate-authorities"; export * from "./internal-kms"; +export * from "./key-value-store"; export * from "./kmip-client-certificates"; export * from "./kmip-clients"; export * from "./kmip-org-configs"; diff --git a/backend/src/db/schemas/key-value-store.ts b/backend/src/db/schemas/key-value-store.ts new file mode 100644 index 000000000..d6ba90b2f --- /dev/null +++ b/backend/src/db/schemas/key-value-store.ts @@ -0,0 +1,18 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const KeyValueStoreSchema = z.object({ + key: z.string(), + integerValue: z.coerce.number().nullable().optional(), + expiresAt: z.date().nullable().optional() +}); + +export type TKeyValueStore = z.infer; +export type TKeyValueStoreInsert = Omit, TImmutableDBKeys>; +export type TKeyValueStoreUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 3e3e81fd0..37abc2b61 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -179,7 +179,8 @@ export enum TableName { SecretScanningConfig = "secret_scanning_configs", // reminders Reminder = "reminders", - ReminderRecipient = "reminders_recipients" + ReminderRecipient = "reminders_recipients", + KeyValueStore = "key_value_store" } export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt" | "commitId"; diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index d485f7ea0..aa3e3dc0f 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -976,6 +976,7 @@ export const secretApprovalRequestServiceFactory = ({ }, tx ); + await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId, tx); return { secrets: { created: newSecrets, updated: updatedSecrets, deleted: deletedSecret }, approval: updatedSecretApproval @@ -983,7 +984,6 @@ export const secretApprovalRequestServiceFactory = ({ }); } - await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId); await snapshotService.performSnapshot(folderId); const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [folderId]); if (!folder) { diff --git a/backend/src/ee/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts index db41d00f7..93147d9e4 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-service.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -509,9 +509,9 @@ export const secretReplicationServiceFactory = ({ tx ); } + await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId, tx); }); - await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId); await secretQueueService.syncSecrets({ projectId, orgId, diff --git a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts index 1d5c1cedf..557e71e6c 100644 --- a/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts +++ b/backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts @@ -361,9 +361,8 @@ export const secretRotationQueueFactory = ({ }, tx ); + await secretV2BridgeDAL.invalidateSecretCacheByProjectId(secretRotation.projectId, tx); }); - - await secretV2BridgeDAL.invalidateSecretCacheByProjectId(secretRotation.projectId); } else { if (!botKey) throw new NotFoundError({ diff --git a/backend/src/keystore/key-value-store-dal.ts b/backend/src/keystore/key-value-store-dal.ts new file mode 100644 index 000000000..bdf5e8725 --- /dev/null +++ b/backend/src/keystore/key-value-store-dal.ts @@ -0,0 +1,91 @@ +import { Knex } from "knex"; + +import { TDbClient } from "@app/db"; +import { TableName } from "@app/db/schemas"; +import { ormify, TOrmify } from "@app/lib/knex"; +import { logger } from "@app/lib/logger"; +import { QueueName } from "@app/queue"; + +export interface TKeyValueStoreDALFactory extends TOrmify { + incrementBy: (key: string, dto: { incr?: number; tx?: Knex; expiresAt?: Date }) => Promise; + findOneInt: (key: string, tx?: Knex) => Promise; + pruneExpiredKeys: () => Promise; +} + +const QUERY_TIMEOUT_MS = 10 * 60 * 1000; // 10 minutes +const CACHE_KEY_PRUNE_BATCH_SIZE = 10000; +const MAX_RETRY_ON_FAILURE = 3; + +export const keyValueStoreDALFactory = (db: TDbClient): TKeyValueStoreDALFactory => { + const keyValueStoreOrm = ormify(db, TableName.KeyValueStore); + + const incrementBy: TKeyValueStoreDALFactory["incrementBy"] = (key, { incr = 1, tx, expiresAt }) => { + return (tx || db)(TableName.KeyValueStore) + .insert({ key, integerValue: 1, expiresAt }) + .onConflict("key") + .merge({ + integerValue: db.raw(`"${TableName.KeyValueStore}"."integerValue" + ?`, [incr]), + expiresAt + }) + .returning("integerValue"); + }; + + const findOneInt: TKeyValueStoreDALFactory["findOneInt"] = async (key, tx) => { + const doc = await (tx || db.replicaNode())(TableName.KeyValueStore) + .where({ key }) + .andWhere( + (builder) => + void builder + .whereNull("expiresAt") // no expiry + .orWhere("expiresAt", ">", db.fn.now()) // or not expired + ) + .first() + .select("integerValue"); + return Number(doc?.integerValue || 0); + }; + + // delete all audit log that have expired + const pruneExpiredKeys: TKeyValueStoreDALFactory["pruneExpiredKeys"] = async () => { + let deletedIds: { key: string }[] = []; + let numberOfRetryOnFailure = 0; + let isRetrying = false; + + logger.info(`${QueueName.DailyResourceCleanUp}: db key value store clean up started`); + do { + try { + // eslint-disable-next-line no-await-in-loop + deletedIds = await db.transaction(async (trx) => { + await trx.raw(`SET statement_timeout = ${QUERY_TIMEOUT_MS}`); + + const findExpiredKeysSubQuery = trx(TableName.KeyValueStore) + .where("expiresAt", "<", db.fn.now()) + .select("key") + .limit(CACHE_KEY_PRUNE_BATCH_SIZE); + + // eslint-disable-next-line no-await-in-loop + const results = await trx(TableName.KeyValueStore) + .whereIn("key", findExpiredKeysSubQuery) + .del() + .returning("key"); + + return results; + }); + + numberOfRetryOnFailure = 0; // reset + } catch (error) { + numberOfRetryOnFailure += 1; + deletedIds = []; + logger.error(error, "Failed to clean up db key value"); + } finally { + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => { + setTimeout(resolve, 10); // time to breathe for db + }); + } + isRetrying = numberOfRetryOnFailure > 0; + } while (deletedIds.length > 0 || (isRetrying && numberOfRetryOnFailure < MAX_RETRY_ON_FAILURE)); + logger.info(`${QueueName.DailyResourceCleanUp}: db key value store clean up completed`); + }; + + return { ...keyValueStoreOrm, incrementBy, findOneInt, pruneExpiredKeys }; +}; diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index a6135e0ca..db6adebf1 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -1,11 +1,15 @@ import { Cluster, Redis } from "ioredis"; +import { Knex } from "knex"; import { buildRedisFromConfig, TRedisConfigKeys } from "@app/lib/config/redis"; import { pgAdvisoryLockHashText } from "@app/lib/crypto/hashtext"; import { applyJitter } from "@app/lib/dates"; import { delay as delayMs } from "@app/lib/delay"; +import { ms } from "@app/lib/ms"; import { ExecutionResult, Redlock, Settings } from "@app/lib/red-lock"; +import { TKeyValueStoreDALFactory } from "./key-value-store-dal"; + export const PgSqlLock = { BootUpMigration: 2023, SuperAdminInit: 2024, @@ -95,13 +99,17 @@ export type TKeyStoreFactory = { deleteItemsByKeyIn: (keys: string[]) => Promise; deleteItems: (arg: TDeleteItems) => Promise; incrementBy: (key: string, value: number) => Promise; + getKeysByPattern: (pattern: string, limit?: number) => Promise; + // pg + pgIncrementBy: (key: string, dto: { incr?: number; expiry?: string; tx?: Knex }) => Promise; + pgGetIntItem: (key: string, prefix?: string) => Promise; + // locks acquireLock( resources: string[], duration: number, settings?: Partial ): Promise<{ release: () => Promise }>; waitTillReady: ({ key, waitingCb, keyCheckCb, waitIteration, delay, jitter }: TWaitTillReady) => Promise; - getKeysByPattern: (pattern: string, limit?: number) => Promise; }; const pickPrimaryOrSecondaryRedis = (primary: Redis | Cluster, secondaries?: Array) => { @@ -114,7 +122,10 @@ interface TKeyStoreFactoryDTO extends TRedisConfigKeys { REDIS_READ_REPLICAS?: { host: string; port: number }[]; } -export const keyStoreFactory = (redisConfigKeys: TKeyStoreFactoryDTO): TKeyStoreFactory => { +export const keyStoreFactory = ( + redisConfigKeys: TKeyStoreFactoryDTO, + keyValueStoreDAL: TKeyValueStoreDALFactory +): TKeyStoreFactory => { const primaryRedis = buildRedisFromConfig(redisConfigKeys); const redisReadReplicas = redisConfigKeys.REDIS_READ_REPLICAS?.map((el) => { if (redisConfigKeys.REDIS_URL) { @@ -189,29 +200,6 @@ export const keyStoreFactory = (redisConfigKeys: TKeyStoreFactoryDTO): TKeyStore const setExpiry = async (key: string, expiryInSeconds: number) => primaryRedis.expire(key, expiryInSeconds); - const waitTillReady = async ({ - key, - waitingCb, - keyCheckCb, - waitIteration = 10, - delay = 1000, - jitter = 200 - }: TWaitTillReady) => { - let attempts = 0; - let isReady = keyCheckCb(await getItem(key)); - while (!isReady) { - if (attempts > waitIteration) return; - // eslint-disable-next-line - await new Promise((resolve) => { - waitingCb?.(); - setTimeout(resolve, Math.max(0, applyJitter(delay, jitter))); - }); - attempts += 1; - // eslint-disable-next-line - isReady = keyCheckCb(await getItem(key)); - } - }; - const getKeysByPattern = async (pattern: string, limit?: number) => { let cursor = "0"; const allKeys: string[] = []; @@ -236,6 +224,37 @@ export const keyStoreFactory = (redisConfigKeys: TKeyStoreFactoryDTO): TKeyStore return allKeys; }; + const pgIncrementBy: TKeyStoreFactory["pgIncrementBy"] = async (key, { incr = 1, tx, expiry }) => { + const expiresAt = expiry ? new Date(Date.now() + ms(expiry)) : undefined; + return keyValueStoreDAL.incrementBy(key, { incr, expiresAt, tx }); + }; + + const pgGetIntItem = async (key: string, prefix?: string) => + keyValueStoreDAL.findOneInt(prefix ? `${prefix}:${key}` : key); + + const waitTillReady = async ({ + key, + waitingCb, + keyCheckCb, + waitIteration = 10, + delay = 1000, + jitter = 200 + }: TWaitTillReady) => { + let attempts = 0; + let isReady = keyCheckCb(await getItem(key)); + while (!isReady) { + if (attempts > waitIteration) return; + // eslint-disable-next-line + await new Promise((resolve) => { + waitingCb?.(); + setTimeout(resolve, Math.max(0, applyJitter(delay, jitter))); + }); + attempts += 1; + // eslint-disable-next-line + isReady = keyCheckCb(await getItem(key)); + } + }; + return { setItem, getItem, @@ -250,6 +269,8 @@ export const keyStoreFactory = (redisConfigKeys: TKeyStoreFactoryDTO): TKeyStore waitTillReady, getKeysByPattern, deleteItemsByKeyIn, - getItems + getItems, + pgGetIntItem, + pgIncrementBy }; }; diff --git a/backend/src/keystore/memory.ts b/backend/src/keystore/memory.ts index cf9ba83bd..2f9b77ced 100644 --- a/backend/src/keystore/memory.ts +++ b/backend/src/keystore/memory.ts @@ -53,6 +53,15 @@ export const inMemoryKeyStore = (): TKeyStoreFactory => { } return null; }, + pgGetIntItem: async (key) => { + const value = store[key]; + if (typeof value === "number") { + return Number(value); + } + }, + pgIncrementBy: async () => { + return 1; + }, incrementBy: async () => { return 1; }, diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index c4170aac3..ca76afbd1 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -410,6 +410,7 @@ const envSchema = z Boolean(data.INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_ID) && Boolean(data.INF_APP_CONNECTION_GITHUB_RADAR_APP_CLIENT_SECRET) && Boolean(data.INF_APP_CONNECTION_GITHUB_RADAR_APP_WEBHOOK_SECRET), + isSecondaryInstance: Boolean(data.INFISICAL_PRIMARY_INSTANCE_URL), isHsmConfigured: Boolean(data.HSM_LIB_PATH) && Boolean(data.HSM_PIN) && Boolean(data.HSM_KEY_LABEL) && data.HSM_SLOT !== undefined, samlDefaultOrgSlug: data.DEFAULT_SAML_ORG_SLUG, diff --git a/backend/src/lib/knex/index.ts b/backend/src/lib/knex/index.ts index 090df561a..dbfd29b81 100644 --- a/backend/src/lib/knex/index.ts +++ b/backend/src/lib/knex/index.ts @@ -250,12 +250,12 @@ export const ormify = ( .returning("*"); if ($incr) { Object.entries($incr).forEach(([incrementField, incrementValue]) => { - void query.increment(incrementField, incrementValue); + void query.increment(incrementField, incrementValue as number); }); } if ($decr) { Object.entries($decr).forEach(([incrementField, incrementValue]) => { - void query.decrement(incrementField, incrementValue); + void query.decrement(incrementField, incrementValue as number); }); } const [docs] = await query; @@ -273,12 +273,12 @@ export const ormify = ( // increment and decrement operation in update if ($incr) { Object.entries($incr).forEach(([incrementField, incrementValue]) => { - void query.increment(incrementField, incrementValue); + void query.increment(incrementField, incrementValue as number); }); } if ($decr) { Object.entries($decr).forEach(([incrementField, incrementValue]) => { - void query.increment(incrementField, incrementValue); + void query.increment(incrementField, incrementValue as number); }); } return (await query) as Tables[Tname]["base"][]; diff --git a/backend/src/main.ts b/backend/src/main.ts index 8af47eb0b..7be9f43ec 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -5,6 +5,7 @@ import "./lib/telemetry/instrumentation"; import dotenv from "dotenv"; import { initializeHsmModule } from "@app/ee/services/hsm/hsm-fns"; +import { keyValueStoreDALFactory } from "@app/keystore/key-value-store-dal"; import { runMigrations } from "./auto-start-migrations"; import { initAuditLogDbConnection, initDbConnection } from "./db"; @@ -54,7 +55,8 @@ const run = async () => { await queue.initialize(); - const keyStore = keyStoreFactory(envConfig); + const keyValueStoreDAL = keyValueStoreDALFactory(db); + const keyStore = keyStoreFactory(envConfig, keyValueStoreDAL); const redis = buildRedisFromConfig(envConfig); const hsmModule = initializeHsmModule(envConfig); diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 4803a6cd2..f79362b69 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -123,6 +123,7 @@ import { sshHostGroupMembershipDALFactory } from "@app/ee/services/ssh-host-grou import { sshHostGroupServiceFactory } from "@app/ee/services/ssh-host-group/ssh-host-group-service"; import { trustedIpDALFactory } from "@app/ee/services/trusted-ip/trusted-ip-dal"; import { trustedIpServiceFactory } from "@app/ee/services/trusted-ip/trusted-ip-service"; +import { keyValueStoreDALFactory } from "@app/keystore/key-value-store-dal"; import { TKeyStoreFactory } from "@app/keystore/keystore"; import { getConfig, TEnvConfig } from "@app/lib/config/env"; import { crypto } from "@app/lib/crypto/cryptography"; @@ -507,6 +508,7 @@ export const registerRoutes = async ( const microsoftTeamsIntegrationDAL = microsoftTeamsIntegrationDALFactory(db); const projectMicrosoftTeamsConfigDAL = projectMicrosoftTeamsConfigDALFactory(db); const secretScanningV2DAL = secretScanningV2DALFactory(db); + const keyValueStoreDAL = keyValueStoreDALFactory(db); const eventBusService = eventBusFactory(server.redis); const sseService = sseServiceFactory(eventBusService, server.redis); @@ -643,6 +645,7 @@ export const registerRoutes = async ( const folderTreeCheckpointDAL = folderTreeCheckpointDALFactory(db); const folderCommitDAL = folderCommitDALFactory(db); const folderTreeCheckpointResourcesDAL = folderTreeCheckpointResourcesDALFactory(db); + const folderCommitQueueService = folderCommitQueueServiceFactory({ queueService, folderTreeCheckpointDAL, @@ -1682,6 +1685,7 @@ export const registerRoutes = async ( userDAL, identityDAL }); + const dailyResourceCleanUp = dailyResourceCleanUpQueueServiceFactory({ auditLogDAL, queueService, @@ -1694,7 +1698,8 @@ export const registerRoutes = async ( identityUniversalAuthClientSecretDAL: identityUaClientSecretDAL, serviceTokenService, orgService, - userNotificationDAL + userNotificationDAL, + keyValueStoreDAL }); const dailyReminderQueueService = dailyReminderQueueServiceFactory({ diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.ts index 470edbbba..e4c151ff1 100644 --- a/backend/src/services/folder-commit/folder-commit-service.ts +++ b/backend/src/services/folder-commit/folder-commit-service.ts @@ -1386,7 +1386,7 @@ export const folderCommitServiceFactory = ({ ); // Invalidate cache to reflect the changes - await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId); + await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId, tx); return { secretChangesCount: secretChanges.length, diff --git a/backend/src/services/resource-cleanup/resource-cleanup-queue.ts b/backend/src/services/resource-cleanup/resource-cleanup-queue.ts index fbd9d3fd6..185ab5e94 100644 --- a/backend/src/services/resource-cleanup/resource-cleanup-queue.ts +++ b/backend/src/services/resource-cleanup/resource-cleanup-queue.ts @@ -1,5 +1,6 @@ import { TAuditLogDALFactory } from "@app/ee/services/audit-log/audit-log-dal"; import { TSnapshotDALFactory } from "@app/ee/services/secret-snapshot/snapshot-dal"; +import { TKeyValueStoreDALFactory } from "@app/keystore/key-value-store-dal"; import { getConfig } from "@app/lib/config/env"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; @@ -27,6 +28,7 @@ type TDailyResourceCleanUpQueueServiceFactoryDep = { queueService: TQueueServiceFactory; orgService: TOrgServiceFactory; userNotificationDAL: Pick; + keyValueStoreDAL: Pick; }; export type TDailyResourceCleanUpQueueServiceFactory = ReturnType; @@ -43,7 +45,8 @@ export const dailyResourceCleanUpQueueServiceFactory = ({ identityUniversalAuthClientSecretDAL, serviceTokenService, orgService, - userNotificationDAL + userNotificationDAL, + keyValueStoreDAL }: TDailyResourceCleanUpQueueServiceFactoryDep) => { const appCfg = getConfig(); @@ -52,6 +55,10 @@ export const dailyResourceCleanUpQueueServiceFactory = ({ } const init = async () => { + if (appCfg.isSecondaryInstance) { + return; + } + await queueService.stopRepeatableJob( QueueName.AuditLogPrune, QueueJobs.AuditLogPrune, @@ -82,6 +89,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({ await orgService.notifyInvitedUsers(); await auditLogDAL.pruneAuditLog(); await userNotificationDAL.pruneNotifications(); + await keyValueStoreDAL.pruneExpiredKeys(); logger.info(`${QueueName.DailyResourceCleanUp}: queue task completed`); } catch (error) { logger.error(error, `${QueueName.DailyResourceCleanUp}: resource cleanup failed`); diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts index 8d9c6958a..fb7101071 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts @@ -50,15 +50,14 @@ interface TSecretV2DalArg { } export const SECRET_DAL_TTL = () => applyJitter(10 * 60, 2 * 60); -export const SECRET_DAL_VERSION_TTL = 15 * 60; +export const SECRET_DAL_VERSION_TTL = "15m"; export const MAX_SECRET_CACHE_BYTES = 25 * 1024 * 1024; export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => { const secretOrm = ormify(db, TableName.SecretV2); - const invalidateSecretCacheByProjectId = async (projectId: string) => { + const invalidateSecretCacheByProjectId = async (projectId: string, tx?: Knex) => { const secretDalVersionKey = SecretServiceCacheKeys.getSecretDalVersion(projectId); - await keyStore.incrementBy(secretDalVersionKey, 1); - await keyStore.setExpiry(secretDalVersionKey, SECRET_DAL_VERSION_TTL); + await keyStore.pgIncrementBy(secretDalVersionKey, { incr: 1, tx, expiry: SECRET_DAL_VERSION_TTL }); }; const findOne = async (filter: Partial, tx?: Knex) => { diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index 48ab07816..9db535fbe 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -118,7 +118,7 @@ type TSecretV2BridgeServiceFactoryDep = { >; snapshotService: Pick; resourceMetadataDAL: Pick; - keyStore: Pick; + keyStore: Pick; reminderService: Pick; }; @@ -360,6 +360,7 @@ export const secretV2BridgeServiceFactory = ({ tx }); + await secretDAL.invalidateSecretCacheByProjectId(projectId, tx); return createdSecret; }); @@ -377,7 +378,6 @@ export const secretV2BridgeServiceFactory = ({ }); } - await secretDAL.invalidateSecretCacheByProjectId(projectId); if (inputSecret.type === SecretType.Shared) { await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ @@ -566,8 +566,8 @@ export const secretV2BridgeServiceFactory = ({ await $validateSecretReferences(projectId, permission, allSecretReferences); } - const updatedSecret = await secretDAL.transaction(async (tx) => - fnSecretBulkUpdate({ + const updatedSecret = await secretDAL.transaction(async (tx) => { + const modifiedSecretsInDB = await fnSecretBulkUpdate({ folderId, orgId: actorOrgId, resourceMetadataDAL, @@ -598,8 +598,11 @@ export const secretV2BridgeServiceFactory = ({ actorId }, tx - }) - ); + }); + + await secretDAL.invalidateSecretCacheByProjectId(projectId, tx); + return modifiedSecretsInDB; + }); if (inputSecret.secretReminderRepeatDays) { await reminderService.createReminder({ actor, @@ -615,7 +618,6 @@ export const secretV2BridgeServiceFactory = ({ }); } - await secretDAL.invalidateSecretCacheByProjectId(projectId); if (inputSecret.type === SecretType.Shared) { await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ @@ -715,8 +717,8 @@ export const secretV2BridgeServiceFactory = ({ ); try { - const deletedSecret = await secretDAL.transaction(async (tx) => - fnSecretBulkDelete({ + const deletedSecret = await secretDAL.transaction(async (tx) => { + const modifiedSecretsInDB = await fnSecretBulkDelete({ projectId, folderId, actorId, @@ -732,10 +734,11 @@ export const secretV2BridgeServiceFactory = ({ } ], tx - }) - ); + }); + await secretDAL.invalidateSecretCacheByProjectId(projectId, tx); + return modifiedSecretsInDB; + }); - await secretDAL.invalidateSecretCacheByProjectId(projectId); if (inputSecret.type === SecretType.Shared) { await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ @@ -1027,7 +1030,7 @@ export const secretV2BridgeServiceFactory = ({ }); throwIfMissingSecretReadValueOrDescribePermission(permission, ProjectPermissionSecretActions.DescribeSecret); - const cachedSecretDalVersion = await keyStore.getItem(SecretServiceCacheKeys.getSecretDalVersion(projectId)); + const cachedSecretDalVersion = await keyStore.pgGetIntItem(SecretServiceCacheKeys.getSecretDalVersion(projectId)); const secretDalVersion = Number(cachedSecretDalVersion || 0); const cacheKey = SecretServiceCacheKeys.getSecretsOfServiceLayer(projectId, secretDalVersion, { ...dto, @@ -1692,7 +1695,7 @@ export const secretV2BridgeServiceFactory = ({ await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, projectId }); const executeBulkInsert = async (tx: Knex) => { - return fnSecretBulkInsert({ + const modifiedSecretsInDB = await fnSecretBulkInsert({ inputSecrets: inputSecrets.map((el) => { const references = secretReferencesGroupByInputSecretKey[el.secretKey]?.nestedReferences; @@ -1728,13 +1731,14 @@ export const secretV2BridgeServiceFactory = ({ }, tx }); + await secretDAL.invalidateSecretCacheByProjectId(projectId, tx); + return modifiedSecretsInDB; }; const newSecrets = providedTx ? await executeBulkInsert(providedTx) : await secretDAL.transaction(executeBulkInsert); - await secretDAL.invalidateSecretCacheByProjectId(projectId); await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ actor, @@ -2099,6 +2103,7 @@ export const secretV2BridgeServiceFactory = ({ } } + await secretDAL.invalidateSecretCacheByProjectId(projectId, tx); return updatedSecrets; }; @@ -2106,7 +2111,6 @@ export const secretV2BridgeServiceFactory = ({ ? await executeBulkUpdate(providedTx) : await secretDAL.transaction(executeBulkUpdate); - await secretDAL.invalidateSecretCacheByProjectId(projectId); await Promise.allSettled(folders.map((el) => (el?.id ? snapshotService.performSnapshot(el.id) : undefined))); await Promise.allSettled( folders.map((el) => @@ -2233,7 +2237,7 @@ export const secretV2BridgeServiceFactory = ({ }); const executeBulkDelete = async (tx: Knex) => { - return fnSecretBulkDelete({ + const modifiedSecretsInDB = await fnSecretBulkDelete({ secretDAL, secretQueueService, folderCommitService, @@ -2249,6 +2253,8 @@ export const secretV2BridgeServiceFactory = ({ commitChanges, tx }); + await secretDAL.invalidateSecretCacheByProjectId(projectId, tx); + return modifiedSecretsInDB; }; try { @@ -2256,7 +2262,6 @@ export const secretV2BridgeServiceFactory = ({ ? await executeBulkDelete(providedTx) : await secretDAL.transaction(executeBulkDelete); - await secretDAL.invalidateSecretCacheByProjectId(projectId); await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ actor, From e8e51fb1e1211524bd5dd2ee449200f7c58feacf Mon Sep 17 00:00:00 2001 From: = Date: Tue, 9 Sep 2025 14:28:37 +0530 Subject: [PATCH 09/18] feat: added back missing replica node redirect for read db operations --- .../dynamic-secret-lease/dynamic-secret-lease-dal.ts | 7 +++++-- backend/src/ee/services/gateway/gateway-dal.ts | 2 +- .../ee/services/group/user-group-membership-dal.ts | 4 ++-- backend/src/ee/services/license/license-dal.ts | 4 ++-- .../secret-approval-request-dal.ts | 4 ++-- .../secret-approval-request-secret-dal.ts | 2 +- .../src/ee/services/secret-snapshot/snapshot-dal.ts | 12 ++++++++---- .../integration-auth/integration-auth-dal.ts | 2 +- backend/src/services/kms/kms-root-config-dal.ts | 2 +- backend/src/services/reminder/reminder-dal.ts | 8 ++++---- .../secret-folder/secret-folder-version-dal.ts | 2 +- .../src/services/secret-import/secret-import-dal.ts | 2 +- .../services/secret-sharing/secret-sharing-dal.ts | 2 +- .../secret-v2-bridge/secret-v2-bridge-dal.ts | 12 ++++++------ .../services/secret-v2-bridge/secret-version-dal.ts | 6 +++--- backend/src/services/super-admin/super-admin-dal.ts | 2 +- backend/src/services/user/user-dal.ts | 12 ++++++++---- 17 files changed, 48 insertions(+), 37 deletions(-) diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts index 525de9efd..974e20061 100644 --- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts +++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-dal.ts @@ -46,7 +46,10 @@ export const dynamicSecretLeaseDALFactory = (db: TDbClient) => { const countLeasesForDynamicSecret = async (dynamicSecretId: string, tx?: Knex) => { try { - const doc = await (tx || db)(TableName.DynamicSecretLease).count("*").where({ dynamicSecretId }).first(); + const doc = await (tx || db.replicaNode())(TableName.DynamicSecretLease) + .count("*") + .where({ dynamicSecretId }) + .first(); return parseInt(doc || "0", 10); } catch (error) { throw new DatabaseError({ error, name: "DynamicSecretCountLeases" }); @@ -55,7 +58,7 @@ export const dynamicSecretLeaseDALFactory = (db: TDbClient) => { const findById = async (id: string, tx?: Knex) => { try { - const doc = await (tx || db)(TableName.DynamicSecretLease) + const doc = await (tx || db.replicaNode())(TableName.DynamicSecretLease) .where({ [`${TableName.DynamicSecretLease}.id` as "id"]: id }) .first() .join( diff --git a/backend/src/ee/services/gateway/gateway-dal.ts b/backend/src/ee/services/gateway/gateway-dal.ts index 31b4b727b..c21ff31c0 100644 --- a/backend/src/ee/services/gateway/gateway-dal.ts +++ b/backend/src/ee/services/gateway/gateway-dal.ts @@ -13,7 +13,7 @@ export const gatewayDALFactory = (db: TDbClient) => { { offset, limit, sort, tx }: TFindOpt = {} ) => { try { - const query = (tx || db)(TableName.Gateway) + const query = (tx || db.replicaNode())(TableName.Gateway) // eslint-disable-next-line @typescript-eslint/no-misused-promises .where(buildFindFilter(filter, TableName.Gateway, ["orgId"])) .join(TableName.Identity, `${TableName.Identity}.id`, `${TableName.Gateway}.identityId`) diff --git a/backend/src/ee/services/group/user-group-membership-dal.ts b/backend/src/ee/services/group/user-group-membership-dal.ts index 5ee97e457..374459b0c 100644 --- a/backend/src/ee/services/group/user-group-membership-dal.ts +++ b/backend/src/ee/services/group/user-group-membership-dal.ts @@ -23,7 +23,7 @@ export const userGroupMembershipDALFactory = (db: TDbClient) => { .whereIn(`${TableName.ProjectMembership}.projectId`, projectIds) .pluck(`${TableName.ProjectMembership}.projectId`); - const userGroupMemberships: string[] = await (tx || db)(TableName.UserGroupMembership) + const userGroupMemberships: string[] = await (tx || db.replicaNode())(TableName.UserGroupMembership) .where(`${TableName.UserGroupMembership}.userId`, userId) .whereNot(`${TableName.UserGroupMembership}.groupId`, groupId) .join( @@ -79,7 +79,7 @@ export const userGroupMembershipDALFactory = (db: TDbClient) => { .pluck(`${TableName.GroupProjectMembership}.groupId`); // main query - const members = await (tx || db)(TableName.UserGroupMembership) + const members = await (tx || db.replicaNode())(TableName.UserGroupMembership) .where(`${TableName.UserGroupMembership}.groupId`, groupId) .where(`${TableName.UserGroupMembership}.isPending`, false) .join(TableName.Users, `${TableName.UserGroupMembership}.userId`, `${TableName.Users}.id`) diff --git a/backend/src/ee/services/license/license-dal.ts b/backend/src/ee/services/license/license-dal.ts index 88a2dadf6..cfea2573d 100644 --- a/backend/src/ee/services/license/license-dal.ts +++ b/backend/src/ee/services/license/license-dal.ts @@ -28,7 +28,7 @@ export const licenseDALFactory = (db: TDbClient) => { const countOrgUsersAndIdentities = async (orgId: string | null, tx?: Knex) => { try { // count org users - const userDoc = await (tx || db)(TableName.OrgMembership) + const userDoc = await (tx || db.replicaNode())(TableName.OrgMembership) .where({ status: OrgMembershipStatus.Accepted }) .andWhere((bd) => { if (orgId) { @@ -42,7 +42,7 @@ export const licenseDALFactory = (db: TDbClient) => { const userCount = Number(userDoc?.[0].count); // count org identities - const identityDoc = await (tx || db)(TableName.IdentityOrgMembership) + const identityDoc = await (tx || db.replicaNode())(TableName.IdentityOrgMembership) .where((bd) => { if (orgId) { void bd.where({ orgId }); diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts index fe4ca94e1..01caef223 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-dal.ts @@ -345,7 +345,7 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => { const findProjectRequestCount = async (projectId: string, userId: string, policyId?: string, tx?: Knex) => { try { - const docs = await (tx || db) + const docs = await (tx || db.replicaNode()) .with( "temp", (tx || db.replicaNode())(TableName.SecretApprovalRequest) @@ -494,7 +494,7 @@ export const secretApprovalRequestDALFactory = (db: TDbClient) => { .distinctOn(`${TableName.SecretApprovalRequest}.id`) .as("inner"); - const query = (tx || db) + const query = (tx || db.replicaNode()) .select("*") .select(db.raw("count(*) OVER() as total_count")) .from(innerQuery) diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts index c1b18e43d..17182cddf 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts @@ -377,7 +377,7 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => { // special query for migration to v2 secret const findByProjectId = async (projectId: string, tx?: Knex) => { try { - const docs = await (tx || db)(TableName.SecretApprovalRequestSecret) + const docs = await (tx || db.replicaNode())(TableName.SecretApprovalRequestSecret) .join( TableName.SecretApprovalRequest, `${TableName.SecretApprovalRequest}.id`, diff --git a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts index c547d85c2..17f1fad05 100644 --- a/backend/src/ee/services/secret-snapshot/snapshot-dal.ts +++ b/backend/src/ee/services/secret-snapshot/snapshot-dal.ts @@ -265,7 +265,7 @@ export const snapshotDALFactory = (db: TDbClient) => { // then joins with respective secrets and folder const findRecursivelySnapshots = async (snapshotId: string, tx?: Knex) => { try { - const data = await (tx || db) + const data = await (tx || db.replicaNode()) .withRecursive("parent", (qb) => { void qb .from(TableName.Snapshot) @@ -419,7 +419,7 @@ export const snapshotDALFactory = (db: TDbClient) => { // then joins with respective secrets and folder const findRecursivelySnapshotsV2Bridge = async (snapshotId: string, tx?: Knex) => { try { - const data = await (tx || db) + const data = await (tx || db.replicaNode()) .withRecursive("parent", (qb) => { void qb .from(TableName.Snapshot) @@ -581,7 +581,11 @@ export const snapshotDALFactory = (db: TDbClient) => { const docs = await (tx || db.replicaNode())(TableName.Snapshot) .where(`${TableName.Snapshot}.folderId`, folderId) .join( - (tx || db)(TableName.Snapshot).groupBy("folderId").max("createdAt").select("folderId").as("latestVersion"), + (tx || db.replicaNode())(TableName.Snapshot) + .groupBy("folderId") + .max("createdAt") + .select("folderId") + .as("latestVersion"), (bd) => { bd.on(`${TableName.Snapshot}.folderId`, "latestVersion.folderId").andOn( `${TableName.Snapshot}.createdAt`, @@ -766,7 +770,7 @@ export const snapshotDALFactory = (db: TDbClient) => { ) .orderBy(`${TableName.Snapshot}.createdAt`, "desc") .where(`${TableName.Snapshot}.folderId`, folderId); - const data = await (tx || db) + const data = await (tx || db.replicaNode()) .with("w", query) .select("*") .from[number]>("w") diff --git a/backend/src/services/integration-auth/integration-auth-dal.ts b/backend/src/services/integration-auth/integration-auth-dal.ts index 7a56afcbb..d3ccf610b 100644 --- a/backend/src/services/integration-auth/integration-auth-dal.ts +++ b/backend/src/services/integration-auth/integration-auth-dal.ts @@ -30,7 +30,7 @@ export const integrationAuthDALFactory = (db: TDbClient) => { const getByOrg = async (orgId: string, tx?: Knex) => { try { - const integrationAuths = await (tx || db)(TableName.IntegrationAuth) + const integrationAuths = await (tx || db.replicaNode())(TableName.IntegrationAuth) .join(TableName.Project, `${TableName.Project}.id`, `${TableName.IntegrationAuth}.projectId`) .join(TableName.Organization, `${TableName.Organization}.id`, `${TableName.Project}.orgId`) .where(`${TableName.Organization}.id`, "=", orgId) diff --git a/backend/src/services/kms/kms-root-config-dal.ts b/backend/src/services/kms/kms-root-config-dal.ts index 31826b79d..4de3fc15a 100644 --- a/backend/src/services/kms/kms-root-config-dal.ts +++ b/backend/src/services/kms/kms-root-config-dal.ts @@ -12,7 +12,7 @@ export const kmsRootConfigDALFactory = (db: TDbClient) => { const findById = async (id: string, tx?: Knex) => { try { - const result = await (tx || db)(TableName.KmsServerRootConfig) + const result = await (tx || db.replicaNode())(TableName.KmsServerRootConfig) .where({ id } as never) .first("*"); return result; diff --git a/backend/src/services/reminder/reminder-dal.ts b/backend/src/services/reminder/reminder-dal.ts index 897a75234..4161552a9 100644 --- a/backend/src/services/reminder/reminder-dal.ts +++ b/backend/src/services/reminder/reminder-dal.ts @@ -39,7 +39,7 @@ export const reminderDALFactory = (db: TDbClient) => { const findSecretDailyReminders = async (tx?: Knex) => { const { startOfDay, endOfDay } = getTodayDateRange(); - const rawReminders = await (tx || db)(TableName.Reminder) + const rawReminders = await (tx || db.replicaNode())(TableName.Reminder) .whereBetween("nextReminderDate", [startOfDay, endOfDay]) .leftJoin(TableName.ReminderRecipient, `${TableName.Reminder}.id`, `${TableName.ReminderRecipient}.reminderId`) .leftJoin(TableName.Users, `${TableName.ReminderRecipient}.userId`, `${TableName.Users}.id`) @@ -90,7 +90,7 @@ export const reminderDALFactory = (db: TDbClient) => { const futureDate = new Date(startOfDay); futureDate.setDate(futureDate.getDate() + daysAhead); - const reminders = await (tx || db)(TableName.Reminder) + const reminders = await (tx || db.replicaNode())(TableName.Reminder) .where("nextReminderDate", ">=", startOfDay) .where("nextReminderDate", "<=", futureDate) .orderBy("nextReminderDate", "asc") @@ -101,7 +101,7 @@ export const reminderDALFactory = (db: TDbClient) => { }; const findSecretReminder = async (secretId: string, tx?: Knex) => { - const rawReminders = await (tx || db)(TableName.Reminder) + const rawReminders = await (tx || db.replicaNode())(TableName.Reminder) .where(`${TableName.Reminder}.secretId`, secretId) .leftJoin(TableName.ReminderRecipient, `${TableName.Reminder}.id`, `${TableName.ReminderRecipient}.reminderId`) .select(selectAllTableCols(TableName.Reminder)) @@ -125,7 +125,7 @@ export const reminderDALFactory = (db: TDbClient) => { }; const findSecretReminders = async (secretIds: string[], tx?: Knex) => { - const rawReminders = await (tx || db)(TableName.Reminder) + const rawReminders = await (tx || db.replicaNode())(TableName.Reminder) .whereIn(`${TableName.Reminder}.secretId`, secretIds) .leftJoin(TableName.ReminderRecipient, `${TableName.Reminder}.id`, `${TableName.ReminderRecipient}.reminderId`) .select(selectAllTableCols(TableName.Reminder)) diff --git a/backend/src/services/secret-folder/secret-folder-version-dal.ts b/backend/src/services/secret-folder/secret-folder-version-dal.ts index 46ff49692..5504c6e0b 100644 --- a/backend/src/services/secret-folder/secret-folder-version-dal.ts +++ b/backend/src/services/secret-folder/secret-folder-version-dal.ts @@ -45,7 +45,7 @@ export const secretFolderVersionDALFactory = (db: TDbClient) => { ) .whereIn(`${TableName.SecretFolderVersion}.folderId`, folderIds) .join( - (tx || db)(TableName.SecretFolderVersion) + (tx || db.replicaNode())(TableName.SecretFolderVersion) .groupBy("folderId") .max("version") .select("folderId") diff --git a/backend/src/services/secret-import/secret-import-dal.ts b/backend/src/services/secret-import/secret-import-dal.ts index db611dc6c..2261c2418 100644 --- a/backend/src/services/secret-import/secret-import-dal.ts +++ b/backend/src/services/secret-import/secret-import-dal.ts @@ -15,7 +15,7 @@ export const secretImportDALFactory = (db: TDbClient) => { // we are using postion based sorting as its a small list // this will return the last value of the position in a folder with secret imports const findLastImportPosition = async (folderId: string, tx?: Knex) => { - const lastPos = await (tx || db)(TableName.SecretImport) + const lastPos = await (tx || db.replicaNode())(TableName.SecretImport) .where({ folderId }) .max("position", { as: "position" }) .first(); diff --git a/backend/src/services/secret-sharing/secret-sharing-dal.ts b/backend/src/services/secret-sharing/secret-sharing-dal.ts index 7cdccd4f8..08e0ad257 100644 --- a/backend/src/services/secret-sharing/secret-sharing-dal.ts +++ b/backend/src/services/secret-sharing/secret-sharing-dal.ts @@ -119,7 +119,7 @@ export const secretSharingDALFactory = (db: TDbClient) => { const findActiveSharedSecrets = async (filters: Partial, tx?: Knex) => { try { const now = new Date(); - return await (tx || db)(TableName.SecretSharing) + return await (tx || db.replicaNode())(TableName.SecretSharing) .where(filters) .andWhere("expiresAt", ">", now) .andWhere("encryptedValue", "<>", "") diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts index fb7101071..93afb3b55 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts @@ -62,7 +62,7 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => { const findOne = async (filter: Partial, tx?: Knex) => { try { - const docs = await (tx || db)(TableName.SecretV2) + const docs = await (tx || db.replicaNode())(TableName.SecretV2) // eslint-disable-next-line @typescript-eslint/no-misused-promises .where(buildFindFilter(filter, TableName.SecretV2)) .leftJoin( @@ -143,7 +143,7 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => { const find = async (filter: TFindFilter, opts: TFindOpt = {}) => { const { offset, limit, sort, tx } = opts; try { - const query = (tx || db)(TableName.SecretV2) + const query = (tx || db.replicaNode())(TableName.SecretV2) // eslint-disable-next-line @typescript-eslint/no-misused-promises .where(buildFindFilter(filter)) .leftJoin( @@ -887,13 +887,13 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => { const findSecretsWithReminderRecipients = async (ids: string[], limit: number, tx?: Knex) => { try { // Create a subquery to get limited secret IDs - const limitedSecretIds = (tx || db)(TableName.SecretV2) + const limitedSecretIds = (tx || db.replicaNode())(TableName.SecretV2) .whereIn(`${TableName.SecretV2}.id`, ids) .limit(limit) .select("id"); // Join with all recipients for the limited secrets - const docs = await (tx || db)(TableName.SecretV2) + const docs = await (tx || db.replicaNode())(TableName.SecretV2) .whereIn(`${TableName.SecretV2}.id`, limitedSecretIds) .leftJoin(TableName.Reminder, `${TableName.SecretV2}.id`, `${TableName.Reminder}.secretId`) .leftJoin(TableName.ReminderRecipient, `${TableName.Reminder}.id`, `${TableName.ReminderRecipient}.reminderId`) @@ -925,13 +925,13 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => { const findSecretsWithReminderRecipientsOld = async (ids: string[], limit: number, tx?: Knex) => { try { // Create a subquery to get limited secret IDs - const limitedSecretIds = (tx || db)(TableName.SecretV2) + const limitedSecretIds = (tx || db.replicaNode())(TableName.SecretV2) .whereIn(`${TableName.SecretV2}.id`, ids) .limit(limit) .select("id"); // Join with all recipients for the limited secrets - const docs = await (tx || db)(TableName.SecretV2) + const docs = await (tx || db.replicaNode())(TableName.SecretV2) .whereIn(`${TableName.SecretV2}.id`, limitedSecretIds) .leftJoin(TableName.Reminder, `${TableName.SecretV2}.id`, `${TableName.Reminder}.secretId`) .leftJoin( diff --git a/backend/src/services/secret-v2-bridge/secret-version-dal.ts b/backend/src/services/secret-v2-bridge/secret-version-dal.ts index 9537b79e3..0282fa537 100644 --- a/backend/src/services/secret-v2-bridge/secret-version-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-version-dal.ts @@ -72,7 +72,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { .where(`${TableName.SecretVersionV2}.folderId`, folderId) .join(TableName.SecretV2, `${TableName.SecretV2}.id`, `${TableName.SecretVersionV2}.secretId`) .join( - (tx || db)(TableName.SecretVersionV2) + (tx || db.replicaNode())(TableName.SecretVersionV2) .where(`${TableName.SecretVersionV2}.folderId`, folderId) .groupBy("secretId") .max("version") @@ -121,7 +121,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { .where("folderId", folderId) .whereIn(`${TableName.SecretVersionV2}.secretId`, secretIds) .join( - (tx || db)(TableName.SecretVersionV2) + (tx || db.replicaNode())(TableName.SecretVersionV2) .groupBy("secretId") .max("version") .select("secretId") @@ -189,7 +189,7 @@ export const secretVersionV2BridgeDALFactory = (db: TDbClient) => { }) => { try { const { offset, limit, sort = [["createdAt", "desc"]] } = findOpt; - const query = (tx || db)(TableName.SecretVersionV2) + const query = (tx || db.replicaNode())(TableName.SecretVersionV2) .leftJoin(TableName.Users, `${TableName.Users}.id`, `${TableName.SecretVersionV2}.userActorId`) .leftJoin( TableName.ProjectMembership, diff --git a/backend/src/services/super-admin/super-admin-dal.ts b/backend/src/services/super-admin/super-admin-dal.ts index d7d11a5d2..571583cc3 100644 --- a/backend/src/services/super-admin/super-admin-dal.ts +++ b/backend/src/services/super-admin/super-admin-dal.ts @@ -11,7 +11,7 @@ export const superAdminDALFactory = (db: TDbClient) => { const superAdminOrm = ormify(db, TableName.SuperAdmin); const findById = async (id: string, tx?: Knex) => { - const config = await (tx || db)(TableName.SuperAdmin) + const config = await (tx || db.replicaNode())(TableName.SuperAdmin) .where(`${TableName.SuperAdmin}.id`, id) .leftJoin(TableName.Organization, `${TableName.SuperAdmin}.defaultAuthOrgId`, `${TableName.Organization}.id`) .leftJoin(TableName.SamlConfig, (qb) => { diff --git a/backend/src/services/user/user-dal.ts b/backend/src/services/user/user-dal.ts index 0f623dff1..4267d13ee 100644 --- a/backend/src/services/user/user-dal.ts +++ b/backend/src/services/user/user-dal.ts @@ -19,12 +19,16 @@ export type TUserDALFactory = ReturnType; export const userDALFactory = (db: TDbClient) => { const userOrm = ormify(db, TableName.Users); const findUserByUsername = async (username: string, tx?: Knex) => - (tx || db)(TableName.Users).whereRaw('lower("username") = :username', { username: username.toLowerCase() }); + (tx || db.replicaNode())(TableName.Users).whereRaw('lower("username") = :username', { + username: username.toLowerCase() + }); const findUserByEmail = async (email: string, tx?: Knex) => - (tx || db)(TableName.Users).whereRaw('lower("email") = :email', { email: email.toLowerCase() }).where({ - isEmailVerified: true - }); + (tx || db.replicaNode())(TableName.Users) + .whereRaw('lower("email") = :email', { email: email.toLowerCase() }) + .where({ + isEmailVerified: true + }); const getUsersByFilter = async ({ limit, From ae42f31d694c43ff1b3f57bf0b419428d412d1f1 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 9 Sep 2025 21:50:14 +0530 Subject: [PATCH 10/18] feat: added replica node for auth method fetch --- api_requests.sh | 89 +++++++++++++++++++ backend/src/services/identity/identity-dal.ts | 2 +- 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100755 api_requests.sh diff --git a/api_requests.sh b/api_requests.sh new file mode 100755 index 000000000..5d21f114f --- /dev/null +++ b/api_requests.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# Configuration +BEARER_TOKEN="your-token-here" +BASE_URL="https://api.example.com" + +# Colors for output +GREEN='\033[0;32m' +BLUE='\033[0;34m' +RED='\033[0;31m' +NC='\033[0m' # No Color + +echo "Starting API requests with timing metrics..." +echo "==========================================" + +# Request 1: POST with body +echo -e "\n${BLUE}1. POST Request${NC}" +echo "URL: $BASE_URL/endpoint1" +start_time=$(date +%s.%3N) +response1=$(curl -s -w "%{http_code}|%{time_total}" \ + -X POST \ + -H "Authorization: Bearer $BEARER_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "key1": "value1", + "key2": "value2" + }' \ + "$BASE_URL/endpoint1") +end_time=$(date +%s.%3N) + +http_code1=$(echo "$response1" | tail -c 10 | cut -d'|' -f1) +curl_time1=$(echo "$response1" | tail -c 10 | cut -d'|' -f2) +duration1=$(echo "$end_time - $start_time" | bc) + +echo "Status Code: $http_code1" +echo "Duration: ${duration1}s (curl time: ${curl_time1}s)" +echo -e "${GREEN}✓ POST request completed${NC}" + +# Request 2: GET with query params +echo -e "\n${BLUE}2. GET Request with Query Parameters${NC}" +echo "URL: $BASE_URL/endpoint2?param1=value1¶m2=value2" +start_time=$(date +%s.%3N) +response2=$(curl -s -w "%{http_code}|%{time_total}" \ + -X GET \ + -H "Authorization: Bearer $BEARER_TOKEN" \ + "$BASE_URL/endpoint2?param1=value1¶m2=value2") +end_time=$(date +%s.%3N) + +http_code2=$(echo "$response2" | tail -c 10 | cut -d'|' -f1) +curl_time2=$(echo "$response2" | tail -c 10 | cut -d'|' -f2) +duration2=$(echo "$end_time - $start_time" | bc) + +echo "Status Code: $http_code2" +echo "Duration: ${duration2}s (curl time: ${curl_time2}s)" +echo -e "${GREEN}✓ GET request completed${NC}" + +# Request 3: DELETE with body +echo -e "\n${BLUE}3. DELETE Request with Body${NC}" +echo "URL: $BASE_URL/endpoint3" +start_time=$(date +%s.%3N) +response3=$(curl -s -w "%{http_code}|%{time_total}" \ + -X DELETE \ + -H "Authorization: Bearer $BEARER_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "id": "123", + "reason": "cleanup" + }' \ + "$BASE_URL/endpoint3") +end_time=$(date +%s.%3N) + +http_code3=$(echo "$response3" | tail -c 10 | cut -d'|' -f1) +curl_time3=$(echo "$response3" | tail -c 10 | cut -d'|' -f2) +duration3=$(echo "$end_time - $start_time" | bc) + +echo "Status Code: $http_code3" +echo "Duration: ${duration3}s (curl time: ${curl_time3}s)" +echo -e "${GREEN}✓ DELETE request completed${NC}" + +# Summary +echo -e "\n==========================================" +echo -e "${BLUE}SUMMARY${NC}" +echo "==========================================" +echo "Request 1 (POST): ${duration1}s" +echo "Request 2 (GET): ${duration2}s" +echo "Request 3 (DELETE): ${duration3}s" +total_time=$(echo "$duration1 + $duration2 + $duration3" | bc) +echo "Total Time: ${total_time}s" +echo -e "${GREEN}All requests completed successfully!${NC}" \ No newline at end of file diff --git a/backend/src/services/identity/identity-dal.ts b/backend/src/services/identity/identity-dal.ts index 363412493..7bc797600 100644 --- a/backend/src/services/identity/identity-dal.ts +++ b/backend/src/services/identity/identity-dal.ts @@ -25,7 +25,7 @@ export const identityDALFactory = (db: TDbClient) => { } as const; const tableName = authMethodToTableName[authMethod]; if (!tableName) return; - const data = await db(tableName).where({ identityId }).first(); + const data = await db.replicaNode()(tableName).where({ identityId }).first(); if (!data) return; return data.accessTokenTrustedIps; }; From c682d87d959b4221e653a9a6f14afb028b4b1af0 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 9 Sep 2025 22:55:32 +0530 Subject: [PATCH 11/18] feat: reptile review fixed --- .../secret-approval-request-service.ts | 1 + backend/src/keystore/key-value-store-dal.ts | 6 +++--- backend/src/lib/knex/index.ts | 2 +- .../services/folder-commit/folder-commit-service.test.ts | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index aa3e3dc0f..17b7d8347 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -787,6 +787,7 @@ export const secretApprovalRequestServiceFactory = ({ }, tx ); + await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId, tx); return { secrets: { created: newSecrets, updated: updatedSecrets, deleted: deletedSecret }, approval: updatedSecretApproval diff --git a/backend/src/keystore/key-value-store-dal.ts b/backend/src/keystore/key-value-store-dal.ts index bdf5e8725..bccedf4ac 100644 --- a/backend/src/keystore/key-value-store-dal.ts +++ b/backend/src/keystore/key-value-store-dal.ts @@ -19,7 +19,7 @@ const MAX_RETRY_ON_FAILURE = 3; export const keyValueStoreDALFactory = (db: TDbClient): TKeyValueStoreDALFactory => { const keyValueStoreOrm = ormify(db, TableName.KeyValueStore); - const incrementBy: TKeyValueStoreDALFactory["incrementBy"] = (key, { incr = 1, tx, expiresAt }) => { + const incrementBy: TKeyValueStoreDALFactory["incrementBy"] = async (key, { incr = 1, tx, expiresAt }) => { return (tx || db)(TableName.KeyValueStore) .insert({ key, integerValue: 1, expiresAt }) .onConflict("key") @@ -27,7 +27,8 @@ export const keyValueStoreDALFactory = (db: TDbClient): TKeyValueStoreDALFactory integerValue: db.raw(`"${TableName.KeyValueStore}"."integerValue" + ?`, [incr]), expiresAt }) - .returning("integerValue"); + .returning("integerValue") + .then((result) => Number(result[0]?.integerValue || 0)); }; const findOneInt: TKeyValueStoreDALFactory["findOneInt"] = async (key, tx) => { @@ -44,7 +45,6 @@ export const keyValueStoreDALFactory = (db: TDbClient): TKeyValueStoreDALFactory return Number(doc?.integerValue || 0); }; - // delete all audit log that have expired const pruneExpiredKeys: TKeyValueStoreDALFactory["pruneExpiredKeys"] = async () => { let deletedIds: { key: string }[] = []; let numberOfRetryOnFailure = 0; diff --git a/backend/src/lib/knex/index.ts b/backend/src/lib/knex/index.ts index dbfd29b81..499e7cb26 100644 --- a/backend/src/lib/knex/index.ts +++ b/backend/src/lib/knex/index.ts @@ -278,7 +278,7 @@ export const ormify = ( } if ($decr) { Object.entries($decr).forEach(([incrementField, incrementValue]) => { - void query.increment(incrementField, incrementValue as number); + void query.decrement(incrementField, incrementValue as number); }); } return (await query) as Tables[Tname]["base"][]; diff --git a/backend/src/services/folder-commit/folder-commit-service.test.ts b/backend/src/services/folder-commit/folder-commit-service.test.ts index 28d603829..0a73d6e0c 100644 --- a/backend/src/services/folder-commit/folder-commit-service.test.ts +++ b/backend/src/services/folder-commit/folder-commit-service.test.ts @@ -661,7 +661,7 @@ describe("folderCommitServiceFactory", () => { // Assert expect(mockFolderCommitDAL.create).toHaveBeenCalled(); - expect(mockSecretV2BridgeDAL.invalidateSecretCacheByProjectId).toHaveBeenCalledWith(projectId); + expect(mockSecretV2BridgeDAL.invalidateSecretCacheByProjectId).toHaveBeenCalledWith(projectId, {}); // Check that we got the right counts expect(result.totalChanges).toEqual(2); From 6f1c884bf5f3be08f2918cb0129f84fd828b98a9 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 10 Sep 2025 16:11:20 +0530 Subject: [PATCH 12/18] feat: review comments --- api_requests.sh | 89 ------------------- .../20250908193226_sql-cache_int.ts | 1 + backend/src/db/schemas/key-value-store.ts | 4 +- 3 files changed, 4 insertions(+), 90 deletions(-) delete mode 100755 api_requests.sh diff --git a/api_requests.sh b/api_requests.sh deleted file mode 100755 index 5d21f114f..000000000 --- a/api_requests.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash - -# Configuration -BEARER_TOKEN="your-token-here" -BASE_URL="https://api.example.com" - -# Colors for output -GREEN='\033[0;32m' -BLUE='\033[0;34m' -RED='\033[0;31m' -NC='\033[0m' # No Color - -echo "Starting API requests with timing metrics..." -echo "==========================================" - -# Request 1: POST with body -echo -e "\n${BLUE}1. POST Request${NC}" -echo "URL: $BASE_URL/endpoint1" -start_time=$(date +%s.%3N) -response1=$(curl -s -w "%{http_code}|%{time_total}" \ - -X POST \ - -H "Authorization: Bearer $BEARER_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "key1": "value1", - "key2": "value2" - }' \ - "$BASE_URL/endpoint1") -end_time=$(date +%s.%3N) - -http_code1=$(echo "$response1" | tail -c 10 | cut -d'|' -f1) -curl_time1=$(echo "$response1" | tail -c 10 | cut -d'|' -f2) -duration1=$(echo "$end_time - $start_time" | bc) - -echo "Status Code: $http_code1" -echo "Duration: ${duration1}s (curl time: ${curl_time1}s)" -echo -e "${GREEN}✓ POST request completed${NC}" - -# Request 2: GET with query params -echo -e "\n${BLUE}2. GET Request with Query Parameters${NC}" -echo "URL: $BASE_URL/endpoint2?param1=value1¶m2=value2" -start_time=$(date +%s.%3N) -response2=$(curl -s -w "%{http_code}|%{time_total}" \ - -X GET \ - -H "Authorization: Bearer $BEARER_TOKEN" \ - "$BASE_URL/endpoint2?param1=value1¶m2=value2") -end_time=$(date +%s.%3N) - -http_code2=$(echo "$response2" | tail -c 10 | cut -d'|' -f1) -curl_time2=$(echo "$response2" | tail -c 10 | cut -d'|' -f2) -duration2=$(echo "$end_time - $start_time" | bc) - -echo "Status Code: $http_code2" -echo "Duration: ${duration2}s (curl time: ${curl_time2}s)" -echo -e "${GREEN}✓ GET request completed${NC}" - -# Request 3: DELETE with body -echo -e "\n${BLUE}3. DELETE Request with Body${NC}" -echo "URL: $BASE_URL/endpoint3" -start_time=$(date +%s.%3N) -response3=$(curl -s -w "%{http_code}|%{time_total}" \ - -X DELETE \ - -H "Authorization: Bearer $BEARER_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "id": "123", - "reason": "cleanup" - }' \ - "$BASE_URL/endpoint3") -end_time=$(date +%s.%3N) - -http_code3=$(echo "$response3" | tail -c 10 | cut -d'|' -f1) -curl_time3=$(echo "$response3" | tail -c 10 | cut -d'|' -f2) -duration3=$(echo "$end_time - $start_time" | bc) - -echo "Status Code: $http_code3" -echo "Duration: ${duration3}s (curl time: ${curl_time3}s)" -echo -e "${GREEN}✓ DELETE request completed${NC}" - -# Summary -echo -e "\n==========================================" -echo -e "${BLUE}SUMMARY${NC}" -echo "==========================================" -echo "Request 1 (POST): ${duration1}s" -echo "Request 2 (GET): ${duration2}s" -echo "Request 3 (DELETE): ${duration3}s" -total_time=$(echo "$duration1 + $duration2 + $duration3" | bc) -echo "Total Time: ${total_time}s" -echo -e "${GREEN}All requests completed successfully!${NC}" \ No newline at end of file diff --git a/backend/src/db/migrations/20250908193226_sql-cache_int.ts b/backend/src/db/migrations/20250908193226_sql-cache_int.ts index dc4060d02..0e15c10d1 100644 --- a/backend/src/db/migrations/20250908193226_sql-cache_int.ts +++ b/backend/src/db/migrations/20250908193226_sql-cache_int.ts @@ -8,6 +8,7 @@ export async function up(knex: Knex): Promise { t.text("key").primary(); t.bigint("integerValue"); t.datetime("expiresAt"); + t.timestamps(true, true, true); }); } } diff --git a/backend/src/db/schemas/key-value-store.ts b/backend/src/db/schemas/key-value-store.ts index d6ba90b2f..448c78f24 100644 --- a/backend/src/db/schemas/key-value-store.ts +++ b/backend/src/db/schemas/key-value-store.ts @@ -10,7 +10,9 @@ import { TImmutableDBKeys } from "./models"; export const KeyValueStoreSchema = z.object({ key: z.string(), integerValue: z.coerce.number().nullable().optional(), - expiresAt: z.date().nullable().optional() + expiresAt: z.date().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date() }); export type TKeyValueStore = z.infer; From a0595fa9b4ba2265049566d76a6330545cce3858 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 10 Sep 2025 19:59:29 +0800 Subject: [PATCH 13/18] fix: address sql server dynamic secret integration --- .../dynamic-secret/providers/models.ts | 1 + .../dynamic-secret/providers/sql-database.ts | 37 ++++++++++++++++--- .../SqlDatabaseInputForm.tsx | 26 ++++++++++++- .../EditDynamicSecretSqlProviderForm.tsx | 26 ++++++++++++- 4 files changed, 83 insertions(+), 7 deletions(-) diff --git a/backend/src/ee/services/dynamic-secret/providers/models.ts b/backend/src/ee/services/dynamic-secret/providers/models.ts index ae1bcfc25..c618a308b 100644 --- a/backend/src/ee/services/dynamic-secret/providers/models.ts +++ b/backend/src/ee/services/dynamic-secret/providers/models.ts @@ -165,6 +165,7 @@ export const DynamicSecretSqlDBSchema = z.object({ revocationStatement: z.string().trim(), renewStatement: z.string().trim().optional(), ca: z.string().optional(), + sslEnabled: z.boolean().optional(), gatewayId: z.string().nullable().optional() }); diff --git a/backend/src/ee/services/dynamic-secret/providers/sql-database.ts b/backend/src/ee/services/dynamic-secret/providers/sql-database.ts index 57ce710b1..babbe1912 100644 --- a/backend/src/ee/services/dynamic-secret/providers/sql-database.ts +++ b/backend/src/ee/services/dynamic-secret/providers/sql-database.ts @@ -1,4 +1,5 @@ import handlebars from "handlebars"; +import RE2 from "re2"; import knex from "knex"; import { z } from "zod"; @@ -154,7 +155,15 @@ export const SqlDatabaseProvider = ({ gatewayService }: TSqlDatabaseProviderDTO) const ssl = providerInputs.ca ? { rejectUnauthorized: false, ca: providerInputs.ca, servername: providerInputs.host } : undefined; + const isMsSQLClient = providerInputs.client === SqlProviders.MsSQL; + const isAzureSql = isMsSQLClient && new RE2(/\.database\.windows\.net$/i).test(providerInputs.host); + const azureServerLabel = isAzureSql ? providerInputs.host.split(".")[0] : undefined; + + const effectiveUser = + isAzureSql && !providerInputs.username.includes("@") + ? `${providerInputs.username}@${azureServerLabel}` + : providerInputs.username; const db = knex({ client: providerInputs.client, @@ -162,7 +171,7 @@ export const SqlDatabaseProvider = ({ gatewayService }: TSqlDatabaseProviderDTO) database: providerInputs.database, port: providerInputs.port, host: providerInputs.client === SqlProviders.Postgres ? providerInputs.hostIp : providerInputs.host, - user: providerInputs.username, + user: effectiveUser, password: providerInputs.password, ssl, // @ts-expect-error this is because of knexjs type signature issue. This is directly passed to driver @@ -170,6 +179,7 @@ export const SqlDatabaseProvider = ({ gatewayService }: TSqlDatabaseProviderDTO) // https://github.com/tediousjs/tedious/blob/ebb023ed90969a7ec0e4b036533ad52739d921f7/test/config.ci.ts#L19 options: isMsSQLClient ? { + ...(providerInputs.sslEnabled !== undefined ? { encrypt: providerInputs.sslEnabled } : {}), trustServerCertificate: !providerInputs.ca, cryptoCredentialsDetails: providerInputs.ca ? { ca: providerInputs.ca } : {} } @@ -212,7 +222,12 @@ export const SqlDatabaseProvider = ({ gatewayService }: TSqlDatabaseProviderDTO) const providerInputs = await validateProviderInputs(inputs); let isConnected = false; const gatewayCallback = async (host = providerInputs.host, port = providerInputs.port) => { - const db = await $getClient({ ...providerInputs, port, host, hostIp: providerInputs.hostIp }); + const db = await $getClient({ + ...providerInputs, + port, + host, + hostIp: providerInputs.hostIp + }); // oracle needs from keyword const testStatement = providerInputs.client === SqlProviders.Oracle ? "SELECT 1 FROM DUAL" : "SELECT 1"; @@ -253,7 +268,11 @@ export const SqlDatabaseProvider = ({ gatewayService }: TSqlDatabaseProviderDTO) const password = generatePassword(providerInputs.client, providerInputs.passwordRequirements); const gatewayCallback = async (host = providerInputs.host, port = providerInputs.port) => { - const db = await $getClient({ ...providerInputs, port, host }); + const db = await $getClient({ + ...providerInputs, + port, + host + }); try { const expiration = new Date(expireAt).toISOString(); @@ -296,7 +315,11 @@ export const SqlDatabaseProvider = ({ gatewayService }: TSqlDatabaseProviderDTO) const username = entityId; const { database } = providerInputs; const gatewayCallback = async (host = providerInputs.host, port = providerInputs.port) => { - const db = await $getClient({ ...providerInputs, port, host }); + const db = await $getClient({ + ...providerInputs, + port, + host + }); try { const revokeStatement = handlebars.compile(providerInputs.revocationStatement)({ username, database }); const queries = revokeStatement.toString().split(";").filter(Boolean); @@ -331,7 +354,11 @@ export const SqlDatabaseProvider = ({ gatewayService }: TSqlDatabaseProviderDTO) if (!providerInputs.renewStatement) return { entityId }; const gatewayCallback = async (host = providerInputs.host, port = providerInputs.port) => { - const db = await $getClient({ ...providerInputs, port, host }); + const db = await $getClient({ + ...providerInputs, + port, + host + }); const expiration = new Date(expireAt).toISOString(); const { database } = providerInputs; diff --git a/frontend/src/pages/secret-manager/SecretDashboardPage/components/ActionBar/CreateDynamicSecretForm/SqlDatabaseInputForm.tsx b/frontend/src/pages/secret-manager/SecretDashboardPage/components/ActionBar/CreateDynamicSecretForm/SqlDatabaseInputForm.tsx index 8bedbbaeb..050a51754 100644 --- a/frontend/src/pages/secret-manager/SecretDashboardPage/components/ActionBar/CreateDynamicSecretForm/SqlDatabaseInputForm.tsx +++ b/frontend/src/pages/secret-manager/SecretDashboardPage/components/ActionBar/CreateDynamicSecretForm/SqlDatabaseInputForm.tsx @@ -19,6 +19,7 @@ import { SecretInput, Select, SelectItem, + Switch, TextArea, Tooltip } from "@app/components/v2"; @@ -66,6 +67,7 @@ const formSchema = z.object({ creationStatement: z.string().min(1), revocationStatement: z.string().min(1), renewStatement: z.string().optional(), + sslEnabled: z.boolean().optional(), ca: z.string().optional(), gatewayId: z.string().optional() }), @@ -200,6 +202,7 @@ export const SqlDatabaseInputForm = ({ const createDynamicSecret = useCreateDynamicSecret(); const { data: gateways, isPending: isGatewaysLoading } = useQuery(gatewaysQueryKeys.list()); + const selectedClient = watch("provider.client"); const handleCreateDynamicSecret = async ({ name, @@ -458,13 +461,34 @@ export const SqlDatabaseInputForm = ({ />
+ {selectedClient === SqlProviders.MsSQL && ( +
+ ( + + + Encrypt Connection (SSL) + + + )} + /> +
+ )} ( diff --git a/frontend/src/pages/secret-manager/SecretDashboardPage/components/DynamicSecretListView/EditDynamicSecretForm/EditDynamicSecretSqlProviderForm.tsx b/frontend/src/pages/secret-manager/SecretDashboardPage/components/DynamicSecretListView/EditDynamicSecretForm/EditDynamicSecretSqlProviderForm.tsx index 2cbad1e19..1eda17bc8 100644 --- a/frontend/src/pages/secret-manager/SecretDashboardPage/components/DynamicSecretListView/EditDynamicSecretForm/EditDynamicSecretSqlProviderForm.tsx +++ b/frontend/src/pages/secret-manager/SecretDashboardPage/components/DynamicSecretListView/EditDynamicSecretForm/EditDynamicSecretSqlProviderForm.tsx @@ -18,6 +18,7 @@ import { SecretInput, Select, SelectItem, + Switch, TextArea, Tooltip } from "@app/components/v2"; @@ -63,6 +64,7 @@ const formSchema = z.object({ creationStatement: z.string().min(1), revocationStatement: z.string().min(1), renewStatement: z.string().optional(), + sslEnabled: z.boolean().optional(), ca: z.string().optional(), gatewayId: z.string().optional().nullable() }) @@ -151,6 +153,7 @@ export const EditDynamicSecretSqlProviderForm = ({ }); const { data: gateways, isPending: isGatewaysLoading } = useQuery(gatewaysQueryKeys.list()); + const selectedClient = watch("inputs.client"); const updateDynamicSecret = useUpdateDynamicSecret(); const selectedGatewayId = watch("inputs.gatewayId"); @@ -407,13 +410,34 @@ export const EditDynamicSecretSqlProviderForm = ({ />
+ {selectedClient === SqlProviders.MsSQL && ( +
+ ( + + + Encrypt Connection (SSL) + + + )} + /> +
+ )} ( From 7ea0cc6cb7e41bd19c0e5149a5c0ea0a4f6eb4c1 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 10 Sep 2025 20:47:14 +0800 Subject: [PATCH 14/18] fix: passed original host and added context comment --- .../dynamic-secret/providers/sql-database.ts | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/backend/src/ee/services/dynamic-secret/providers/sql-database.ts b/backend/src/ee/services/dynamic-secret/providers/sql-database.ts index babbe1912..562aa58f2 100644 --- a/backend/src/ee/services/dynamic-secret/providers/sql-database.ts +++ b/backend/src/ee/services/dynamic-secret/providers/sql-database.ts @@ -151,15 +151,26 @@ export const SqlDatabaseProvider = ({ gatewayService }: TSqlDatabaseProviderDTO) return { ...providerInputs, hostIp }; }; - const $getClient = async (providerInputs: z.infer & { hostIp: string }) => { + const $getClient = async ( + providerInputs: z.infer & { hostIp: string; originalHost: string } + ) => { const ssl = providerInputs.ca ? { rejectUnauthorized: false, ca: providerInputs.ca, servername: providerInputs.host } : undefined; const isMsSQLClient = providerInputs.client === SqlProviders.MsSQL; - const isAzureSql = isMsSQLClient && new RE2(/\.database\.windows\.net$/i).test(providerInputs.host); - const azureServerLabel = isAzureSql ? providerInputs.host.split(".")[0] : undefined; + /* + We route through the gateway by setting connection.host = "localhost". + Azure SQL identifies the logical server from the TDS login name when the host + isn’t the Azure FQDN. Therefore, when using the gateway, ensure username is + "user@" so Azure opens the correct logical server. + Direct connections to the Azure FQDN usually don’t require this suffix. + */ + const isGatewayForwardedTraffic = providerInputs.host === "localhost"; + const isAzureSql = isMsSQLClient && new RE2(/\.database\.windows\.net$/i).test(providerInputs.originalHost); + const azureServerLabel = + isAzureSql && isGatewayForwardedTraffic ? providerInputs.originalHost?.split(".")[0] : undefined; const effectiveUser = isAzureSql && !providerInputs.username.includes("@") ? `${providerInputs.username}@${azureServerLabel}` @@ -226,7 +237,8 @@ export const SqlDatabaseProvider = ({ gatewayService }: TSqlDatabaseProviderDTO) ...providerInputs, port, host, - hostIp: providerInputs.hostIp + hostIp: providerInputs.hostIp, + originalHost: providerInputs.host }); // oracle needs from keyword const testStatement = providerInputs.client === SqlProviders.Oracle ? "SELECT 1 FROM DUAL" : "SELECT 1"; @@ -271,7 +283,8 @@ export const SqlDatabaseProvider = ({ gatewayService }: TSqlDatabaseProviderDTO) const db = await $getClient({ ...providerInputs, port, - host + host, + originalHost: providerInputs.host }); try { const expiration = new Date(expireAt).toISOString(); @@ -318,7 +331,8 @@ export const SqlDatabaseProvider = ({ gatewayService }: TSqlDatabaseProviderDTO) const db = await $getClient({ ...providerInputs, port, - host + host, + originalHost: providerInputs.host }); try { const revokeStatement = handlebars.compile(providerInputs.revocationStatement)({ username, database }); @@ -357,7 +371,8 @@ export const SqlDatabaseProvider = ({ gatewayService }: TSqlDatabaseProviderDTO) const db = await $getClient({ ...providerInputs, port, - host + host, + originalHost: providerInputs.host }); const expiration = new Date(expireAt).toISOString(); const { database } = providerInputs; From a2a96d41cfc94f45c35c7c09911847b86aa70b29 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 10 Sep 2025 21:41:03 +0800 Subject: [PATCH 15/18] misc: added azure server label condition --- .../src/ee/services/dynamic-secret/providers/sql-database.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/ee/services/dynamic-secret/providers/sql-database.ts b/backend/src/ee/services/dynamic-secret/providers/sql-database.ts index e2b89ac04..2b9636724 100644 --- a/backend/src/ee/services/dynamic-secret/providers/sql-database.ts +++ b/backend/src/ee/services/dynamic-secret/providers/sql-database.ts @@ -171,7 +171,7 @@ export const SqlDatabaseProvider = ({ gatewayService }: TSqlDatabaseProviderDTO) const azureServerLabel = isAzureSql && providerInputs.gatewayId ? providerInputs.originalHost?.split(".")[0] : undefined; const effectiveUser = - isAzureSql && !providerInputs.username.includes("@") + isAzureSql && !providerInputs.username.includes("@") && azureServerLabel ? `${providerInputs.username}@${azureServerLabel}` : providerInputs.username; From 0c2ffc75f6a472260d9cf39ddc087580c8718c65 Mon Sep 17 00:00:00 2001 From: Carlos Monastyrski Date: Wed, 10 Sep 2025 12:36:38 -0300 Subject: [PATCH 16/18] Improve self-hosted refresh logic --- .../src/ee/services/license/license-service.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/backend/src/ee/services/license/license-service.ts b/backend/src/ee/services/license/license-service.ts index 23d8b0646..ca1615ce8 100644 --- a/backend/src/ee/services/license/license-service.ts +++ b/backend/src/ee/services/license/license-service.ts @@ -99,6 +99,17 @@ export const licenseServiceFactory = ({ const workspacesUsed = await projectDAL.countOfOrgProjects(null); currentPlan.workspacesUsed = workspacesUsed; + const usedIdentitySeats = await licenseDAL.countOrgUsersAndIdentities(null); + if (usedIdentitySeats !== currentPlan.identitiesUsed) { + const usedSeats = await licenseDAL.countOfOrgMembers(null); + await licenseServerOnPremApi.request.patch(`/api/license/v1/license`, { + usedSeats, + usedIdentitySeats + }); + currentPlan.identitiesUsed = usedIdentitySeats; + currentPlan.membersUsed = usedSeats; + } + onPremFeatures = currentPlan; logger.info("Successfully synchronized license key features"); } catch (error) { @@ -226,10 +237,13 @@ export const licenseServiceFactory = ({ }; const refreshPlan = async (orgId: string) => { + await keyStore.deleteItem(FEATURE_CACHE_KEY(orgId)); if (instanceType === InstanceType.Cloud) { - await keyStore.deleteItem(FEATURE_CACHE_KEY(orgId)); await getPlan(orgId); } + if (instanceType === InstanceType.EnterpriseOnPrem) { + await syncLicenseKeyOnPremFeatures(true); + } }; const generateOrgCustomerId = async (orgName: string, email?: string | null) => { From feff3dd765d4e7256bbec5f695bc1adce003450e Mon Sep 17 00:00:00 2001 From: x032205 Date: Wed, 10 Sep 2025 12:02:41 -0400 Subject: [PATCH 17/18] feat(dynamic-secret): made POST and PATCH endpoints return inputs --- backend/src/ee/routes/v1/dynamic-secret-router.ts | 8 ++++++-- .../ee/services/dynamic-secret/dynamic-secret-service.ts | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/backend/src/ee/routes/v1/dynamic-secret-router.ts b/backend/src/ee/routes/v1/dynamic-secret-router.ts index b916bab67..b1b3cea8e 100644 --- a/backend/src/ee/routes/v1/dynamic-secret-router.ts +++ b/backend/src/ee/routes/v1/dynamic-secret-router.ts @@ -84,7 +84,9 @@ export const registerDynamicSecretRouter = async (server: FastifyZodProvider) => }), response: { 200: z.object({ - dynamicSecret: SanitizedDynamicSecretSchema + dynamicSecret: SanitizedDynamicSecretSchema.extend({ + inputs: z.unknown() + }) }) } }, @@ -151,7 +153,9 @@ export const registerDynamicSecretRouter = async (server: FastifyZodProvider) => }), response: { 200: z.object({ - dynamicSecret: SanitizedDynamicSecretSchema + dynamicSecret: SanitizedDynamicSecretSchema.extend({ + inputs: z.unknown() + }) }) } }, diff --git a/backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts b/backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts index 73dcbe6e3..2baafb053 100644 --- a/backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts +++ b/backend/src/ee/services/dynamic-secret/dynamic-secret-service.ts @@ -180,7 +180,7 @@ export const dynamicSecretServiceFactory = ({ return cfg; }); - return dynamicSecretCfg; + return { ...dynamicSecretCfg, inputs }; }; const updateByName: TDynamicSecretServiceFactory["updateByName"] = async ({ @@ -337,7 +337,7 @@ export const dynamicSecretServiceFactory = ({ return cfg; }); - return updatedDynamicCfg; + return { ...updatedDynamicCfg, inputs: updatedInput }; }; const deleteByName: TDynamicSecretServiceFactory["deleteByName"] = async ({ From 0153cf946713643091b19538948c41148bb147c9 Mon Sep 17 00:00:00 2001 From: = Date: Wed, 10 Sep 2025 22:55:15 +0530 Subject: [PATCH 18/18] fix: resolved validate schema failing in pipeline --- backend/src/services/kms/kms-root-config-dal.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/services/kms/kms-root-config-dal.ts b/backend/src/services/kms/kms-root-config-dal.ts index 4de3fc15a..1b9b9a230 100644 --- a/backend/src/services/kms/kms-root-config-dal.ts +++ b/backend/src/services/kms/kms-root-config-dal.ts @@ -12,7 +12,7 @@ export const kmsRootConfigDALFactory = (db: TDbClient) => { const findById = async (id: string, tx?: Knex) => { try { - const result = await (tx || db.replicaNode())(TableName.KmsServerRootConfig) + const result = await (tx || db?.replicaNode?.() || db)(TableName.KmsServerRootConfig) .where({ id } as never) .first("*"); return result;