Merge branch 'main' into ENG-3586

backend/src/services/app-connection/app-connection-service.ts

@@ -600,7 +600,7 @@ export const appConnectionServiceFactory = ({
     azureClientSecrets: azureClientSecretsConnectionService(connectAppConnectionById, appConnectionDAL, kmsService),
     azureDevOps: azureDevOpsConnectionService(connectAppConnectionById, appConnectionDAL, kmsService),
     auth0: auth0ConnectionService(connectAppConnectionById, appConnectionDAL, kmsService),
-    hcvault: hcVaultConnectionService(connectAppConnectionById),
+    hcvault: hcVaultConnectionService(connectAppConnectionById, gatewayService),
     windmill: windmillConnectionService(connectAppConnectionById),
     teamcity: teamcityConnectionService(connectAppConnectionById),
     oci: ociConnectionService(connectAppConnectionById, licenseService),

backend/src/services/app-connection/auth0/auth0-connection-fns.ts

@@ -91,7 +91,7 @@ export const validateAuth0ConnectionCredentials = async ({ credentials }: TAuth0
     };
   } catch (e: unknown) {
     throw new BadRequestError({
-      message: (e as Error).message ?? `Unable to validate connection: verify credentials`
+      message: (e as Error).message ?? "Unable to validate connection: verify credentials"
     });
   }
 };

backend/src/services/app-connection/azure-app-configuration/azure-app-configuration-connection-fns.ts

@@ -70,7 +70,7 @@ export const validateAzureAppConfigurationConnectionCredentials = async (
           tokenError = e;
         } else {
           throw new BadRequestError({
-            message: `Unable to validate connection: verify credentials`
+            message: "Unable to validate connection: verify credentials"
           });
         }
       }

backend/src/services/app-connection/azure-client-secrets/azure-client-secrets-connection-fns.ts

@@ -186,7 +186,7 @@ export const validateAzureClientSecretsConnectionCredentials = async (config: TA
           tokenError = e;
         } else {
           throw new BadRequestError({
-            message: `Unable to validate connection: verify credentials`
+            message: "Unable to validate connection: verify credentials"
           });
         }
       }

backend/src/services/app-connection/azure-devops/azure-devops-fns.ts

@@ -204,7 +204,7 @@ export const validateAzureDevOpsConnectionCredentials = async (config: TAzureDev
           tokenError = e;
         } else {
           throw new BadRequestError({
-            message: `Unable to validate connection: verify credentials`
+            message: "Unable to validate connection: verify credentials"
           });
         }
       }

backend/src/services/app-connection/azure-key-vault/azure-key-vault-connection-fns.ts

@@ -186,7 +186,7 @@ export const validateAzureKeyVaultConnectionCredentials = async (config: TAzureK
           tokenError = e;
         } else {
           throw new BadRequestError({
-            message: `Unable to validate connection: verify credentials`
+            message: "Unable to validate connection: verify credentials"
           });
         }
       }

backend/src/services/app-connection/camunda/camunda-connection-fns.ts

@@ -82,7 +82,7 @@ export const validateCamundaConnectionCredentials = async (appConnection: TCamun
     };
   } catch (e: unknown) {
     throw new BadRequestError({
-      message: `Unable to validate connection: verify credentials`
+      message: "Unable to validate connection: verify credentials"
     });
   }
 };

backend/src/services/app-connection/databricks/databricks-connection-fns.ts

@@ -89,7 +89,7 @@ export const validateDatabricksConnectionCredentials = async (appConnection: TDa
     };
   } catch (e: unknown) {
     throw new BadRequestError({
-      message: `Unable to validate connection: verify credentials`
+      message: "Unable to validate connection: verify credentials"
     });
   }
 };

backend/src/services/app-connection/github-radar/github-radar-connection-fns.ts

@@ -114,7 +114,7 @@ export const validateGitHubRadarConnectionCredentials = async (config: TGitHubRa
     }
 
     throw new BadRequestError({
-      message: `Unable to validate connection: verify credentials`
+      message: "Unable to validate connection: verify credentials"
     });
   }
 

backend/src/services/app-connection/github/github-connection-fns.ts

@@ -447,7 +447,7 @@ export const validateGitHubConnectionCredentials = async (
     }
 
     throw new BadRequestError({
-      message: `Unable to validate connection: verify credentials`
+      message: "Unable to validate connection: verify credentials"
     });
   }
 

backend/src/services/app-connection/hc-vault/hc-vault-connection-fns.ts

@@ -1,18 +1,18 @@
-import { AxiosError } from "axios";
+import { AxiosError, AxiosRequestConfig, AxiosResponse } from "axios";
+import https from "https";
 
+import { verifyHostInputValidity } from "@app/ee/services/dynamic-secret/dynamic-secret-fns";
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { request } from "@app/lib/config/request";
 import { BadRequestError } from "@app/lib/errors";
 import { removeTrailingSlash } from "@app/lib/fn";
+import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway";
+import { logger } from "@app/lib/logger";
 import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
 import { AppConnection } from "@app/services/app-connection/app-connection-enums";
 
 import { HCVaultConnectionMethod } from "./hc-vault-connection-enums";
-import {
-  THCVaultConnection,
-  THCVaultConnectionConfig,
-  THCVaultMountResponse,
-  TValidateHCVaultConnectionCredentials
-} from "./hc-vault-connection-types";
+import { THCVaultConnection, THCVaultConnectionConfig, THCVaultMountResponse } from "./hc-vault-connection-types";
 
 export const getHCVaultInstanceUrl = async (config: THCVaultConnectionConfig) => {
   const instanceUrl = removeTrailingSlash(config.credentials.instanceUrl);
@@ -37,7 +37,78 @@ type TokenRespData = {
   };
 };
 
-export const getHCVaultAccessToken = async (connection: TValidateHCVaultConnectionCredentials) => {
+export const requestWithHCVaultGateway = async <T>(
+  appConnection: { gatewayId?: string | null },
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
+  requestConfig: AxiosRequestConfig
+): Promise<AxiosResponse<T>> => {
+  const { gatewayId } = appConnection;
+
+  // If gateway isn't set up, don't proxy request
+  if (!gatewayId) {
+    return request.request(requestConfig);
+  }
+
+  const url = new URL(requestConfig.url as string);
+
+  await blockLocalAndPrivateIpAddresses(url.toString());
+
+  const [targetHost] = await verifyHostInputValidity(url.hostname, true);
+  const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(gatewayId);
+  const [relayHost, relayPort] = relayDetails.relayAddress.split(":");
+
+  return withGatewayProxy(
+    async (proxyPort) => {
+      const httpsAgent = new https.Agent({
+        servername: targetHost
+      });
+
+      url.protocol = "https:";
+      url.host = `localhost:${proxyPort}`;
+
+      const finalRequestConfig: AxiosRequestConfig = {
+        ...requestConfig,
+        url: url.toString(),
+        httpsAgent,
+        headers: {
+          ...requestConfig.headers,
+          Host: targetHost
+        }
+      };
+
+      try {
+        return await request.request(finalRequestConfig);
+      } catch (error) {
+        if (error instanceof AxiosError) {
+          logger.error(
+            { message: error.message, data: (error.response as undefined | { data: unknown })?.data },
+            "Error during HashiCorp Vault gateway request:"
+          );
+        }
+        throw error;
+      }
+    },
+    {
+      protocol: GatewayProxyProtocol.Tcp,
+      targetHost,
+      targetPort: url.port ? Number(url.port) : 8200, // 8200 is the default port for Vault self-hosted/dedicated
+      relayHost,
+      relayPort: Number(relayPort),
+      identityId: relayDetails.identityId,
+      orgId: relayDetails.orgId,
+      tlsOptions: {
+        ca: relayDetails.certChain,
+        cert: relayDetails.certificate,
+        key: relayDetails.privateKey.toString()
+      }
+    }
+  );
+};
+
+export const getHCVaultAccessToken = async (
+  connection: THCVaultConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
   // Return access token directly if not using AppRole method
   if (connection.method !== HCVaultConnectionMethod.AppRole) {
     return connection.credentials.accessToken;
@@ -46,16 +117,16 @@ export const getHCVaultAccessToken = async (connection: TValidateHCVaultConnecti
   // Generate temporary token for AppRole method
   try {
     const { instanceUrl, roleId, secretId } = connection.credentials;
-    const tokenResp = await request.post<TokenRespData>(
-      `${removeTrailingSlash(instanceUrl)}/v1/auth/approle/login`,
-      { role_id: roleId, secret_id: secretId },
-      {
-        headers: {
-          "Content-Type": "application/json",
-          ...(connection.credentials.namespace ? { "X-Vault-Namespace": connection.credentials.namespace } : {})
-        }
-      }
-    );
+
+    const tokenResp = await requestWithHCVaultGateway<TokenRespData>(connection, gatewayService, {
+      url: `${removeTrailingSlash(instanceUrl)}/v1/auth/approle/login`,
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        ...(connection.credentials.namespace ? { "X-Vault-Namespace": connection.credentials.namespace } : {})
+      },
+      data: { role_id: roleId, secret_id: secretId }
+    });
 
     if (tokenResp.status !== 200) {
       throw new BadRequestError({
@@ -71,38 +142,55 @@ export const getHCVaultAccessToken = async (connection: TValidateHCVaultConnecti
   }
 };
 
-export const validateHCVaultConnectionCredentials = async (config: THCVaultConnectionConfig) => {
-  const instanceUrl = await getHCVaultInstanceUrl(config);
+export const validateHCVaultConnectionCredentials = async (
+  connection: THCVaultConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
+  const instanceUrl = await getHCVaultInstanceUrl(connection);
 
   try {
-    const accessToken = await getHCVaultAccessToken(config);
+    const accessToken = await getHCVaultAccessToken(connection, gatewayService);
 
     // Verify token
-    await request.get(`${instanceUrl}/v1/auth/token/lookup-self`, {
+    await requestWithHCVaultGateway(connection, gatewayService, {
+      url: `${instanceUrl}/v1/auth/token/lookup-self`,
+      method: "GET",
       headers: { "X-Vault-Token": accessToken }
     });
 
-    return config.credentials;
+    return connection.credentials;
   } catch (error: unknown) {
+    logger.error(error, "Unable to verify HC Vault connection");
+
     if (error instanceof AxiosError) {
       throw new BadRequestError({
         message: `Failed to validate credentials: ${error.message || "Unknown error"}`
       });
     }
+
+    if (error instanceof BadRequestError) {
+      throw error;
+    }
+
     throw new BadRequestError({
       message: "Unable to validate connection: verify credentials"
     });
   }
 };
 
-export const listHCVaultMounts = async (appConnection: THCVaultConnection) => {
-  const instanceUrl = await getHCVaultInstanceUrl(appConnection);
-  const accessToken = await getHCVaultAccessToken(appConnection);
+export const listHCVaultMounts = async (
+  connection: THCVaultConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
+  const instanceUrl = await getHCVaultInstanceUrl(connection);
+  const accessToken = await getHCVaultAccessToken(connection, gatewayService);
 
-  const { data } = await request.get<THCVaultMountResponse>(`${instanceUrl}/v1/sys/mounts`, {
+  const { data } = await requestWithHCVaultGateway<THCVaultMountResponse>(connection, gatewayService, {
+    url: `${instanceUrl}/v1/sys/mounts`,
+    method: "GET",
     headers: {
       "X-Vault-Token": accessToken,
-      ...(appConnection.credentials.namespace ? { "X-Vault-Namespace": appConnection.credentials.namespace } : {})
+      ...(connection.credentials.namespace ? { "X-Vault-Namespace": connection.credentials.namespace } : {})
     }
   });
 

backend/src/services/app-connection/hc-vault/hc-vault-connection-schemas.ts

@@ -55,11 +55,18 @@ export const HCVaultConnectionSchema = z.intersection(
 export const SanitizedHCVaultConnectionSchema = z.discriminatedUnion("method", [
   BaseHCVaultConnectionSchema.extend({
     method: z.literal(HCVaultConnectionMethod.AccessToken),
-    credentials: HCVaultConnectionAccessTokenCredentialsSchema.pick({})
+    credentials: HCVaultConnectionAccessTokenCredentialsSchema.pick({
+      namespace: true,
+      instanceUrl: true
+    })
   }),
   BaseHCVaultConnectionSchema.extend({
     method: z.literal(HCVaultConnectionMethod.AppRole),
-    credentials: HCVaultConnectionAppRoleCredentialsSchema.pick({})
+    credentials: HCVaultConnectionAppRoleCredentialsSchema.pick({
+      namespace: true,
+      instanceUrl: true,
+      roleId: true
+    })
   })
 ]);
 
@@ -81,7 +88,7 @@ export const ValidateHCVaultConnectionCredentialsSchema = z.discriminatedUnion("
 ]);
 
 export const CreateHCVaultConnectionSchema = ValidateHCVaultConnectionCredentialsSchema.and(
-  GenericCreateAppConnectionFieldsSchema(AppConnection.HCVault)
+  GenericCreateAppConnectionFieldsSchema(AppConnection.HCVault, { supportsGateways: true })
 );
 
 export const UpdateHCVaultConnectionSchema = z
@@ -91,7 +98,7 @@ export const UpdateHCVaultConnectionSchema = z
       .optional()
       .describe(AppConnections.UPDATE(AppConnection.HCVault).credentials)
   })
-  .and(GenericUpdateAppConnectionFieldsSchema(AppConnection.HCVault));
+  .and(GenericUpdateAppConnectionFieldsSchema(AppConnection.HCVault, { supportsGateways: true }));
 
 export const HCVaultConnectionListItemSchema = z.object({
   name: z.literal("HCVault"),

backend/src/services/app-connection/hc-vault/hc-vault-connection-service.ts

@@ -1,3 +1,4 @@
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { logger } from "@app/lib/logger";
 import { OrgServiceActor } from "@app/lib/types";
 
@@ -11,12 +12,15 @@ type TGetAppConnectionFunc = (
   actor: OrgServiceActor
 ) => Promise<THCVaultConnection>;
 
-export const hcVaultConnectionService = (getAppConnection: TGetAppConnectionFunc) => {
+export const hcVaultConnectionService = (
+  getAppConnection: TGetAppConnectionFunc,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
   const listMounts = async (connectionId: string, actor: OrgServiceActor) => {
     const appConnection = await getAppConnection(AppConnection.HCVault, connectionId, actor);
 
     try {
-      const mounts = await listHCVaultMounts(appConnection);
+      const mounts = await listHCVaultMounts(appConnection, gatewayService);
       return mounts;
     } catch (error) {
       logger.error(error, "Failed to establish connection with Hashicorp Vault");

backend/src/services/app-connection/shared/sql/sql-connection-fns.ts

@@ -56,7 +56,7 @@ const getConnectionConfig = ({
           ? {
               rejectUnauthorized: sslRejectUnauthorized,
               ca: sslCertificate,
-              servername: host
+              serverName: host
             }
           : false
       };
@@ -90,7 +90,7 @@ export const getSqlConnectionClient = async (appConnection: Pick<TSqlConnection,
     connection: {
       database,
       port,
-      host,
+      host: app === AppConnection.Postgres ? host : baseHost,
       user: username,
       password,
       connectionTimeoutMillis: EXTERNAL_REQUEST_TIMEOUT,
@@ -135,7 +135,7 @@ export const executeWithPotentialGateway = async <T>(
       },
       {
         protocol: GatewayProxyProtocol.Tcp,
-        targetHost,
+        targetHost: app === AppConnection.Postgres ? targetHost : credentials.host,
         targetPort: credentials.port,
         relayHost,
         relayPort: Number(relayPort),

backend/src/services/auth/auth-login-service.ts

@@ -453,19 +453,24 @@ export const authLoginServiceFactory = ({
 
     const selectedOrg = await orgDAL.findById(organizationId);
 
-    // Check if authEnforced is true, if that's the case, throw an error
-    if (selectedOrg.authEnforced) {
-      throw new BadRequestError({
-        message: "Authentication is required by your organization before you can log in."
-      });
-    }
-
     if (!selectedOrgMembership) {
       throw new ForbiddenRequestError({
         message: `User does not have access to the organization named ${selectedOrg?.name}`
       });
     }
 
+    // Check if authEnforced is true and the current auth method is not an enforced method
+    if (
+      selectedOrg.authEnforced &&
+      !isAuthMethodSaml(decodedToken.authMethod) &&
+      decodedToken.authMethod !== AuthMethod.OIDC &&
+      !(selectedOrg.bypassOrgAuthEnabled && selectedOrgMembership.userRole === OrgMembershipRole.Admin)
+    ) {
+      throw new BadRequestError({
+        message: "Login with the auth method required by your organization."
+      });
+    }
+
     if (selectedOrg.googleSsoAuthEnforced && decodedToken.authMethod !== AuthMethod.GOOGLE) {
       const canBypass = selectedOrg.bypassOrgAuthEnabled && selectedOrgMembership.userRole === OrgMembershipRole.Admin;
 

backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts

@@ -64,6 +64,8 @@ type DBConfigurationColumn = {
   directoryUrl: string;
   accountEmail: string;
   hostedZoneId: string;
+  eabKid?: string;
+  eabHmacKey?: string;
 };
 
 export const castDbEntryToAcmeCertificateAuthority = (
@@ -89,7 +91,9 @@ export const castDbEntryToAcmeCertificateAuthority = (
         hostedZoneId: dbConfigurationCol.hostedZoneId
       },
       directoryUrl: dbConfigurationCol.directoryUrl,
-      accountEmail: dbConfigurationCol.accountEmail
+      accountEmail: dbConfigurationCol.accountEmail,
+      eabKid: dbConfigurationCol.eabKid,
+      eabHmacKey: dbConfigurationCol.eabHmacKey
     },
     status: ca.status as CaStatus
   };
@@ -128,7 +132,7 @@ export const AcmeCertificateAuthorityFns = ({
       });
     }
 
-    const { dnsAppConnectionId, directoryUrl, accountEmail, dnsProviderConfig } = configuration;
+    const { dnsAppConnectionId, directoryUrl, accountEmail, dnsProviderConfig, eabKid, eabHmacKey } = configuration;
     const appConnection = await appConnectionDAL.findById(dnsAppConnectionId);
 
     if (!appConnection) {
@@ -171,7 +175,9 @@ export const AcmeCertificateAuthorityFns = ({
               directoryUrl,
               accountEmail,
               dnsProvider: dnsProviderConfig.provider,
-              hostedZoneId: dnsProviderConfig.hostedZoneId
+              hostedZoneId: dnsProviderConfig.hostedZoneId,
+              eabKid,
+              eabHmacKey
             }
           },
           tx
@@ -214,7 +220,7 @@ export const AcmeCertificateAuthorityFns = ({
   }) => {
     const updatedCa = await certificateAuthorityDAL.transaction(async (tx) => {
       if (configuration) {
-        const { dnsAppConnectionId, directoryUrl, accountEmail, dnsProviderConfig } = configuration;
+        const { dnsAppConnectionId, directoryUrl, accountEmail, dnsProviderConfig, eabKid, eabHmacKey } = configuration;
         const appConnection = await appConnectionDAL.findById(dnsAppConnectionId);
 
         if (!appConnection) {
@@ -254,7 +260,9 @@ export const AcmeCertificateAuthorityFns = ({
               directoryUrl,
               accountEmail,
               dnsProvider: dnsProviderConfig.provider,
-              hostedZoneId: dnsProviderConfig.hostedZoneId
+              hostedZoneId: dnsProviderConfig.hostedZoneId,
+              eabKid,
+              eabHmacKey
             }
           },
           tx
@@ -354,10 +362,19 @@ export const AcmeCertificateAuthorityFns = ({
 
     await blockLocalAndPrivateIpAddresses(acmeCa.configuration.directoryUrl);
 
-    const acmeClient = new acme.Client({
+    const acmeClientOptions: acme.ClientOptions = {
       directoryUrl: acmeCa.configuration.directoryUrl,
       accountKey
-    });
+    };
+
+    if (acmeCa.configuration.eabKid && acmeCa.configuration.eabHmacKey) {
+      acmeClientOptions.externalAccountBinding = {
+        kid: acmeCa.configuration.eabKid,
+        hmacKey: acmeCa.configuration.eabHmacKey
+      };
+    }
+
+    const acmeClient = new acme.Client(acmeClientOptions);
 
     const alg = keyAlgorithmToAlgCfg(CertKeyAlgorithm.RSA_2048);
 

backend/src/services/certificate-authority/acme/acme-certificate-authority-schemas.ts

@@ -18,7 +18,9 @@ export const AcmeCertificateAuthorityConfigurationSchema = z.object({
     hostedZoneId: z.string().trim().min(1).describe(CertificateAuthorities.CONFIGURATIONS.ACME.hostedZoneId)
   }),
   directoryUrl: z.string().url().trim().min(1).describe(CertificateAuthorities.CONFIGURATIONS.ACME.directoryUrl),
-  accountEmail: z.string().trim().min(1).describe(CertificateAuthorities.CONFIGURATIONS.ACME.accountEmail)
+  accountEmail: z.string().trim().min(1).describe(CertificateAuthorities.CONFIGURATIONS.ACME.accountEmail),
+  eabKid: z.string().trim().max(64).optional().describe(CertificateAuthorities.CONFIGURATIONS.ACME.eabKid),
+  eabHmacKey: z.string().trim().max(512).optional().describe(CertificateAuthorities.CONFIGURATIONS.ACME.eabHmacKey)
 });
 
 export const AcmeCertificateAuthorityCredentialsSchema = z.object({

backend/src/services/certificate-authority/internal/internal-certificate-authority-service.ts

@@ -1190,7 +1190,9 @@ export const internalCertificateAuthorityServiceFactory = ({
         });
       }
 
-      collectionId = certificateTemplate.pkiCollectionId as string;
+      if (!collectionId) {
+        collectionId = certificateTemplate.pkiCollectionId as string;
+      }
       ca = await certificateAuthorityDAL.findByIdWithAssociatedCa(certificateTemplate.caId);
     }
 

backend/src/services/external-migration/external-migration-fns/vault.ts

@@ -1,12 +1,21 @@
+import https from "node:https";
+
 import axios, { AxiosInstance } from "axios";
 import { v4 as uuidv4 } from "uuid";
 
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { BadRequestError } from "@app/lib/errors";
+import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway";
 import { logger } from "@app/lib/logger";
 import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
 
 import { InfisicalImportData, VaultMappingType } from "../external-migration-types";
 
+enum KvVersion {
+  V1 = "1",
+  V2 = "2"
+}
+
 type VaultData = {
   namespace: string;
   mount: string;
@@ -14,7 +23,42 @@ type VaultData = {
   secretData: Record<string, string>;
 };
 
-const vaultFactory = () => {
+const vaultFactory = (gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">) => {
+  const $gatewayProxyWrapper = async <T>(
+    inputs: {
+      gatewayId: string;
+      targetHost?: string;
+      targetPort?: number;
+    },
+    gatewayCallback: (host: string, port: number, httpsAgent?: https.Agent) => Promise<T>
+  ): Promise<T> => {
+    const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(inputs.gatewayId);
+    const [relayHost, relayPort] = relayDetails.relayAddress.split(":");
+
+    const callbackResult = await withGatewayProxy(
+      async (port, httpsAgent) => {
+        const res = await gatewayCallback("http://localhost", port, httpsAgent);
+        return res;
+      },
+      {
+        protocol: GatewayProxyProtocol.Http,
+        targetHost: inputs.targetHost,
+        targetPort: inputs.targetPort,
+        relayHost,
+        relayPort: Number(relayPort),
+        identityId: relayDetails.identityId,
+        orgId: relayDetails.orgId,
+        tlsOptions: {
+          ca: relayDetails.certChain,
+          cert: relayDetails.certificate,
+          key: relayDetails.privateKey.toString()
+        }
+      }
+    );
+
+    return callbackResult;
+  };
+
   const getMounts = async (request: AxiosInstance) => {
     const response = await request
       .get<{
@@ -31,11 +75,24 @@ const vaultFactory = () => {
 
   const getPaths = async (
     request: AxiosInstance,
-    { mountPath, secretPath = "" }: { mountPath: string; secretPath?: string }
+    { mountPath, secretPath = "" }: { mountPath: string; secretPath?: string },
+    kvVersion: KvVersion
   ) => {
     try {
-      // For KV v2: /v1/{mount}/metadata/{path}?list=true
-      const path = secretPath ? `${mountPath}/metadata/${secretPath}` : `${mountPath}/metadata`;
+      if (kvVersion === KvVersion.V2) {
+        // For KV v2: /v1/{mount}/metadata/{path}?list=true
+        const path = secretPath ? `${mountPath}/metadata/${secretPath}` : `${mountPath}/metadata`;
+        const response = await request.get<{
+          data: {
+            keys: string[];
+          };
+        }>(`/v1/${path}?list=true`);
+
+        return response.data.data.keys;
+      }
+
+      // kv version v1: /v1/{mount}?list=true
+      const path = secretPath ? `${mountPath}/${secretPath}` : mountPath;
       const response = await request.get<{
         data: {
           keys: string[];
@@ -56,21 +113,42 @@ const vaultFactory = () => {
 
   const getSecrets = async (
     request: AxiosInstance,
-    { mountPath, secretPath }: { mountPath: string; secretPath: string }
+    { mountPath, secretPath }: { mountPath: string; secretPath: string },
+    kvVersion: KvVersion
   ) => {
-    // For KV v2: /v1/{mount}/data/{path}
+    if (kvVersion === KvVersion.V2) {
+      // For KV v2: /v1/{mount}/data/{path}
+      const response = await request
+        .get<{
+          data: {
+            data: Record<string, string>; // KV v2 has nested data structure
+            metadata: {
+              created_time: string;
+              deletion_time: string;
+              destroyed: boolean;
+              version: number;
+            };
+          };
+        }>(`/v1/${mountPath}/data/${secretPath}`)
+        .catch((err) => {
+          if (axios.isAxiosError(err)) {
+            logger.error(err.response?.data, "External migration: Failed to get Vault secret");
+          }
+          throw err;
+        });
+
+      return response.data.data.data;
+    }
+
+    // kv version v1
+
     const response = await request
       .get<{
-        data: {
-          data: Record<string, string>; // KV v2 has nested data structure
-          metadata: {
-            created_time: string;
-            deletion_time: string;
-            destroyed: boolean;
-            version: number;
-          };
-        };
-      }>(`/v1/${mountPath}/data/${secretPath}`)
+        data: Record<string, string>; // KV v1 has flat data structure
+        lease_duration: number;
+        lease_id: string;
+        renewable: boolean;
+      }>(`/v1/${mountPath}/${secretPath}`)
       .catch((err) => {
         if (axios.isAxiosError(err)) {
           logger.error(err.response?.data, "External migration: Failed to get Vault secret");
@@ -78,7 +156,7 @@ const vaultFactory = () => {
         throw err;
       });
 
-    return response.data.data.data;
+    return response.data.data;
   };
 
   // helper function to check if a mount is KV v2 (will be useful if we add support for Vault KV v1)
@@ -89,9 +167,10 @@ const vaultFactory = () => {
   const recursivelyGetAllPaths = async (
     request: AxiosInstance,
     mountPath: string,
+    kvVersion: KvVersion,
     currentPath: string = ""
   ): Promise<string[]> => {
-    const paths = await getPaths(request, { mountPath, secretPath: currentPath });
+    const paths = await getPaths(request, { mountPath, secretPath: currentPath }, kvVersion);
 
     if (paths === null || paths.length === 0) {
       return [];
@@ -105,7 +184,7 @@ const vaultFactory = () => {
 
       if (path.endsWith("/")) {
         // it's a folder so we recurse into it
-        const subSecrets = await recursivelyGetAllPaths(request, mountPath, fullItemPath);
+        const subSecrets = await recursivelyGetAllPaths(request, mountPath, kvVersion, fullItemPath);
         allSecrets.push(...subSecrets);
       } else {
         // it's a secret so we add it to our results
@@ -119,60 +198,93 @@ const vaultFactory = () => {
   async function collectVaultData({
     baseUrl,
     namespace,
-    accessToken
+    accessToken,
+    gatewayId
   }: {
     baseUrl: string;
     namespace?: string;
     accessToken: string;
+    gatewayId?: string;
   }): Promise<VaultData[]> {
-    const request = axios.create({
-      baseURL: baseUrl,
-      headers: {
-        "X-Vault-Token": accessToken,
-        ...(namespace ? { "X-Vault-Namespace": namespace } : {})
+    const getData = async (host: string, port?: number, httpsAgent?: https.Agent) => {
+      const allData: VaultData[] = [];
+
+      const request = axios.create({
+        baseURL: port ? `${host}:${port}` : host,
+        headers: {
+          "X-Vault-Token": accessToken,
+          ...(namespace ? { "X-Vault-Namespace": namespace } : {})
+        },
+        httpsAgent
+      });
+
+      // Get all mounts in this namespace
+      const mounts = await getMounts(request);
+
+      for (const mount of Object.keys(mounts)) {
+        if (!mount.endsWith("/")) {
+          delete mounts[mount];
+        }
       }
-    });
 
-    const allData: VaultData[] = [];
+      for await (const [mountPath, mountInfo] of Object.entries(mounts)) {
+        // skip non-KV mounts
+        if (!mountInfo.type.startsWith("kv")) {
+          // eslint-disable-next-line no-continue
+          continue;
+        }
 
-    // Get all mounts in this namespace
-    const mounts = await getMounts(request);
+        const kvVersion = mountInfo.options?.version === "2" ? KvVersion.V2 : KvVersion.V1;
 
-    for (const mount of Object.keys(mounts)) {
-      if (!mount.endsWith("/")) {
-        delete mounts[mount];
+        // get all paths in this mount
+        const paths = await recursivelyGetAllPaths(request, `${mountPath.replace(/\/$/, "")}`, kvVersion);
+
+        const cleanMountPath = mountPath.replace(/\/$/, "");
+
+        for await (const secretPath of paths) {
+          // get the actual secret data
+          const secretData = await getSecrets(
+            request,
+            {
+              mountPath: cleanMountPath,
+              secretPath: secretPath.replace(`${cleanMountPath}/`, "")
+            },
+            kvVersion
+          );
+
+          allData.push({
+            namespace: namespace || "",
+            mount: mountPath.replace(/\/$/, ""),
+            path: secretPath.replace(`${cleanMountPath}/`, ""),
+            secretData
+          });
+        }
       }
+
+      return allData;
+    };
+
+    let data;
+
+    if (gatewayId) {
+      const url = new URL(baseUrl);
+
+      const { port, protocol, hostname } = url;
+      const cleanedProtocol = protocol.slice(0, -1);
+
+      data = await $gatewayProxyWrapper(
+        {
+          gatewayId,
+          targetHost: `${cleanedProtocol}://${hostname}`,
+          targetPort: port ? Number(port) : 8200 // 8200, default port for Vault self-hosted/dedicated
+        },
+        getData
+      );
+    } else {
+      data = await getData(baseUrl);
     }
 
-    for await (const [mountPath, mountInfo] of Object.entries(mounts)) {
-      // skip non-KV mounts
-      if (!mountInfo.type.startsWith("kv")) {
-        // eslint-disable-next-line no-continue
-        continue;
-      }
-
-      // get all paths in this mount
-      const paths = await recursivelyGetAllPaths(request, `${mountPath.replace(/\/$/, "")}`);
-
-      const cleanMountPath = mountPath.replace(/\/$/, "");
-
-      for await (const secretPath of paths) {
-        // get the actual secret data
-        const secretData = await getSecrets(request, {
-          mountPath: cleanMountPath,
-          secretPath: secretPath.replace(`${cleanMountPath}/`, "")
-        });
-
-        allData.push({
-          namespace: namespace || "",
-          mount: mountPath.replace(/\/$/, ""),
-          path: secretPath.replace(`${cleanMountPath}/`, ""),
-          secretData
-        });
-      }
-    }
-
-    return allData;
+    return data;
   }
 
   return {
@@ -296,17 +408,126 @@ export const transformToInfisicalFormatNamespaceToProjects = (
   };
 };
 
-export const importVaultDataFn = async ({
-  vaultAccessToken,
-  vaultNamespace,
-  vaultUrl,
-  mappingType
-}: {
-  vaultAccessToken: string;
-  vaultNamespace?: string;
-  vaultUrl: string;
-  mappingType: VaultMappingType;
-}) => {
+export const transformToInfisicalFormatKeyVaultToProjectsCustomC1 = (vaultData: VaultData[]): InfisicalImportData => {
+  const projects: Array<{ name: string; id: string }> = [];
+  const environments: Array<{ name: string; id: string; projectId: string; envParentId?: string }> = [];
+  const folders: Array<{ id: string; name: string; environmentId: string; parentFolderId?: string }> = [];
+  const secrets: Array<{ id: string; name: string; environmentId: string; value: string; folderId?: string }> = [];
+
+  // track created entities to avoid duplicates
+  const projectMap = new Map<string, string>(); // team name -> projectId
+  const environmentMap = new Map<string, string>(); // team-name:envName -> environmentId
+  const folderMap = new Map<string, string>(); // team-name:envName:folderPath -> folderId
+
+  for (const data of vaultData) {
+    const { path, secretData } = data;
+
+    const pathParts = path.split("/").filter(Boolean);
+    if (pathParts.length < 2) {
+      // eslint-disable-next-line no-continue
+      continue;
+    }
+
+    // first level: environment (dev, prod, staging, etc.)
+    const environmentName = pathParts[0];
+    // second level: team name (team1, team2, etc.)
+    const teamName = pathParts[1];
+    // remaining parts: folder structure
+    const folderParts = pathParts.slice(2);
+
+    // create project (team) if if doesn't exist
+    if (!projectMap.has(teamName)) {
+      const projectId = uuidv4();
+      projectMap.set(teamName, projectId);
+      projects.push({
+        name: teamName,
+        id: projectId
+      });
+    }
+    const projectId = projectMap.get(teamName)!;
+
+    // create environment (dev, prod, etc.) for team
+    const envKey = `${teamName}:${environmentName}`;
+    if (!environmentMap.has(envKey)) {
+      const environmentId = uuidv4();
+      environmentMap.set(envKey, environmentId);
+      environments.push({
+        name: environmentName,
+        id: environmentId,
+        projectId
+      });
+    }
+    const environmentId = environmentMap.get(envKey)!;
+
+    // create folder structure for path segments
+    let currentFolderId: string | undefined;
+    let currentPath = "";
+
+    for (const folderName of folderParts) {
+      currentPath = currentPath ? `${currentPath}/${folderName}` : folderName;
+      const folderKey = `${teamName}:${environmentName}:${currentPath}`;
+
+      if (!folderMap.has(folderKey)) {
+        const folderId = uuidv4();
+        folderMap.set(folderKey, folderId);
+        folders.push({
+          id: folderId,
+          name: folderName,
+          environmentId,
+          parentFolderId: currentFolderId || environmentId
+        });
+        currentFolderId = folderId;
+      } else {
+        currentFolderId = folderMap.get(folderKey)!;
+      }
+    }
+
+    for (const [key, value] of Object.entries(secretData)) {
+      secrets.push({
+        id: uuidv4(),
+        name: key,
+        environmentId,
+        value: String(value),
+        folderId: currentFolderId
+      });
+    }
+  }
+
+  return {
+    projects,
+    environments,
+    folders,
+    secrets
+  };
+};
+
+// refer to internal doc for more details on which ID's belong to which orgs.
+// when its a custom migration, then it doesn't matter which mapping type is used (as of now).
+export const vaultMigrationTransformMappings: Record<
+  string,
+  (vaultData: VaultData[], mappingType: VaultMappingType) => InfisicalImportData
+> = {
+  "68c57ab3-cea5-41fc-ae38-e156b10c14d2": transformToInfisicalFormatKeyVaultToProjectsCustomC1
+} as const;
+
+export const importVaultDataFn = async (
+  {
+    vaultAccessToken,
+    vaultNamespace,
+    vaultUrl,
+    mappingType,
+    gatewayId,
+    orgId
+  }: {
+    vaultAccessToken: string;
+    vaultNamespace?: string;
+    vaultUrl: string;
+    mappingType: VaultMappingType;
+    gatewayId?: string;
+    orgId: string;
+  },
+  { gatewayService }: { gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId"> }
+) => {
   await blockLocalAndPrivateIpAddresses(vaultUrl);
 
   if (mappingType === VaultMappingType.Namespace && !vaultNamespace) {
@@ -315,15 +536,33 @@ export const importVaultDataFn = async ({
     });
   }
 
-  const vaultApi = vaultFactory();
+  let transformFn: (vaultData: VaultData[], mappingType: VaultMappingType) => InfisicalImportData;
+
+  if (mappingType === VaultMappingType.Custom) {
+    transformFn = vaultMigrationTransformMappings[orgId];
+
+    if (!transformFn) {
+      throw new BadRequestError({
+        message: "Please contact our sales team to enable custom vault migrations."
+      });
+    }
+  } else {
+    transformFn = transformToInfisicalFormatNamespaceToProjects;
+  }
+
+  logger.info(
+    { orgId, mappingType },
+    `[importVaultDataFn]: Running ${orgId in vaultMigrationTransformMappings ? "custom" : "default"} transform`
+  );
+
+  const vaultApi = vaultFactory(gatewayService);
 
   const vaultData = await vaultApi.collectVaultData({
     accessToken: vaultAccessToken,
     baseUrl: vaultUrl,
-    namespace: vaultNamespace
+    namespace: vaultNamespace,
+    gatewayId
   });
 
-  const infisicalData = transformToInfisicalFormatNamespaceToProjects(vaultData, mappingType);
-
-  return infisicalData;
+  return transformFn(vaultData, mappingType);
 };

backend/src/services/external-migration/external-migration-service.ts

@@ -1,17 +1,30 @@
 import { OrgMembershipRole } from "@app/db/schemas";
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";
 import { crypto } from "@app/lib/crypto/cryptography";
 import { BadRequestError, ForbiddenRequestError } from "@app/lib/errors";
 
 import { TUserDALFactory } from "../user/user-dal";
-import { decryptEnvKeyDataFn, importVaultDataFn, parseEnvKeyDataFn } from "./external-migration-fns";
+import {
+  decryptEnvKeyDataFn,
+  importVaultDataFn,
+  parseEnvKeyDataFn,
+  vaultMigrationTransformMappings
+} from "./external-migration-fns";
 import { TExternalMigrationQueueFactory } from "./external-migration-queue";
-import { ExternalPlatforms, TImportEnvKeyDataDTO, TImportVaultDataDTO } from "./external-migration-types";
+import {
+  ExternalMigrationProviders,
+  ExternalPlatforms,
+  THasCustomVaultMigrationDTO,
+  TImportEnvKeyDataDTO,
+  TImportVaultDataDTO
+} from "./external-migration-types";
 
 type TExternalMigrationServiceFactoryDep = {
   permissionService: TPermissionServiceFactory;
   externalMigrationQueue: TExternalMigrationQueueFactory;
   userDAL: Pick<TUserDALFactory, "findById">;
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
 };
 
 export type TExternalMigrationServiceFactory = ReturnType<typeof externalMigrationServiceFactory>;
@@ -19,7 +32,8 @@ export type TExternalMigrationServiceFactory = ReturnType<typeof externalMigrati
 export const externalMigrationServiceFactory = ({
   permissionService,
   externalMigrationQueue,
-  userDAL
+  userDAL,
+  gatewayService
 }: TExternalMigrationServiceFactoryDep) => {
   const importEnvKeyData = async ({
     decryptionKey,
@@ -72,6 +86,7 @@ export const externalMigrationServiceFactory = ({
     vaultNamespace,
     mappingType,
     vaultUrl,
+    gatewayId,
     actor,
     actorId,
     actorOrgId,
@@ -91,12 +106,19 @@ export const externalMigrationServiceFactory = ({
 
     const user = await userDAL.findById(actorId);
 
-    const vaultData = await importVaultDataFn({
-      vaultAccessToken,
-      vaultNamespace,
-      vaultUrl,
-      mappingType
-    });
+    const vaultData = await importVaultDataFn(
+      {
+        vaultAccessToken,
+        vaultNamespace,
+        vaultUrl,
+        mappingType,
+        gatewayId,
+        orgId: actorOrgId
+      },
+      {
+        gatewayService
+      }
+    );
 
     const stringifiedJson = JSON.stringify({
       data: vaultData,
@@ -117,8 +139,37 @@ export const externalMigrationServiceFactory = ({
     });
   };
 
+  const hasCustomVaultMigration = async ({
+    actor,
+    actorId,
+    actorOrgId,
+    actorAuthMethod,
+    provider
+  }: THasCustomVaultMigrationDTO) => {
+    const { membership } = await permissionService.getOrgPermission(
+      actor,
+      actorId,
+      actorOrgId,
+      actorAuthMethod,
+      actorOrgId
+    );
+
+    if (membership.role !== OrgMembershipRole.Admin) {
+      throw new ForbiddenRequestError({ message: "Only admins can check custom migration status" });
+    }
+
+    if (provider !== ExternalMigrationProviders.Vault) {
+      throw new BadRequestError({
+        message: "Invalid provider. Vault is the only supported provider for custom migrations."
+      });
+    }
+
+    return actorOrgId in vaultMigrationTransformMappings;
+  };
+
   return {
     importEnvKeyData,
-    importVaultData
+    importVaultData,
+    hasCustomVaultMigration
   };
 };

backend/src/services/external-migration/external-migration-types.ts

@@ -4,7 +4,8 @@ import { ActorAuthMethod, ActorType } from "../auth/auth-type";
 
 export enum VaultMappingType {
   Namespace = "namespace",
-  KeyVault = "key-vault"
+  KeyVault = "key-vault",
+  Custom = "custom"
 }
 
 export type InfisicalImportData = {
@@ -26,11 +27,16 @@ export type TImportEnvKeyDataDTO = {
   encryptedJson: { nonce: string; data: string };
 } & Omit<TOrgPermission, "orgId">;
 
+export type THasCustomVaultMigrationDTO = {
+  provider: ExternalMigrationProviders;
+} & Omit<TOrgPermission, "orgId">;
+
 export type TImportVaultDataDTO = {
   vaultAccessToken: string;
   vaultNamespace?: string;
   mappingType: VaultMappingType;
   vaultUrl: string;
+  gatewayId?: string;
 } & Omit<TOrgPermission, "orgId">;
 
 export type TImportInfisicalDataCreate = {
@@ -110,3 +116,8 @@ export enum ExternalPlatforms {
   EnvKey = "EnvKey",
   Vault = "Vault"
 }
+
+export enum ExternalMigrationProviders {
+  Vault = "vault",
+  EnvKey = "env-key"
+}

backend/src/services/identity-ua/identity-ua-service.ts

@@ -8,11 +8,18 @@ import {
   validatePrivilegeChangeOperation
 } from "@app/ee/services/permission/permission-fns";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";
-import { PgSqlLock, TKeyStoreFactory } from "@app/keystore/keystore";
+import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
 import { getConfig } from "@app/lib/config/env";
 import { crypto } from "@app/lib/crypto/cryptography";
-import { BadRequestError, NotFoundError, PermissionBoundaryError, UnauthorizedError } from "@app/lib/errors";
+import {
+  BadRequestError,
+  NotFoundError,
+  PermissionBoundaryError,
+  RateLimitError,
+  UnauthorizedError
+} from "@app/lib/errors";
 import { checkIPAgainstBlocklist, extractIPDetails, isValidIpOrCidr, TIp } from "@app/lib/ip";
+import { logger } from "@app/lib/logger";
 
 import { ActorType, AuthTokenType } from "../auth/auth-type";
 import { TIdentityOrgDALFactory } from "../identity/identity-org-dal";
@@ -40,7 +47,10 @@ type TIdentityUaServiceFactoryDep = {
   identityOrgMembershipDAL: TIdentityOrgDALFactory;
   permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
   licenseService: Pick<TLicenseServiceFactory, "getPlan">;
-  keyStore: Pick<TKeyStoreFactory, "setItemWithExpiry" | "getItem" | "deleteItem" | "getKeysByPattern" | "deleteItems">;
+  keyStore: Pick<
+    TKeyStoreFactory,
+    "setItemWithExpiry" | "getItem" | "deleteItem" | "getKeysByPattern" | "deleteItems" | "acquireLock"
+  >;
 };
 
 export type TIdentityUaServiceFactory = ReturnType<typeof identityUaServiceFactory>;
@@ -74,17 +84,21 @@ export const identityUaServiceFactory = ({
 
     const LOCKOUT_KEY = `lockout:identity:${identityUa.identityId}:${IdentityAuthMethod.UNIVERSAL_AUTH}:${clientId}`;
 
-    const identityMembershipOrg = await identityOrgMembershipDAL.findOne({ identityId: identityUa.identityId });
-    if (!identityMembershipOrg) {
-      throw new UnauthorizedError({
-        message: "Invalid credentials"
+    let lock: Awaited<ReturnType<typeof keyStore.acquireLock>>;
+    try {
+      lock = await keyStore.acquireLock([KeyStorePrefixes.IdentityLockoutLock(LOCKOUT_KEY)], 500, {
+        retryCount: 3,
+        retryDelay: 300,
+        retryJitter: 100
       });
+    } catch (e) {
+      logger.info(
+        `identity login failed to acquire lock [identityId=${identityUa.identityId}] [authMethod=${IdentityAuthMethod.UNIVERSAL_AUTH}]`
+      );
+      throw new RateLimitError({ message: "Rate limit exceeded" });
     }
 
-    const identityTx = await identityUaDAL.transaction(async (tx) => {
-      await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.IdentityLogin(identityUa.identityId, clientId)]);
-
-      // Lockout Check
+    try {
       const lockoutRaw = await keyStore.getItem(LOCKOUT_KEY);
 
       let lockout: LockoutObject | undefined;
@@ -98,6 +112,13 @@ export const identityUaServiceFactory = ({
         });
       }
 
+      const identityMembershipOrg = await identityOrgMembershipDAL.findOne({ identityId: identityUa.identityId });
+      if (!identityMembershipOrg) {
+        throw new UnauthorizedError({
+          message: "Invalid credentials"
+        });
+      }
+
       const clientSecretPrefix = clientSecret.slice(0, 4);
       const clientSecretInfo = await identityUaClientSecretDAL.find({
         identityUAId: identityUa.id,
@@ -159,7 +180,7 @@ export const identityUaServiceFactory = ({
         }
       }
 
-      if (clientSecretNumUsesLimit > 0 && clientSecretNumUses === clientSecretNumUsesLimit) {
+      if (clientSecretNumUsesLimit > 0 && clientSecretNumUses >= clientSecretNumUsesLimit) {
         // number of times client secret can be used for
         // a login operation reached
         await identityUaClientSecretDAL.updateById(validClientSecretInfo.id, {
@@ -183,57 +204,61 @@ export const identityUaServiceFactory = ({
               accessTokenMaxTTL: 1000000000
             };
 
-      const uaClientSecretDoc = await identityUaClientSecretDAL.incrementUsage(validClientSecretInfo.id, tx);
-      await identityOrgMembershipDAL.updateById(
-        identityMembershipOrg.id,
-        {
-          lastLoginAuthMethod: IdentityAuthMethod.UNIVERSAL_AUTH,
-          lastLoginTime: new Date()
-        },
-        tx
-      );
-      const newToken = await identityAccessTokenDAL.create(
+      const identityAccessToken = await identityUaDAL.transaction(async (tx) => {
+        const uaClientSecretDoc = await identityUaClientSecretDAL.incrementUsage(validClientSecretInfo!.id, tx);
+        await identityOrgMembershipDAL.updateById(
+          identityMembershipOrg.id,
+          {
+            lastLoginAuthMethod: IdentityAuthMethod.UNIVERSAL_AUTH,
+            lastLoginTime: new Date()
+          },
+          tx
+        );
+        const newToken = await identityAccessTokenDAL.create(
+          {
+            identityId: identityUa.identityId,
+            isAccessTokenRevoked: false,
+            identityUAClientSecretId: uaClientSecretDoc.id,
+            accessTokenNumUses: 0,
+            accessTokenNumUsesLimit: identityUa.accessTokenNumUsesLimit,
+            accessTokenPeriod: identityUa.accessTokenPeriod,
+            authMethod: IdentityAuthMethod.UNIVERSAL_AUTH,
+            ...accessTokenTTLParams
+          },
+          tx
+        );
+
+        return newToken;
+      });
+
+      const appCfg = getConfig();
+      const accessToken = crypto.jwt().sign(
         {
           identityId: identityUa.identityId,
-          isAccessTokenRevoked: false,
-          identityUAClientSecretId: uaClientSecretDoc.id,
-          accessTokenNumUses: 0,
-          accessTokenNumUsesLimit: identityUa.accessTokenNumUsesLimit,
-          accessTokenPeriod: identityUa.accessTokenPeriod,
-          authMethod: IdentityAuthMethod.UNIVERSAL_AUTH,
-          ...accessTokenTTLParams
-        },
-        tx
+          clientSecretId: validClientSecretInfo.id,
+          identityAccessTokenId: identityAccessToken.id,
+          authTokenType: AuthTokenType.IDENTITY_ACCESS_TOKEN
+        } as TIdentityAccessTokenJwtPayload,
+        appCfg.AUTH_SECRET,
+        // akhilmhdh: for non-expiry tokens you should not even set the value, including undefined. Even for undefined jsonwebtoken throws error
+        Number(identityAccessToken.accessTokenTTL) === 0
+          ? undefined
+          : {
+              expiresIn: Number(identityAccessToken.accessTokenTTL)
+            }
       );
 
-      return { newToken, validClientSecretInfo, accessTokenTTLParams };
-    });
-
-    const appCfg = getConfig();
-    const accessToken = crypto.jwt().sign(
-      {
-        identityId: identityUa.identityId,
-        clientSecretId: identityTx.validClientSecretInfo.id,
-        identityAccessTokenId: identityTx.newToken.id,
-        authTokenType: AuthTokenType.IDENTITY_ACCESS_TOKEN
-      } as TIdentityAccessTokenJwtPayload,
-      appCfg.AUTH_SECRET,
-      // akhilmhdh: for non-expiry tokens you should not even set the value, including undefined. Even for undefined jsonwebtoken throws error
-      Number(identityTx.newToken.accessTokenTTL) === 0
-        ? undefined
-        : {
-            expiresIn: Number(identityTx.newToken.accessTokenTTL)
-          }
-    );
-
-    return {
-      accessToken,
-      identityUa,
-      validClientSecretInfo: identityTx.validClientSecretInfo,
-      identityAccessToken: identityTx.newToken,
-      identityMembershipOrg,
-      ...identityTx.accessTokenTTLParams
-    };
+      return {
+        accessToken,
+        identityUa,
+        validClientSecretInfo,
+        identityAccessToken,
+        identityMembershipOrg,
+        ...accessTokenTTLParams
+      };
+    } finally {
+      await lock.release();
+    }
   };
 
   const attachUniversalAuth = async ({

backend/src/services/integration-auth/integration-delete-secret.ts

@@ -39,7 +39,7 @@ const getIntegrationSecretsV2 = async (
   },
   secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find" | "findByFolderId">,
   folderDAL: Pick<TSecretFolderDALFactory, "findByManySecretPath">,
-  secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">
+  secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds" | "findByIds">
 ) => {
   const content: Record<string, boolean> = {};
   if (dto.depth > MAX_SYNC_SECRET_DEPTH) {
@@ -300,7 +300,7 @@ export const deleteIntegrationSecrets = async ({
   projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
   secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find" | "findByFolderId">;
   folderDAL: Pick<TSecretFolderDALFactory, "findByManySecretPath" | "findBySecretPath">;
-  secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
+  secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds" | "findByIds">;
   secretDAL: Pick<TSecretDALFactory, "findByFolderId">;
   kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
 }) => {

backend/src/services/integration/integration-service.ts

@@ -40,7 +40,7 @@ type TIntegrationServiceFactoryDep = {
   projectBotService: TProjectBotServiceFactory;
   secretQueueService: Pick<TSecretQueueFactory, "syncIntegrations">;
   secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find" | "findByFolderId">;
-  secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
+  secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds" | "findByIds">;
   kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
   secretDAL: Pick<TSecretDALFactory, "findByFolderId">;
 };

backend/src/services/offline-usage-report/offline-usage-report-dal.ts

@@ -0,0 +1,208 @@
+import { TDbClient } from "@app/db";
+import { ProjectType, TableName } from "@app/db/schemas";
+
+export type TOfflineUsageReportDALFactory = ReturnType<typeof offlineUsageReportDALFactory>;
+
+export const offlineUsageReportDALFactory = (db: TDbClient) => {
+  const getUserMetrics = async () => {
+    // Get total users and admin users
+    const userMetrics = (await db
+      .from(TableName.Users)
+      .select(
+        db.raw(
+          `
+          COUNT(*) as total_users,
+          COUNT(CASE WHEN "superAdmin" = true THEN 1 END) as admin_users
+        `
+        )
+      )
+      .where({ isGhost: false })
+      .first()) as { total_users: string; admin_users: string } | undefined;
+
+    // Get users by auth method
+    const authMethodStats = (await db
+      .from(TableName.Users)
+      .select(
+        db.raw(`
+          unnest("authMethods") as auth_method,
+          COUNT(*) as count
+        `)
+      )
+      .where({ isGhost: false })
+      .whereNotNull("authMethods")
+      .groupBy(db.raw('unnest("authMethods")'))) as Array<{ auth_method: string; count: string }>;
+
+    const usersByAuthMethod = authMethodStats.reduce(
+      (acc: Record<string, number>, row: { auth_method: string; count: string }) => {
+        acc[row.auth_method] = parseInt(row.count, 10);
+        return acc;
+      },
+      {} as Record<string, number>
+    );
+
+    return {
+      totalUsers: parseInt(userMetrics?.total_users || "0", 10),
+      adminUsers: parseInt(userMetrics?.admin_users || "0", 10),
+      usersByAuthMethod
+    };
+  };
+
+  const getMachineIdentityMetrics = async () => {
+    // Get total machine identities
+    const identityMetrics = (await db
+      .from(TableName.Identity)
+      .select(
+        db.raw(
+          `
+          COUNT(*) as total_identities
+        `
+        )
+      )
+      .first()) as { total_identities: string } | undefined;
+
+    // Get identities by auth method
+    const authMethodStats = (await db
+      .from(TableName.Identity)
+      .select("authMethod")
+      .count("* as count")
+      .whereNotNull("authMethod")
+      .groupBy("authMethod")) as Array<{ authMethod: string; count: string }>;
+
+    const machineIdentitiesByAuthMethod = authMethodStats.reduce(
+      (acc: Record<string, number>, row: { authMethod: string; count: string }) => {
+        acc[row.authMethod] = parseInt(row.count, 10);
+        return acc;
+      },
+      {} as Record<string, number>
+    );
+
+    return {
+      totalMachineIdentities: parseInt(identityMetrics?.total_identities || "0", 10),
+      machineIdentitiesByAuthMethod
+    };
+  };
+
+  const getProjectMetrics = async () => {
+    // Get total projects and projects by type
+    const projectMetrics = (await db
+      .from(TableName.Project)
+      .select("type")
+      .count("* as count")
+      .groupBy("type")) as Array<{ type: string; count: string }>;
+
+    const totalProjects = projectMetrics.reduce(
+      (sum, row: { type: string; count: string }) => sum + parseInt(row.count, 10),
+      0
+    );
+    const projectsByType = projectMetrics.reduce(
+      (acc: Record<string, number>, row: { type: string; count: string }) => {
+        acc[row.type] = parseInt(row.count, 10);
+        return acc;
+      },
+      {} as Record<string, number>
+    );
+
+    // Calculate average secrets per project
+    const secretsPerProject = (await db
+      .from(`${TableName.SecretV2} as s`)
+      .select("p.id as projectId")
+      .count("s.id as count")
+      .leftJoin(`${TableName.SecretFolder} as sf`, "s.folderId", "sf.id")
+      .leftJoin(`${TableName.Environment} as e`, "sf.envId", "e.id")
+      .leftJoin(`${TableName.Project} as p`, "e.projectId", "p.id")
+      .where("p.type", ProjectType.SecretManager)
+      .groupBy("p.id")
+      .whereNotNull("p.id")) as Array<{ projectId: string; count: string }>;
+
+    const averageSecretsPerProject =
+      secretsPerProject.length > 0
+        ? secretsPerProject.reduce(
+            (sum, row: { projectId: string; count: string }) => sum + parseInt(row.count, 10),
+            0
+          ) / secretsPerProject.length
+        : 0;
+
+    return {
+      totalProjects,
+      projectsByType,
+      averageSecretsPerProject: Math.round(averageSecretsPerProject * 100) / 100
+    };
+  };
+
+  const getSecretMetrics = async () => {
+    // Get total secrets
+    const totalSecretsResult = (await db.from(TableName.SecretV2).count("* as count").first()) as
+      | { count: string }
+      | undefined;
+
+    const totalSecrets = parseInt(totalSecretsResult?.count || "0", 10);
+
+    // Get secrets by project
+    const secretsByProject = (await db
+      .from(`${TableName.SecretV2} as s`)
+      .select("p.id as projectId", "p.name as projectName")
+      .count("s.id as secretCount")
+      .leftJoin(`${TableName.SecretFolder} as sf`, "s.folderId", "sf.id")
+      .leftJoin(`${TableName.Environment} as e`, "sf.envId", "e.id")
+      .leftJoin(`${TableName.Project} as p`, "e.projectId", "p.id")
+      .where("p.type", ProjectType.SecretManager)
+      .groupBy("p.id", "p.name")
+      .whereNotNull("p.id")) as Array<{ projectId: string; projectName: string; secretCount: string }>;
+
+    return {
+      totalSecrets,
+      secretsByProject: secretsByProject.map(
+        (row: { projectId: string; projectName: string; secretCount: string }) => ({
+          projectId: row.projectId,
+          projectName: row.projectName,
+          secretCount: parseInt(row.secretCount, 10)
+        })
+      )
+    };
+  };
+
+  const getSecretSyncMetrics = async () => {
+    const totalSecretSyncsResult = (await db.from(TableName.SecretSync).count("* as count").first()) as
+      | { count: string }
+      | undefined;
+
+    return {
+      totalSecretSyncs: parseInt(totalSecretSyncsResult?.count || "0", 10)
+    };
+  };
+
+  const getDynamicSecretMetrics = async () => {
+    const totalDynamicSecretsResult = (await db.from(TableName.DynamicSecret).count("* as count").first()) as
+      | { count: string }
+      | undefined;
+
+    return {
+      totalDynamicSecrets: parseInt(totalDynamicSecretsResult?.count || "0", 10)
+    };
+  };
+
+  const getSecretRotationMetrics = async () => {
+    // Check both v1 and v2 secret rotation tables
+    const [v1RotationsResult, v2RotationsResult] = await Promise.all([
+      db.from(TableName.SecretRotation).count("* as count").first() as Promise<{ count: string } | undefined>,
+      db.from(TableName.SecretRotationV2).count("* as count").first() as Promise<{ count: string } | undefined>
+    ]);
+
+    const totalV1Rotations = parseInt(v1RotationsResult?.count || "0", 10);
+    const totalV2Rotations = parseInt(v2RotationsResult?.count || "0", 10);
+
+    return {
+      totalSecretRotations: totalV1Rotations + totalV2Rotations
+    };
+  };
+
+  return {
+    getUserMetrics,
+    getMachineIdentityMetrics,
+    getProjectMetrics,
+    getSecretMetrics,
+    getSecretSyncMetrics,
+    getDynamicSecretMetrics,
+    getSecretRotationMetrics
+  };
+};

backend/src/services/offline-usage-report/offline-usage-report-service.ts

@@ -0,0 +1,133 @@
+import crypto from "crypto";
+
+import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
+import { getConfig } from "@app/lib/config/env";
+import { BadRequestError } from "@app/lib/errors";
+
+import { TOfflineUsageReportDALFactory } from "./offline-usage-report-dal";
+
+type TOfflineUsageReportServiceFactoryDep = {
+  offlineUsageReportDAL: TOfflineUsageReportDALFactory;
+  licenseService: Pick<TLicenseServiceFactory, "getCustomerId" | "getLicenseId">;
+};
+
+export type TOfflineUsageReportServiceFactory = ReturnType<typeof offlineUsageReportServiceFactory>;
+
+export const offlineUsageReportServiceFactory = ({
+  offlineUsageReportDAL,
+  licenseService
+}: TOfflineUsageReportServiceFactoryDep) => {
+  const signReportContent = (content: string, licenseId: string): string => {
+    const contentHash = crypto.createHash("sha256").update(content).digest("hex");
+    const hmac = crypto.createHmac("sha256", licenseId);
+    hmac.update(contentHash);
+    return hmac.digest("hex");
+  };
+
+  const verifyReportContent = (content: string, signature: string, licenseId: string): boolean => {
+    const expectedSignature = signReportContent(content, licenseId);
+    return signature === expectedSignature;
+  };
+
+  const generateUsageReportCSV = async () => {
+    const cfg = getConfig();
+    if (!cfg.LICENSE_KEY_OFFLINE) {
+      throw new BadRequestError({
+        message: "Offline usage reports are not enabled. LICENSE_KEY_OFFLINE must be configured."
+      });
+    }
+
+    const customerId = licenseService.getCustomerId() as string;
+    const licenseId = licenseService.getLicenseId();
+
+    const [
+      userMetrics,
+      machineIdentityMetrics,
+      projectMetrics,
+      secretMetrics,
+      secretSyncMetrics,
+      dynamicSecretMetrics,
+      secretRotationMetrics
+    ] = await Promise.all([
+      offlineUsageReportDAL.getUserMetrics(),
+      offlineUsageReportDAL.getMachineIdentityMetrics(),
+      offlineUsageReportDAL.getProjectMetrics(),
+      offlineUsageReportDAL.getSecretMetrics(),
+      offlineUsageReportDAL.getSecretSyncMetrics(),
+      offlineUsageReportDAL.getDynamicSecretMetrics(),
+      offlineUsageReportDAL.getSecretRotationMetrics()
+    ]);
+
+    const headers = [
+      "Total Users",
+      "Admin Users",
+      "Total Identities",
+      "Total Projects",
+      "Total Secrets",
+      "Total Secret Syncs",
+      "Total Dynamic Secrets",
+      "Total Secret Rotations",
+      "Avg Secrets Per Project"
+    ];
+
+    const allUserAuthMethods = Object.keys(userMetrics.usersByAuthMethod);
+    allUserAuthMethods.forEach((method) => {
+      headers.push(`Users Auth ${method}`);
+    });
+
+    const allIdentityAuthMethods = Object.keys(machineIdentityMetrics.machineIdentitiesByAuthMethod);
+    allIdentityAuthMethods.forEach((method) => {
+      headers.push(`Identities Auth ${method}`);
+    });
+
+    const allProjectTypes = Object.keys(projectMetrics.projectsByType);
+    allProjectTypes.forEach((type) => {
+      headers.push(`Projects ${type}`);
+    });
+
+    headers.push("Signature");
+
+    const dataRow: (string | number)[] = [
+      userMetrics.totalUsers,
+      userMetrics.adminUsers,
+      machineIdentityMetrics.totalMachineIdentities,
+      projectMetrics.totalProjects,
+      secretMetrics.totalSecrets,
+      secretSyncMetrics.totalSecretSyncs,
+      dynamicSecretMetrics.totalDynamicSecrets,
+      secretRotationMetrics.totalSecretRotations,
+      projectMetrics.averageSecretsPerProject
+    ];
+
+    allUserAuthMethods.forEach((method) => {
+      dataRow.push(userMetrics.usersByAuthMethod[method] || 0);
+    });
+    allIdentityAuthMethods.forEach((method) => {
+      dataRow.push(machineIdentityMetrics.machineIdentitiesByAuthMethod[method] || 0);
+    });
+
+    allProjectTypes.forEach((type) => {
+      dataRow.push(projectMetrics.projectsByType[type] || 0);
+    });
+
+    const headersWithoutSignature = headers.slice(0, -1);
+    const contentWithoutSignature = [headersWithoutSignature.join(","), dataRow.join(",")].join("\n");
+
+    const signature = signReportContent(contentWithoutSignature, licenseId);
+    dataRow.push(signature);
+
+    const csvContent = [headers.join(","), dataRow.join(",")].join("\n");
+
+    return {
+      csvContent,
+      signature,
+      filename: `infisical-usage-report-${customerId}-${new Date().toISOString().split("T")[0]}.csv`
+    };
+  };
+
+  return {
+    generateUsageReportCSV,
+    verifyReportSignature: (csvContent: string, signature: string, licenseId: string) =>
+      verifyReportContent(csvContent, signature, licenseId)
+  };
+};

backend/src/services/offline-usage-report/offline-usage-report-types.ts

@@ -0,0 +1,42 @@
+export interface TUsageMetrics {
+  // User metrics
+  totalUsers: number;
+  usersByAuthMethod: Record<string, number>;
+  adminUsers: number;
+
+  // Machine identity metrics
+  totalMachineIdentities: number;
+  machineIdentitiesByAuthMethod: Record<string, number>;
+
+  // Project metrics
+  totalProjects: number;
+  projectsByType: Record<string, number>;
+  averageSecretsPerProject: number;
+
+  // Secret metrics
+  totalSecrets: number;
+  totalSecretSyncs: number;
+  totalDynamicSecrets: number;
+  totalSecretRotations: number;
+}
+
+export interface TUsageReportMetadata {
+  generatedAt: string;
+  instanceId: string;
+  reportVersion: string;
+}
+
+export interface TUsageReport {
+  metadata: TUsageReportMetadata;
+  metrics: TUsageMetrics;
+  signature?: string;
+}
+
+export interface TGenerateUsageReportDTO {
+  includeSignature?: boolean;
+}
+
+export interface TVerifyUsageReportDTO {
+  reportData: string;
+  signature: string;
+}

backend/src/services/org-membership/org-membership-dal.ts

@@ -153,10 +153,64 @@ export const orgMembershipDALFactory = (db: TDbClient) => {
     }
   };
 
+  const findOrgMembershipsWithUsersByOrgId = async (orgId: string) => {
+    try {
+      const members = await db
+        .replicaNode()(TableName.OrgMembership)
+        .where(`${TableName.OrgMembership}.orgId`, orgId)
+        .join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`)
+        .leftJoin<TUserEncryptionKeys>(
+          TableName.UserEncryptionKey,
+          `${TableName.UserEncryptionKey}.userId`,
+          `${TableName.Users}.id`
+        )
+        .leftJoin(TableName.IdentityMetadata, (queryBuilder) => {
+          void queryBuilder
+            .on(`${TableName.OrgMembership}.userId`, `${TableName.IdentityMetadata}.userId`)
+            .andOn(`${TableName.OrgMembership}.orgId`, `${TableName.IdentityMetadata}.orgId`);
+        })
+        .select(
+          db.ref("id").withSchema(TableName.OrgMembership),
+          db.ref("inviteEmail").withSchema(TableName.OrgMembership),
+          db.ref("orgId").withSchema(TableName.OrgMembership),
+          db.ref("role").withSchema(TableName.OrgMembership),
+          db.ref("roleId").withSchema(TableName.OrgMembership),
+          db.ref("status").withSchema(TableName.OrgMembership),
+          db.ref("isActive").withSchema(TableName.OrgMembership),
+          db.ref("email").withSchema(TableName.Users),
+          db.ref("username").withSchema(TableName.Users),
+          db.ref("firstName").withSchema(TableName.Users),
+          db.ref("lastName").withSchema(TableName.Users),
+          db.ref("isEmailVerified").withSchema(TableName.Users),
+          db.ref("id").withSchema(TableName.Users).as("userId")
+        )
+        .where({ isGhost: false });
+
+      return members.map((member) => ({
+        id: member.id,
+        orgId: member.orgId,
+        role: member.role,
+        status: member.status,
+        isActive: member.isActive,
+        inviteEmail: member.inviteEmail,
+        user: {
+          id: member.userId,
+          email: member.email,
+          username: member.username,
+          firstName: member.firstName,
+          lastName: member.lastName
+        }
+      }));
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find org memberships with users by org id" });
+    }
+  };
+
   return {
     ...orgMembershipOrm,
     findOrgMembershipById,
     findRecentInvitedMemberships,
-    updateLastInvitedAtByIds
+    updateLastInvitedAtByIds,
+    findOrgMembershipsWithUsersByOrgId
   };
 };

backend/src/services/org/org-dal.ts

@@ -83,6 +83,7 @@ export const orgDALFactory = (db: TDbClient) => {
         .select(db.ref("id").withSchema(TableName.OrgMembership).as("orgMembershipId"))
         .select(db.ref("role").withSchema(TableName.OrgMembership).as("orgMembershipRole"))
         .select(db.ref("roleId").withSchema(TableName.OrgMembership).as("orgMembershipRoleId"))
+        .select(db.ref("status").withSchema(TableName.OrgMembership).as("orgMembershipStatus"))
         .select(db.ref("name").withSchema(TableName.OrgRoles).as("orgMembershipRoleName"));
 
       const formattedDocs = sqlNestRelationships({
@@ -112,7 +113,8 @@ export const orgDALFactory = (db: TDbClient) => {
               orgMembershipId,
               orgMembershipRole,
               orgMembershipRoleName,
-              orgMembershipRoleId
+              orgMembershipRoleId,
+              orgMembershipStatus
             }) => ({
               user: {
                 id: userId,
@@ -121,6 +123,7 @@ export const orgDALFactory = (db: TDbClient) => {
                 firstName,
                 lastName
               },
+              status: orgMembershipStatus,
               membershipId: orgMembershipId,
               role: orgMembershipRoleName || orgMembershipRole, // custom role name or pre-defined role name
               roleId: orgMembershipRoleId
@@ -488,6 +491,15 @@ export const orgDALFactory = (db: TDbClient) => {
     }
   };
 
+  const bulkCreateMemberships = async (data: TOrgMembershipsInsert[], tx?: Knex) => {
+    try {
+      const memberships = await (tx || db)(TableName.OrgMembership).insert(data).returning("*");
+      return memberships;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Create org memberships" });
+    }
+  };
+
   const updateMembershipById = async (id: string, data: TOrgMembershipsUpdate, tx?: Knex) => {
     try {
       const [membership] = await (tx || db)(TableName.OrgMembership).where({ id }).update(data).returning("*");
@@ -668,6 +680,7 @@ export const orgDALFactory = (db: TDbClient) => {
     findMembership,
     findMembershipWithScimFilter,
     createMembership,
+    bulkCreateMemberships,
     updateMembershipById,
     deleteMembershipById,
     deleteMembershipsById,

backend/src/services/org/org-service.ts

@@ -528,15 +528,18 @@ export const orgServiceFactory = ({
   /*
    * Create organization
    * */
-  const createOrganization = async ({
-    userId,
-    userEmail,
-    orgName
-  }: {
-    userId: string;
-    orgName: string;
-    userEmail?: string | null;
-  }) => {
+  const createOrganization = async (
+    {
+      userId,
+      userEmail,
+      orgName
+    }: {
+      userId?: string;
+      orgName: string;
+      userEmail?: string | null;
+    },
+    trx?: Knex
+  ) => {
     const { privateKey, publicKey } = await crypto.encryption().asymmetric().generateKeyPair();
     const key = crypto.randomBytes(32).toString("base64");
     const {
@@ -555,22 +558,25 @@ export const orgServiceFactory = ({
     } = crypto.encryption().symmetric().encryptWithRootEncryptionKey(key);
 
     const customerId = await licenseService.generateOrgCustomerId(orgName, userEmail);
-    const organization = await orgDAL.transaction(async (tx) => {
+
+    const createOrg = async (tx: Knex) => {
       // akhilmhdh: for now this is auto created. in future we can input from user and for previous users just modifiy
       const org = await orgDAL.create(
         { name: orgName, customerId, slug: slugify(`${orgName}-${alphaNumericNanoId(4)}`) },
         tx
       );
-      await orgDAL.createMembership(
-        {
-          userId,
-          orgId: org.id,
-          role: OrgMembershipRole.Admin,
-          status: OrgMembershipStatus.Accepted,
-          isActive: true
-        },
-        tx
-      );
+      if (userId) {
+        await orgDAL.createMembership(
+          {
+            userId,
+            orgId: org.id,
+            role: OrgMembershipRole.Admin,
+            status: OrgMembershipStatus.Accepted,
+            isActive: true
+          },
+          tx
+        );
+      }
       await orgBotDAL.create(
         {
           name: org.name,
@@ -590,7 +596,9 @@ export const orgServiceFactory = ({
         tx
       );
       return org;
-    });
+    };
+
+    const organization = await (trx ? createOrg(trx) : orgDAL.transaction(createOrg));
 
     await licenseService.updateSubscriptionOrgMemberCount(organization.id);
     return organization;

backend/src/services/secret-folder/secret-folder-service.ts

@@ -30,6 +30,7 @@ import {
   TDeleteFolderDTO,
   TDeleteManyFoldersDTO,
   TGetFolderByIdDTO,
+  TGetFolderByPathDTO,
   TGetFolderDTO,
   TGetFoldersDeepByEnvsDTO,
   TUpdateFolderDTO,
@@ -1398,6 +1399,31 @@ export const secretFolderServiceFactory = ({
     };
   };
 
+  const getFolderByPath = async (
+    { projectId, environment, secretPath }: TGetFolderByPathDTO,
+    actor: OrgServiceActor
+  ) => {
+    // folder check is allowed to be read by anyone
+    // permission is to check if user has access
+    await permissionService.getProjectPermission({
+      actor: actor.type,
+      actorId: actor.id,
+      projectId,
+      actorAuthMethod: actor.authMethod,
+      actorOrgId: actor.orgId,
+      actionProjectType: ActionProjectType.SecretManager
+    });
+
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
+
+    if (!folder)
+      throw new NotFoundError({
+        message: `Could not find folder with path "${secretPath}" in environment "${environment}" for project with ID "${projectId}"`
+      });
+
+    return folder;
+  };
+
   return {
     createFolder,
     updateFolder,
@@ -1405,6 +1431,7 @@ export const secretFolderServiceFactory = ({
     deleteFolder,
     getFolders,
     getFolderById,
+    getFolderByPath,
     getProjectFolderCount,
     getFoldersMultiEnv,
     getFoldersDeepByEnvs,

backend/src/services/secret-folder/secret-folder-types.ts

@@ -91,3 +91,9 @@ export type TDeleteManyFoldersDTO = {
     idOrName: string;
   }>;
 };
+
+export type TGetFolderByPathDTO = {
+  projectId: string;
+  environment: string;
+  secretPath: string;
+};

backend/src/services/secret-import/secret-import-dal.ts

@@ -127,6 +127,27 @@ export const secretImportDALFactory = (db: TDbClient) => {
     }
   };
 
+  const findByIds = async (ids: string[], tx?: Knex) => {
+    try {
+      const docs = await (tx || db.replicaNode())(TableName.SecretImport)
+        .whereIn(`${TableName.SecretImport}.id`, ids)
+        .join(TableName.Environment, `${TableName.SecretImport}.importEnv`, `${TableName.Environment}.id`)
+        .select(
+          db.ref("*").withSchema(TableName.SecretImport) as unknown as keyof TSecretImports,
+          db.ref("slug").withSchema(TableName.Environment),
+          db.ref("name").withSchema(TableName.Environment),
+          db.ref("id").withSchema(TableName.Environment).as("envId")
+        );
+
+      return docs.map(({ envId, slug, name, ...el }) => ({
+        ...el,
+        importEnv: { id: envId, slug, name }
+      }));
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find secret imports by ids" });
+    }
+  };
+
   const getProjectImportCount = async (
     { search, ...filter }: Partial<TSecretImports & { projectId: string; search?: string }>,
     tx?: Knex
@@ -325,6 +346,7 @@ export const secretImportDALFactory = (db: TDbClient) => {
     ...secretImportOrm,
     find,
     findById,
+    findByIds,
     findByFolderIds,
     findLastImportPosition,
     updateAllPosition,

backend/src/services/secret-import/secret-import-fns.ts

@@ -1,3 +1,5 @@
+import RE2 from "re2";
+
 import { SecretType, TSecretImports, TSecrets, TSecretsV2 } from "@app/db/schemas";
 import { groupBy, unique } from "@app/lib/fn";
 
@@ -54,6 +56,74 @@ type TSecretImportSecretsV2 = {
 
 const LEVEL_BREAK = 10;
 const getImportUniqKey = (envSlug: string, path: string) => `${envSlug}=${path}`;
+const RESERVED_IMPORT_REGEX = new RE2("/__reserve_replication_([a-f0-9-]{36})");
+
+/**
+ * Processes reserved imports by resolving them to their replication source.
+ */
+const processReservedImports = async <
+  T extends {
+    isReserved?: boolean | null;
+    importPath: string;
+    importEnv: { id: string; slug: string; name: string };
+    folderId: string;
+  }
+>(
+  imports: T[],
+  secretImportDAL: Pick<TSecretImportDALFactory, "findByIds">
+): Promise<T[]> => {
+  const reservedImportIds: string[] = [];
+
+  imports.forEach((secretImport) => {
+    if (secretImport.isReserved) {
+      const reservedMatch = RESERVED_IMPORT_REGEX.exec(secretImport.importPath);
+      if (reservedMatch) {
+        const referencedImportId = reservedMatch[1];
+        reservedImportIds.push(referencedImportId);
+      }
+    }
+  });
+
+  if (reservedImportIds.length === 0) {
+    return imports;
+  }
+
+  try {
+    const importDetailsMap = new Map<
+      string,
+      { importPath: string; importEnv: { id: string; slug: string; name: string } }
+    >();
+
+    const referencedImports = await secretImportDAL.findByIds(reservedImportIds);
+    referencedImports.forEach((referencedImport) => {
+      importDetailsMap.set(referencedImport.id, {
+        importPath: referencedImport.importPath,
+        importEnv: referencedImport.importEnv
+      });
+    });
+
+    return imports.map((secretImport) => {
+      if (secretImport.isReserved) {
+        const reservedMatch = RESERVED_IMPORT_REGEX.exec(secretImport.importPath);
+        if (reservedMatch) {
+          const referencedImportId = reservedMatch[1];
+          const referencedDetails = importDetailsMap.get(referencedImportId);
+
+          if (referencedDetails) {
+            return {
+              ...secretImport,
+              importPath: referencedDetails.importPath,
+              importEnv: referencedDetails.importEnv
+            };
+          }
+        }
+      }
+      return secretImport;
+    });
+  } catch (error) {
+    return imports;
+  }
+};
 export const fnSecretsFromImports = async ({
   allowedImports: possibleCyclicImports,
   folderDAL,
@@ -167,7 +237,7 @@ export const fnSecretsV2FromImports = async ({
   folderDAL: Pick<TSecretFolderDALFactory, "findByManySecretPath">;
   viewSecretValue: boolean;
   secretDAL: Pick<TSecretV2BridgeDALFactory, "find">;
-  secretImportDAL: Pick<TSecretImportDALFactory, "findByFolderIds">;
+  secretImportDAL: Pick<TSecretImportDALFactory, "findByFolderIds" | "findByIds">;
   decryptor: (value?: Buffer | null) => string;
   expandSecretReferences?: (inputSecret: {
     value?: string;
@@ -188,6 +258,10 @@ export const fnSecretsV2FromImports = async ({
     })[];
   }[] = [{ secretImports: rootSecretImports, depth: 0, parentImportedSecrets: [] }];
 
+  const processedSecretImports = await processReservedImports(rootSecretImports, secretImportDAL);
+
+  stack[0] = { secretImports: processedSecretImports, depth: 0, parentImportedSecrets: [] };
+
   const processedImports: TSecretImportSecretsV2[] = [];
 
   while (stack.length) {

backend/src/services/secret-sync/aws-parameter-store/aws-parameter-store-sync-fns.ts

@@ -12,7 +12,7 @@ type TAWSParameterStoreRecord = Record<string, AWS.SSM.Parameter>;
 type TAWSParameterStoreMetadataRecord = Record<string, AWS.SSM.ParameterMetadata>;
 type TAWSParameterStoreTagsRecord = Record<string, Record<string, string>>;
 
-const MAX_RETRIES = 5;
+const MAX_RETRIES = 10;
 const BATCH_SIZE = 10;
 
 const getSSM = async (secretSync: TAwsParameterStoreSyncWithCredentials) => {

backend/src/services/secret-sync/aws-secrets-manager/aws-secrets-manager-sync-fns.ts

@@ -38,7 +38,7 @@ type TAwsSecretsRecord = Record<string, SecretListEntry>;
 type TAwsSecretValuesRecord = Record<string, SecretValueEntry>;
 type TAwsSecretDescriptionsRecord = Record<string, DescribeSecretResponse>;
 
-const MAX_RETRIES = 5;
+const MAX_RETRIES = 10;
 const BATCH_SIZE = 20;
 
 const getSecretsManagerClient = async (secretSync: TAwsSecretsManagerSyncWithCredentials) => {

backend/src/services/secret-sync/hc-vault/hc-vault-sync-fns.ts

@@ -1,9 +1,13 @@
 import { isAxiosError } from "axios";
 
-import { request } from "@app/lib/config/request";
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { removeTrailingSlash } from "@app/lib/fn";
-import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
-import { getHCVaultAccessToken, getHCVaultInstanceUrl } from "@app/services/app-connection/hc-vault";
+import {
+  getHCVaultAccessToken,
+  getHCVaultInstanceUrl,
+  requestWithHCVaultGateway,
+  THCVaultConnection
+} from "@app/services/app-connection/hc-vault";
 import {
   THCVaultListVariables,
   THCVaultListVariablesResponse,
@@ -14,19 +18,20 @@ import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
 import { matchesSchema } from "@app/services/secret-sync/secret-sync-fns";
 import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
 
-const listHCVaultVariables = async ({ instanceUrl, namespace, mount, accessToken, path }: THCVaultListVariables) => {
-  await blockLocalAndPrivateIpAddresses(instanceUrl);
-
+const listHCVaultVariables = async (
+  { instanceUrl, namespace, mount, accessToken, path }: THCVaultListVariables,
+  connection: THCVaultConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
   try {
-    const { data } = await request.get<THCVaultListVariablesResponse>(
-      `${instanceUrl}/v1/${removeTrailingSlash(mount)}/data/${path}`,
-      {
-        headers: {
-          "X-Vault-Token": accessToken,
-          ...(namespace ? { "X-Vault-Namespace": namespace } : {})
-        }
+    const { data } = await requestWithHCVaultGateway<THCVaultListVariablesResponse>(connection, gatewayService, {
+      url: `${instanceUrl}/v1/${removeTrailingSlash(mount)}/data/${path}`,
+      method: "GET",
+      headers: {
+        "X-Vault-Token": accessToken,
+        ...(namespace ? { "X-Vault-Namespace": namespace } : {})
       }
-    );
+    });
 
     return data.data.data;
   } catch (error: unknown) {
@@ -39,33 +44,29 @@ const listHCVaultVariables = async ({ instanceUrl, namespace, mount, accessToken
 };
 
 // Hashicorp Vault updates all variables in one batch. This is to respect their versioning
-const updateHCVaultVariables = async ({
-  path,
-  instanceUrl,
-  namespace,
-  accessToken,
-  mount,
-  data
-}: TPostHCVaultVariable) => {
-  await blockLocalAndPrivateIpAddresses(instanceUrl);
-
-  return request.post(
-    `${instanceUrl}/v1/${removeTrailingSlash(mount)}/data/${path}`,
-    {
-      data
+const updateHCVaultVariables = async (
+  { path, instanceUrl, namespace, accessToken, mount, data }: TPostHCVaultVariable,
+  connection: THCVaultConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
+  return requestWithHCVaultGateway(connection, gatewayService, {
+    url: `${instanceUrl}/v1/${removeTrailingSlash(mount)}/data/${path}`,
+    method: "POST",
+    headers: {
+      "X-Vault-Token": accessToken,
+      ...(namespace ? { "X-Vault-Namespace": namespace } : {}),
+      "Content-Type": "application/json"
     },
-    {
-      headers: {
-        "X-Vault-Token": accessToken,
-        ...(namespace ? { "X-Vault-Namespace": namespace } : {}),
-        "Content-Type": "application/json"
-      }
-    }
-  );
+    data: { data }
+  });
 };
 
 export const HCVaultSyncFns = {
-  syncSecrets: async (secretSync: THCVaultSyncWithCredentials, secretMap: TSecretMap) => {
+  syncSecrets: async (
+    secretSync: THCVaultSyncWithCredentials,
+    secretMap: TSecretMap,
+    gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+  ) => {
     const {
       connection,
       environment,
@@ -74,16 +75,20 @@ export const HCVaultSyncFns = {
     } = secretSync;
 
     const { namespace } = connection.credentials;
-    const accessToken = await getHCVaultAccessToken(connection);
+    const accessToken = await getHCVaultAccessToken(connection, gatewayService);
     const instanceUrl = await getHCVaultInstanceUrl(connection);
 
-    const variables = await listHCVaultVariables({
-      instanceUrl,
-      accessToken,
-      namespace,
-      mount,
-      path
-    });
+    const variables = await listHCVaultVariables(
+      {
+        instanceUrl,
+        accessToken,
+        namespace,
+        mount,
+        path
+      },
+      connection,
+      gatewayService
+    );
     let tainted = false;
 
     for (const entry of Object.entries(secretMap)) {
@@ -110,24 +115,36 @@ export const HCVaultSyncFns = {
     if (!tainted) return;
 
     try {
-      await updateHCVaultVariables({ accessToken, instanceUrl, namespace, mount, path, data: variables });
+      await updateHCVaultVariables(
+        { accessToken, instanceUrl, namespace, mount, path, data: variables },
+        connection,
+        gatewayService
+      );
     } catch (error) {
       throw new SecretSyncError({
         error
       });
     }
   },
-  removeSecrets: async (secretSync: THCVaultSyncWithCredentials, secretMap: TSecretMap) => {
+  removeSecrets: async (
+    secretSync: THCVaultSyncWithCredentials,
+    secretMap: TSecretMap,
+    gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+  ) => {
     const {
       connection,
       destinationConfig: { mount, path }
     } = secretSync;
 
     const { namespace } = connection.credentials;
-    const accessToken = await getHCVaultAccessToken(connection);
+    const accessToken = await getHCVaultAccessToken(connection, gatewayService);
     const instanceUrl = await getHCVaultInstanceUrl(connection);
 
-    const variables = await listHCVaultVariables({ instanceUrl, namespace, accessToken, mount, path });
+    const variables = await listHCVaultVariables(
+      { instanceUrl, namespace, accessToken, mount, path },
+      connection,
+      gatewayService
+    );
 
     for await (const [key] of Object.entries(variables)) {
       if (key in secretMap) {
@@ -136,30 +153,41 @@ export const HCVaultSyncFns = {
     }
 
     try {
-      await updateHCVaultVariables({ accessToken, instanceUrl, namespace, mount, path, data: variables });
+      await updateHCVaultVariables(
+        { accessToken, instanceUrl, namespace, mount, path, data: variables },
+        connection,
+        gatewayService
+      );
     } catch (error) {
       throw new SecretSyncError({
         error
       });
     }
   },
-  getSecrets: async (secretSync: THCVaultSyncWithCredentials) => {
+  getSecrets: async (
+    secretSync: THCVaultSyncWithCredentials,
+    gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+  ) => {
     const {
       connection,
       destinationConfig: { mount, path }
     } = secretSync;
 
     const { namespace } = connection.credentials;
-    const accessToken = await getHCVaultAccessToken(connection);
+    const accessToken = await getHCVaultAccessToken(connection, gatewayService);
     const instanceUrl = await getHCVaultInstanceUrl(connection);
 
-    const variables = await listHCVaultVariables({
-      instanceUrl,
-      namespace,
-      accessToken,
-      mount,
-      path
-    });
+    const variables = await listHCVaultVariables(
+      {
+        instanceUrl,
+        namespace,
+        accessToken,
+        mount,
+        path
+      },
+      connection,
+      gatewayService
+    );
 
     return Object.fromEntries(Object.entries(variables).map(([key, value]) => [key, { value }]));
   }

backend/src/services/secret-sync/secret-sync-fns.ts

@@ -244,7 +244,7 @@ export const SecretSyncFns = {
       case SecretSync.Windmill:
         return WindmillSyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.HCVault:
-        return HCVaultSyncFns.syncSecrets(secretSync, schemaSecretMap);
+        return HCVaultSyncFns.syncSecrets(secretSync, schemaSecretMap, gatewayService);
       case SecretSync.TeamCity:
         return TeamCitySyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.OCIVault:
@@ -283,7 +283,7 @@ export const SecretSyncFns = {
   },
   getSecrets: async (
     secretSync: TSecretSyncWithCredentials,
-    { kmsService, appConnectionDAL }: TSyncSecretDeps
+    { kmsService, appConnectionDAL, gatewayService }: TSyncSecretDeps
   ): Promise<TSecretMap> => {
     let secretMap: TSecretMap;
     switch (secretSync.destination) {
@@ -341,7 +341,7 @@ export const SecretSyncFns = {
         secretMap = await WindmillSyncFns.getSecrets(secretSync);
         break;
       case SecretSync.HCVault:
-        secretMap = await HCVaultSyncFns.getSecrets(secretSync);
+        secretMap = await HCVaultSyncFns.getSecrets(secretSync, gatewayService);
         break;
       case SecretSync.TeamCity:
         secretMap = await TeamCitySyncFns.getSecrets(secretSync);
@@ -451,7 +451,7 @@ export const SecretSyncFns = {
       case SecretSync.Windmill:
         return WindmillSyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.HCVault:
-        return HCVaultSyncFns.removeSecrets(secretSync, schemaSecretMap);
+        return HCVaultSyncFns.removeSecrets(secretSync, schemaSecretMap, gatewayService);
       case SecretSync.TeamCity:
         return TeamCitySyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.OCIVault:

backend/src/services/secret-sync/secret-sync-queue.ts

@@ -80,8 +80,8 @@ type TSecretSyncQueueFactoryDep = {
     | "deleteMany"
     | "invalidateSecretCacheByProjectId"
   >;
-  secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
-  secretSyncDAL: Pick<TSecretSyncDALFactory, "findById" | "find" | "updateById" | "deleteById">;
+  secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds" | "findByIds">;
+  secretSyncDAL: Pick<TSecretSyncDALFactory, "findById" | "find" | "updateById" | "deleteById" | "update">;
   auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
   projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
   projectDAL: TProjectDALFactory;
@@ -104,17 +104,15 @@ type SecretSyncActionJob = Job<
   TQueueSecretSyncSyncSecretsByIdDTO | TQueueSecretSyncImportSecretsByIdDTO | TQueueSecretSyncRemoveSecretsByIdDTO
 >;
 
+const JITTER_MS = 10 * 1000;
+const REQUEUE_MS = 30 * 1000;
+const REQUEUE_LIMIT = 30;
+const CONNECTION_CONCURRENCY_LIMIT = 3;
+
 const getRequeueDelay = (failureCount?: number) => {
-  if (!failureCount) return 0;
-
-  const baseDelay = 1000;
-  const maxDelay = 30000;
-
-  const delay = Math.min(baseDelay * 2 ** failureCount, maxDelay);
-
-  const jitter = delay * (0.5 + Math.random() * 0.5);
-
-  return jitter;
+  const jitter = Math.random() * JITTER_MS;
+  if (!failureCount) return jitter;
+  return REQUEUE_MS + jitter;
 };
 
 export const secretSyncQueueFactory = ({
@@ -193,6 +191,46 @@ export const secretSyncQueueFactory = ({
     folderCommitService
   });
 
+  const $isConnectionConcurrencyLimitReached = async (connectionId: string) => {
+    const concurrencyCount = await keyStore.getItem(KeyStorePrefixes.AppConnectionConcurrentJobs(connectionId));
+
+    if (!concurrencyCount) return false;
+
+    const count = Number.parseInt(concurrencyCount, 10);
+
+    if (Number.isNaN(count)) return false;
+
+    return count >= CONNECTION_CONCURRENCY_LIMIT;
+  };
+
+  const $incrementConnectionConcurrencyCount = async (connectionId: string) => {
+    const concurrencyCount = await keyStore.getItem(KeyStorePrefixes.AppConnectionConcurrentJobs(connectionId));
+
+    const currentCount = Number.parseInt(concurrencyCount || "0", 10);
+
+    const incrementedCount = Number.isNaN(currentCount) ? 1 : currentCount + 1;
+
+    await keyStore.setItemWithExpiry(
+      KeyStorePrefixes.AppConnectionConcurrentJobs(connectionId),
+      (REQUEUE_MS * REQUEUE_LIMIT) / 1000, // in seconds
+      incrementedCount
+    );
+  };
+
+  const $decrementConnectionConcurrencyCount = async (connectionId: string) => {
+    const concurrencyCount = await keyStore.getItem(KeyStorePrefixes.AppConnectionConcurrentJobs(connectionId));
+
+    const currentCount = Number.parseInt(concurrencyCount || "0", 10);
+
+    const decrementedCount = Math.max(0, Number.isNaN(currentCount) ? 0 : currentCount - 1);
+
+    await keyStore.setItemWithExpiry(
+      KeyStorePrefixes.AppConnectionConcurrentJobs(connectionId),
+      (REQUEUE_MS * REQUEUE_LIMIT) / 1000, // in seconds
+      decrementedCount
+    );
+  };
+
   const $getInfisicalSecrets = async (
     secretSync: TSecretSyncRaw | TSecretSyncWithCredentials,
     includeImports = true
@@ -416,15 +454,11 @@ export const secretSyncQueueFactory = ({
     return importedSecretMap;
   };
 
-  const $handleSyncSecretsJob = async (job: TSecretSyncSyncSecretsDTO) => {
+  const $handleSyncSecretsJob = async (job: TSecretSyncSyncSecretsDTO, secretSync: TSecretSyncRaw) => {
     const {
       data: { syncId, auditLogInfo }
     } = job;
 
-    const secretSync = await secretSyncDAL.findById(syncId);
-
-    if (!secretSync) throw new Error(`Cannot find secret sync with ID ${syncId}`);
-
     await enterpriseSyncCheck(
       licenseService,
       secretSync.destination as SecretSync,
@@ -566,15 +600,11 @@ export const secretSyncQueueFactory = ({
     logger.info("SecretSync Sync Job with ID %s Completed", job.id);
   };
 
-  const $handleImportSecretsJob = async (job: TSecretSyncImportSecretsDTO) => {
+  const $handleImportSecretsJob = async (job: TSecretSyncImportSecretsDTO, secretSync: TSecretSyncRaw) => {
     const {
       data: { syncId, auditLogInfo, importBehavior }
     } = job;
 
-    const secretSync = await secretSyncDAL.findById(syncId);
-
-    if (!secretSync) throw new Error(`Cannot find secret sync with ID ${syncId}`);
-
     await secretSyncDAL.updateById(syncId, {
       importStatus: SecretSyncStatus.Running
     });
@@ -683,15 +713,11 @@ export const secretSyncQueueFactory = ({
     logger.info("SecretSync Import Job with ID %s Completed", job.id);
   };
 
-  const $handleRemoveSecretsJob = async (job: TSecretSyncRemoveSecretsDTO) => {
+  const $handleRemoveSecretsJob = async (job: TSecretSyncRemoveSecretsDTO, secretSync: TSecretSyncRaw) => {
     const {
       data: { syncId, auditLogInfo, deleteSyncOnComplete }
     } = job;
 
-    const secretSync = await secretSyncDAL.findById(syncId);
-
-    if (!secretSync) throw new Error(`Cannot find secret sync with ID ${syncId}`);
-
     await enterpriseSyncCheck(
       licenseService,
       secretSync.destination as SecretSync,
@@ -894,6 +920,17 @@ export const secretSyncQueueFactory = ({
 
     const secretSyncs = await secretSyncDAL.find({ folderId: folder.id, isAutoSyncEnabled: true });
 
+    await secretSyncDAL.update(
+      {
+        $in: {
+          id: secretSyncs.map((sync) => sync.id)
+        }
+      },
+      {
+        syncStatus: SecretSyncStatus.Pending
+      }
+    );
+
     await Promise.all(secretSyncs.map((secretSync) => queueSecretSyncSyncSecretsById({ syncId: secretSync.id })));
   };
 
@@ -904,7 +941,7 @@ export const secretSyncQueueFactory = ({
       case QueueJobs.SecretSyncSyncSecrets: {
         const { failedToAcquireLockCount = 0, ...rest } = job.data as TQueueSecretSyncSyncSecretsByIdDTO;
 
-        if (failedToAcquireLockCount < 10) {
+        if (failedToAcquireLockCount < REQUEUE_LIMIT) {
           await queueSecretSyncSyncSecretsById({ ...rest, failedToAcquireLockCount: failedToAcquireLockCount + 1 });
           return;
         }
@@ -974,6 +1011,26 @@ export const secretSyncQueueFactory = ({
       | TQueueSecretSyncImportSecretsByIdDTO
       | TQueueSecretSyncRemoveSecretsByIdDTO;
 
+    const secretSync = await secretSyncDAL.findById(syncId);
+
+    if (!secretSync) throw new Error(`Cannot find secret sync with ID ${syncId}`);
+
+    const { connectionId } = secretSync;
+
+    if (job.name === QueueJobs.SecretSyncSyncSecrets) {
+      const isConcurrentLimitReached = await $isConnectionConcurrencyLimitReached(connectionId);
+
+      if (isConcurrentLimitReached) {
+        logger.info(
+          `SecretSync Concurrency limit reached [syncId=${syncId}] [job=${job.name}] [connectionId=${connectionId}]`
+        );
+
+        await $handleAcquireLockFailure(job as SecretSyncActionJob);
+
+        return;
+      }
+    }
+
     let lock: Awaited<ReturnType<typeof keyStore.acquireLock>>;
 
     try {
@@ -993,20 +1050,26 @@ export const secretSyncQueueFactory = ({
 
     try {
       switch (job.name) {
-        case QueueJobs.SecretSyncSyncSecrets:
-          await $handleSyncSecretsJob(job as TSecretSyncSyncSecretsDTO);
+        case QueueJobs.SecretSyncSyncSecrets: {
+          await $incrementConnectionConcurrencyCount(connectionId);
+          await $handleSyncSecretsJob(job as TSecretSyncSyncSecretsDTO, secretSync);
           break;
+        }
         case QueueJobs.SecretSyncImportSecrets:
-          await $handleImportSecretsJob(job as TSecretSyncImportSecretsDTO);
+          await $handleImportSecretsJob(job as TSecretSyncImportSecretsDTO, secretSync);
           break;
         case QueueJobs.SecretSyncRemoveSecrets:
-          await $handleRemoveSecretsJob(job as TSecretSyncRemoveSecretsDTO);
+          await $handleRemoveSecretsJob(job as TSecretSyncRemoveSecretsDTO, secretSync);
           break;
         default:
           // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
           throw new Error(`Unhandled Secret Sync Job ${job.name}`);
       }
     } finally {
+      if (job.name === QueueJobs.SecretSyncSyncSecrets) {
+        await $decrementConnectionConcurrencyCount(connectionId);
+      }
+
       await lock.release();
     }
   });

backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts

@@ -108,7 +108,7 @@ type TSecretV2BridgeServiceFactoryDep = {
     | "findBySecretPathMultiEnv"
     | "findSecretPathByFolderIds"
   >;
-  secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
+  secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds" | "findByIds">;
   secretQueueService: Pick<TSecretQueueFactory, "syncSecrets" | "handleSecretReminder" | "removeSecretReminder">;
   secretApprovalPolicyService: Pick<TSecretApprovalPolicyServiceFactory, "getSecretApprovalPolicy">;
   secretApprovalRequestDAL: Pick<TSecretApprovalRequestDALFactory, "create" | "transaction">;

backend/src/services/secret/secret-queue.ts

@@ -86,7 +86,7 @@ type TSecretQueueFactoryDep = {
   integrationAuthService: Pick<TIntegrationAuthServiceFactory, "getIntegrationAccessToken">;
   folderDAL: TSecretFolderDALFactory;
   secretDAL: TSecretDALFactory;
-  secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
+  secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds" | "findByIds">;
   webhookDAL: Pick<TWebhookDALFactory, "findAllWebhooks" | "transaction" | "update" | "bulkUpdate">;
   projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne" | "find">;
   projectDAL: TProjectDALFactory;

backend/src/services/smtp/emails/OrganizationAssignmentTemplate.tsx

@@ -0,0 +1,69 @@
+import { Heading, Section, Text } from "@react-email/components";
+import React from "react";
+
+import { BaseButton } from "./BaseButton";
+import { BaseEmailWrapper, BaseEmailWrapperProps } from "./BaseEmailWrapper";
+import { BaseLink } from "./BaseLink";
+
+interface OrganizationAssignmentTemplateProps extends Omit<BaseEmailWrapperProps, "preview" | "title"> {
+  inviterFirstName?: string;
+  inviterUsername?: string;
+  organizationName: string;
+  callback_url: string;
+}
+
+export const OrganizationAssignmentTemplate = ({
+  organizationName,
+  inviterFirstName,
+  inviterUsername,
+  callback_url,
+  siteUrl
+}: OrganizationAssignmentTemplateProps) => {
+  return (
+    <BaseEmailWrapper
+      title="New Organization"
+      preview="You've been added to a new organization on Infisical."
+      siteUrl={siteUrl}
+    >
+      <Heading className="text-black text-[18px] leading-[28px] text-center font-normal p-0 mx-0">
+        You've been added to the organization
+        <br />
+        <strong>{organizationName}</strong> on <strong>Infisical</strong>
+      </Heading>
+      <Section className="px-[24px] mb-[28px] mt-[36px] pt-[12px] pb-[8px] border text-center border-solid border-gray-200 rounded-md bg-gray-50">
+        <Text className="text-black text-[14px] leading-[24px]">
+          {inviterFirstName && inviterUsername ? (
+            <>
+              <strong>{inviterFirstName}</strong> (
+              <BaseLink href={`mailto:${inviterUsername}`}>{inviterUsername}</BaseLink>) has added you as an
+              organization admin to <strong>{organizationName}</strong>.
+            </>
+          ) : (
+            <>
+              An instance admin has added you as an organization admin to <strong>{organizationName}</strong>.
+            </>
+          )}
+        </Text>
+      </Section>
+      <Section className="text-center">
+        <BaseButton href={callback_url}>View Dashboard</BaseButton>
+      </Section>
+      <Section className="mt-[24px] bg-gray-50 pt-[2px] pb-[16px] border border-solid border-gray-200 px-[24px] rounded-md text-gray-800">
+        <Text className="mb-[0px]">
+          <strong>About Infisical:</strong> Infisical is an all-in-one platform to securely manage application secrets,
+          certificates, SSH keys, and configurations across your team and infrastructure.
+        </Text>
+      </Section>
+    </BaseEmailWrapper>
+  );
+};
+
+export default OrganizationAssignmentTemplate;
+
+OrganizationAssignmentTemplate.PreviewProps = {
+  organizationName: "Example Organization",
+  inviterFirstName: "Jane",
+  inviterUsername: "[email protected]",
+  siteUrl: "https://infisical.com",
+  callback_url: "https://app.infisical.com"
+} as OrganizationAssignmentTemplateProps;

backend/src/services/smtp/emails/index.ts

@@ -9,6 +9,7 @@ export * from "./IntegrationSyncFailedTemplate";
 export * from "./NewDeviceLoginTemplate";
 export * from "./OrgAdminBreakglassAccessTemplate";
 export * from "./OrgAdminProjectGrantAccessTemplate";
+export * from "./OrganizationAssignmentTemplate";
 export * from "./OrganizationInvitationTemplate";
 export * from "./PasswordResetTemplate";
 export * from "./PasswordSetupTemplate";

backend/src/services/smtp/smtp-service.ts

@@ -18,6 +18,7 @@ import {
   NewDeviceLoginTemplate,
   OrgAdminBreakglassAccessTemplate,
   OrgAdminProjectGrantAccessTemplate,
+  OrganizationAssignmentTemplate,
   OrganizationInvitationTemplate,
   PasswordResetTemplate,
   PasswordSetupTemplate,
@@ -61,6 +62,7 @@ export enum SmtpTemplates {
   // HistoricalSecretList = "historicalSecretLeakIncident", not used anymore?
   NewDeviceJoin = "newDevice",
   OrgInvite = "organizationInvitation",
+  OrgAssignment = "organizationAssignment",
   ResetPassword = "passwordReset",
   SetupPassword = "passwordSetup",
   SecretLeakIncident = "secretLeakIncident",
@@ -94,6 +96,7 @@ export enum SmtpHost {
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 const EmailTemplateMap: Record<SmtpTemplates, React.FC<any>> = {
   [SmtpTemplates.OrgInvite]: OrganizationInvitationTemplate,
+  [SmtpTemplates.OrgAssignment]: OrganizationAssignmentTemplate,
   [SmtpTemplates.NewDeviceJoin]: NewDeviceLoginTemplate,
   [SmtpTemplates.SignupEmailVerification]: SignupEmailVerificationTemplate,
   [SmtpTemplates.EmailMfa]: EmailMfaTemplate,

backend/src/services/super-admin/super-admin-service.ts

@@ -1,6 +1,13 @@
 import { CronJob } from "cron";
 
-import { IdentityAuthMethod, OrgMembershipRole, TSuperAdmin, TSuperAdminUpdate } from "@app/db/schemas";
+import {
+  IdentityAuthMethod,
+  OrgMembershipRole,
+  OrgMembershipStatus,
+  TSuperAdmin,
+  TSuperAdminUpdate,
+  TUsers
+} from "@app/db/schemas";
 import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
 import { PgSqlLock, TKeyStoreFactory } from "@app/keystore/keystore";
 import {
@@ -13,7 +20,12 @@ import {
 import { crypto } from "@app/lib/crypto/cryptography";
 import { BadRequestError, NotFoundError } from "@app/lib/errors";
 import { logger } from "@app/lib/logger";
+import { OrgServiceActor } from "@app/lib/types";
+import { isDisposableEmail } from "@app/lib/validator";
+import { TAuthTokenServiceFactory } from "@app/services/auth-token/auth-token-service";
+import { TokenType } from "@app/services/auth-token/auth-token-types";
 import { TIdentityDALFactory } from "@app/services/identity/identity-dal";
+import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
 
 import { TAuthLoginFactory } from "../auth/auth-login-service";
 import { ActorType, AuthMethod, AuthTokenType } from "../auth/auth-type";
@@ -43,7 +55,9 @@ import {
   TAdminGetUsersDTO,
   TAdminIntegrationConfig,
   TAdminSignUpDTO,
-  TGetOrganizationsDTO
+  TCreateOrganizationDTO,
+  TGetOrganizationsDTO,
+  TResendOrgInviteDTO
 } from "./super-admin-types";
 
 type TSuperAdminServiceFactoryDep = {
@@ -59,11 +73,13 @@ type TSuperAdminServiceFactoryDep = {
   authService: Pick<TAuthLoginFactory, "generateUserTokens">;
   kmsService: Pick<TKmsServiceFactory, "encryptWithRootKey" | "decryptWithRootKey" | "updateEncryptionStrategy">;
   kmsRootConfigDAL: TKmsRootConfigDALFactory;
-  orgService: Pick<TOrgServiceFactory, "createOrganization">;
+  orgService: Pick<TOrgServiceFactory, "createOrganization" | "inviteUserToOrganization">;
   keyStore: Pick<TKeyStoreFactory, "getItem" | "setItemWithExpiry" | "deleteItem" | "deleteItems">;
-  licenseService: Pick<TLicenseServiceFactory, "onPremFeatures">;
+  licenseService: Pick<TLicenseServiceFactory, "onPremFeatures" | "updateSubscriptionOrgMemberCount">;
   microsoftTeamsService: Pick<TMicrosoftTeamsServiceFactory, "initializeTeamsBot">;
   invalidateCacheQueue: TInvalidateCacheQueueFactory;
+  smtpService: Pick<TSmtpService, "sendMail">;
+  tokenService: TAuthTokenServiceFactory;
 };
 
 export type TSuperAdminServiceFactory = ReturnType<typeof superAdminServiceFactory>;
@@ -123,7 +139,9 @@ export const superAdminServiceFactory = ({
   identityTokenAuthDAL,
   identityOrgMembershipDAL,
   microsoftTeamsService,
-  invalidateCacheQueue
+  invalidateCacheQueue,
+  smtpService,
+  tokenService
 }: TSuperAdminServiceFactoryDep) => {
   const initServerCfg = async () => {
     // TODO(akhilmhdh): bad  pattern time less change this later to me itself
@@ -732,6 +750,159 @@ export const superAdminServiceFactory = ({
     return organizations;
   };
 
+  const createOrganization = async (
+    { name, inviteAdminEmails: emails }: TCreateOrganizationDTO,
+    actor: OrgServiceActor
+  ) => {
+    const appCfg = getConfig();
+
+    const inviteAdminEmails = [...new Set(emails)];
+
+    if (!appCfg.isDevelopmentMode && appCfg.isCloud)
+      throw new BadRequestError({ message: "This endpoint is not supported for cloud instances" });
+
+    const serverAdmin = await userDAL.findById(actor.id);
+    const plan = licenseService.onPremFeatures;
+
+    const isEmailInvalid = await isDisposableEmail(inviteAdminEmails);
+    if (isEmailInvalid) {
+      throw new BadRequestError({
+        message: "Disposable emails are not allowed",
+        name: "InviteUser"
+      });
+    }
+
+    const { organization, users: usersToEmail } = await orgDAL.transaction(async (tx) => {
+      const org = await orgService.createOrganization(
+        {
+          orgName: name,
+          userEmail: serverAdmin?.email ?? serverAdmin?.username // identities can be server admins so we can't require this
+        },
+        tx
+      );
+
+      const users: Pick<TUsers, "id" | "firstName" | "lastName" | "email" | "username" | "isAccepted">[] = [];
+
+      for await (const inviteeEmail of inviteAdminEmails) {
+        const usersByUsername = await userDAL.findUserByUsername(inviteeEmail, tx);
+        let inviteeUser =
+          usersByUsername?.length > 1
+            ? usersByUsername.find((el) => el.username === inviteeEmail)
+            : usersByUsername?.[0];
+
+        // if the user doesn't exist we create the user with the email
+        if (!inviteeUser) {
+          // TODO(carlos): will be removed once the function receives usernames instead of emails
+          const usersByEmail = await userDAL.findUserByEmail(inviteeEmail, tx);
+          if (usersByEmail?.length === 1) {
+            [inviteeUser] = usersByEmail;
+          } else {
+            inviteeUser = await userDAL.create(
+              {
+                isAccepted: false,
+                email: inviteeEmail,
+                username: inviteeEmail,
+                authMethods: [AuthMethod.EMAIL],
+                isGhost: false
+              },
+              tx
+            );
+          }
+        }
+
+        const inviteeUserId = inviteeUser?.id;
+        const existingEncryptionKey = await userDAL.findUserEncKeyByUserId(inviteeUserId, tx);
+
+        // when user is missing the encrytion keys
+        // this could happen either if user doesn't exist or user didn't find step 3 of generating the encryption keys of srp
+        // So what we do is we generate a random secure password and then encrypt it with a random pub-private key
+        // Then when user sign in (as login is not possible as isAccepted is false) we rencrypt the private key with the user password
+        if (!inviteeUser || (inviteeUser && !inviteeUser?.isAccepted && !existingEncryptionKey)) {
+          await userDAL.createUserEncryption(
+            {
+              userId: inviteeUserId,
+              encryptionVersion: 2
+            },
+            tx
+          );
+        }
+
+        if (plan?.slug !== "enterprise" && plan?.identityLimit && plan.identitiesUsed >= plan.identityLimit) {
+          // limit imposed on number of identities allowed / number of identities used exceeds the number of identities allowed
+          throw new BadRequestError({
+            name: "InviteUser",
+            message: "Failed to invite member due to member limit reached. Upgrade plan to invite more members."
+          });
+        }
+
+        await orgDAL.createMembership(
+          {
+            userId: inviteeUser.id,
+            inviteEmail: inviteeEmail,
+            orgId: org.id,
+            role: OrgMembershipRole.Admin,
+            status: inviteeUser.isAccepted ? OrgMembershipStatus.Accepted : OrgMembershipStatus.Invited,
+            isActive: true
+          },
+          tx
+        );
+
+        users.push(inviteeUser);
+      }
+
+      return { organization: org, users };
+    });
+
+    await licenseService.updateSubscriptionOrgMemberCount(organization.id);
+
+    await Promise.allSettled(
+      usersToEmail.map(async (user) => {
+        if (!user.email) return;
+
+        if (user.isAccepted) {
+          return smtpService.sendMail({
+            template: SmtpTemplates.OrgAssignment,
+            subjectLine: "You've been added to an Infisical organization",
+            recipients: [user.email],
+            substitutions: {
+              inviterFirstName: serverAdmin?.firstName,
+              inviterUsername: serverAdmin?.email,
+              organizationName: organization.name,
+              email: user.email,
+              organizationId: organization.id,
+              callback_url: `${appCfg.SITE_URL}/login?org_id=${organization.id}`
+            }
+          });
+        }
+
+        // new user, send regular invite
+
+        const token = await tokenService.createTokenForUser({
+          type: TokenType.TOKEN_EMAIL_ORG_INVITATION,
+          userId: user.id,
+          orgId: organization.id
+        });
+
+        return smtpService.sendMail({
+          template: SmtpTemplates.OrgInvite,
+          subjectLine: "Infisical organization invitation",
+          recipients: [user.email],
+          substitutions: {
+            inviterFirstName: serverAdmin?.firstName,
+            inviterUsername: serverAdmin?.email,
+            organizationName: organization.name,
+            email: user.email,
+            organizationId: organization.id,
+            token,
+            callback_url: `${appCfg.SITE_URL}/signupinvite`
+          }
+        });
+      })
+    );
+
+    return organization;
+  };
+
   const deleteOrganization = async (organizationId: string) => {
     const organization = await orgDAL.deleteById(organizationId);
     return organization;
@@ -763,6 +934,86 @@ export const superAdminServiceFactory = ({
     return organizationMembership;
   };
 
+  const joinOrganization = async (orgId: string, actor: OrgServiceActor) => {
+    const serverAdmin = await userDAL.findById(actor.id);
+
+    if (!serverAdmin) {
+      throw new NotFoundError({ message: "Could not find server admin user" });
+    }
+
+    const org = await orgDAL.findById(orgId);
+
+    if (!org) {
+      throw new NotFoundError({ message: `Could not organization with ID "${orgId}"` });
+    }
+
+    const existingOrgMembership = await orgMembershipDAL.findOne({ userId: serverAdmin.id, orgId });
+
+    if (existingOrgMembership) {
+      throw new BadRequestError({ message: `You are already a part of the organization with ID ${orgId}` });
+    }
+
+    const orgMembership = await orgDAL.createMembership({
+      userId: serverAdmin.id,
+      orgId: org.id,
+      role: OrgMembershipRole.Admin,
+      status: OrgMembershipStatus.Accepted,
+      isActive: true
+    });
+
+    return orgMembership;
+  };
+
+  const resendOrgInvite = async ({ organizationId, membershipId }: TResendOrgInviteDTO, actor: OrgServiceActor) => {
+    const orgMembership = await orgMembershipDAL.findOne({ id: membershipId, orgId: organizationId });
+
+    if (!orgMembership) {
+      throw new NotFoundError({ name: "Organization Membership", message: "Organization membership not found" });
+    }
+
+    if (orgMembership.status === OrgMembershipStatus.Accepted) {
+      throw new BadRequestError({
+        message: "This user has already accepted their invitation."
+      });
+    }
+
+    if (!orgMembership.userId) {
+      throw new NotFoundError({ message: "Cannot find user associated with Org Membership." });
+    }
+
+    if (!orgMembership.inviteEmail) {
+      throw new BadRequestError({ message: "No invite email associated with user." });
+    }
+
+    const org = await orgDAL.findOrgById(orgMembership.orgId);
+
+    const appCfg = getConfig();
+    const serverAdmin = await userDAL.findById(actor.id);
+
+    const token = await tokenService.createTokenForUser({
+      type: TokenType.TOKEN_EMAIL_ORG_INVITATION,
+      userId: orgMembership.userId,
+      orgId: orgMembership.orgId
+    });
+
+    await smtpService.sendMail({
+      template: SmtpTemplates.OrgInvite,
+      subjectLine: "Infisical organization invitation",
+      recipients: [orgMembership.inviteEmail],
+      substitutions: {
+        inviterFirstName: serverAdmin?.firstName,
+        inviterUsername: serverAdmin?.email,
+        organizationName: org?.name,
+        email: orgMembership.inviteEmail,
+        organizationId: orgMembership.orgId,
+        token,
+        callback_url: `${appCfg.SITE_URL}/signupinvite`
+      }
+    });
+
+    return orgMembership;
+  };
+
   const getIdentities = async ({ offset, limit, searchTerm }: TAdminGetIdentitiesDTO) => {
     const identities = await identityDAL.getIdentitiesByFilter({
       limit,
@@ -901,6 +1152,9 @@ export const superAdminServiceFactory = ({
     initializeEnvConfigSync,
     getEnvOverrides,
     getEnvOverridesOrganized,
-    deleteUsers
+    deleteUsers,
+    createOrganization,
+    joinOrganization,
+    resendOrgInvite
   };
 };

backend/src/services/super-admin/super-admin-types.ts

@@ -34,6 +34,16 @@ export type TGetOrganizationsDTO = {
   searchTerm: string;
 };
 
+export type TCreateOrganizationDTO = {
+  name: string;
+  inviteAdminEmails: string[];
+};
+
+export type TResendOrgInviteDTO = {
+  organizationId: string;
+  membershipId: string;
+};
+
 export enum LoginMethod {
   EMAIL = "email",
   GOOGLE = "google",