Merge branch 'main' into ENG-3506-LDAP

backend/src/ee/routes/v1/github-org-sync-router.ts

@@ -126,4 +126,39 @@ export const registerGithubOrgSyncRouter = async (server: FastifyZodProvider) =>
       return { githubOrgSyncConfig };
     }
   });
+
+  server.route({
+    url: "/sync-all-teams",
+    method: "POST",
+    config: {
+      rateLimit: writeLimit
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    schema: {
+      response: {
+        200: z.object({
+          totalUsers: z.number(),
+          errors: z.array(z.string()),
+          createdTeams: z.array(z.string()),
+          updatedTeams: z.array(z.string()),
+          removedMemberships: z.number(),
+          syncDuration: z.number()
+        })
+      }
+    },
+    handler: async (req) => {
+      const result = await server.services.githubOrgSync.syncAllTeams({
+        orgPermission: req.permission
+      });
+
+      return {
+        totalUsers: result.totalUsers,
+        errors: result.errors,
+        createdTeams: result.createdTeams,
+        updatedTeams: result.updatedTeams,
+        removedMemberships: result.removedMemberships,
+        syncDuration: result.syncDuration
+      };
+    }
+  });
 };

backend/src/ee/services/audit-log/audit-log-service.ts

@@ -6,9 +6,9 @@ import { getConfig } from "@app/lib/config/env";
 import { BadRequestError } from "@app/lib/errors";
 import { ActorType } from "@app/services/auth/auth-type";
 
-import { OrgPermissionActions, OrgPermissionSubjects } from "../permission/org-permission";
+import { OrgPermissionAuditLogsActions, OrgPermissionSubjects } from "../permission/org-permission";
 import { TPermissionServiceFactory } from "../permission/permission-service-types";
-import { ProjectPermissionActions, ProjectPermissionSub } from "../permission/project-permission";
+import { ProjectPermissionAuditLogsActions, ProjectPermissionSub } from "../permission/project-permission";
 import { TAuditLogDALFactory } from "./audit-log-dal";
 import { TAuditLogQueueServiceFactory } from "./audit-log-queue";
 import { EventType, TAuditLogServiceFactory } from "./audit-log-types";
@@ -41,7 +41,10 @@ export const auditLogServiceFactory = ({
         actorOrgId,
         actionProjectType: ActionProjectType.Any
       });
-      ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.AuditLogs);
+      ForbiddenError.from(permission).throwUnlessCan(
+        ProjectPermissionAuditLogsActions.Read,
+        ProjectPermissionSub.AuditLogs
+      );
     } else {
       // Organization-wide logs
       const { permission } = await permissionService.getOrgPermission(
@@ -52,7 +55,10 @@ export const auditLogServiceFactory = ({
         actorOrgId
       );
 
-      ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.AuditLogs);
+      ForbiddenError.from(permission).throwUnlessCan(
+        OrgPermissionAuditLogsActions.Read,
+        OrgPermissionSubjects.AuditLogs
+      );
     }
 
     // If project ID is not provided, then we need to return all the audit logs for the organization itself.

backend/src/ee/services/github-org-sync/github-org-sync-service.ts

@@ -1,14 +1,19 @@
+/* eslint-disable @typescript-eslint/return-await */
+/* eslint-disable no-await-in-loop */
 import { ForbiddenError } from "@casl/ability";
 import { Octokit } from "@octokit/core";
 import { paginateGraphql } from "@octokit/plugin-paginate-graphql";
 import { Octokit as OctokitRest } from "@octokit/rest";
+import RE2 from "re2";
 
 import { OrgMembershipRole } from "@app/db/schemas";
 import { BadRequestError, NotFoundError } from "@app/lib/errors";
 import { groupBy } from "@app/lib/fn";
 import { logger } from "@app/lib/logger";
+import { retryWithBackoff } from "@app/lib/retry";
 import { TKmsServiceFactory } from "@app/services/kms/kms-service";
 import { KmsDataKey } from "@app/services/kms/kms-types";
+import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal";
 
 import { TGroupDALFactory } from "../group/group-dal";
 import { TUserGroupMembershipDALFactory } from "../group/user-group-membership-dal";
@@ -16,20 +21,67 @@ import { TLicenseServiceFactory } from "../license/license-service";
 import { OrgPermissionActions, OrgPermissionSubjects } from "../permission/org-permission";
 import { TPermissionServiceFactory } from "../permission/permission-service-types";
 import { TGithubOrgSyncDALFactory } from "./github-org-sync-dal";
-import { TCreateGithubOrgSyncDTO, TDeleteGithubOrgSyncDTO, TUpdateGithubOrgSyncDTO } from "./github-org-sync-types";
+import {
+  TCreateGithubOrgSyncDTO,
+  TDeleteGithubOrgSyncDTO,
+  TSyncAllTeamsDTO,
+  TSyncResult,
+  TUpdateGithubOrgSyncDTO,
+  TValidateGithubTokenDTO
+} from "./github-org-sync-types";
 
 const OctokitWithPlugin = Octokit.plugin(paginateGraphql);
 
+// Type definitions for GitHub API errors
+interface GitHubApiError extends Error {
+  status?: number;
+  response?: {
+    status?: number;
+    headers?: {
+      "x-ratelimit-reset"?: string;
+    };
+  };
+}
+
+interface OrgMembershipWithUser {
+  id: string;
+  orgId: string;
+  role: string;
+  status: string;
+  isActive: boolean;
+  inviteEmail: string | null;
+  user: {
+    id: string;
+    email: string;
+    username: string | null;
+    firstName: string | null;
+    lastName: string | null;
+  } | null;
+}
+
+interface GroupMembership {
+  id: string;
+  groupId: string;
+  groupName: string;
+  orgMembershipId: string;
+  firstName: string | null;
+  lastName: string | null;
+}
+
 type TGithubOrgSyncServiceFactoryDep = {
   githubOrgSyncDAL: TGithubOrgSyncDALFactory;
   permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
   kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
   userGroupMembershipDAL: Pick<
     TUserGroupMembershipDALFactory,
-    "findGroupMembershipsByUserIdInOrg" | "insertMany" | "delete"
+    "findGroupMembershipsByUserIdInOrg" | "findGroupMembershipsByGroupIdInOrg" | "insertMany" | "delete"
   >;
   groupDAL: Pick<TGroupDALFactory, "insertMany" | "transaction" | "find">;
   licenseService: Pick<TLicenseServiceFactory, "getPlan">;
+  orgMembershipDAL: Pick<
+    TOrgMembershipDALFactory,
+    "find" | "findOrgMembershipById" | "findOrgMembershipsWithUsersByOrgId"
+  >;
 };
 
 export type TGithubOrgSyncServiceFactory = ReturnType<typeof githubOrgSyncServiceFactory>;
@@ -40,7 +92,8 @@ export const githubOrgSyncServiceFactory = ({
   kmsService,
   userGroupMembershipDAL,
   groupDAL,
-  licenseService
+  licenseService,
+  orgMembershipDAL
 }: TGithubOrgSyncServiceFactoryDep) => {
   const createGithubOrgSync = async ({
     githubOrgName,
@@ -304,8 +357,8 @@ export const githubOrgSyncServiceFactory = ({
     const removeFromTeams = infisicalUserGroups.filter((el) => !githubUserTeamSet.has(el.groupName));
 
     if (newTeams.length || updateTeams.length || removeFromTeams.length) {
-      await groupDAL.transaction(async (tx) => {
-        if (newTeams.length) {
+      if (newTeams.length) {
+        await groupDAL.transaction(async (tx) => {
           const newGroups = await groupDAL.insertMany(
             newTeams.map((newGroupName) => ({
               name: newGroupName,
@@ -322,9 +375,11 @@ export const githubOrgSyncServiceFactory = ({
             })),
             tx
           );
-        }
+        });
+      }
 
-        if (updateTeams.length) {
+      if (updateTeams.length) {
+        await groupDAL.transaction(async (tx) => {
           await userGroupMembershipDAL.insertMany(
             updateTeams.map((el) => ({
               groupId: githubUserTeamOnInfisicalGroupByName[el][0].id,
@@ -332,16 +387,433 @@ export const githubOrgSyncServiceFactory = ({
             })),
             tx
           );
-        }
+        });
+      }
 
-        if (removeFromTeams.length) {
+      if (removeFromTeams.length) {
+        await groupDAL.transaction(async (tx) => {
           await userGroupMembershipDAL.delete(
             { userId, $in: { groupId: removeFromTeams.map((el) => el.groupId) } },
             tx
           );
-        }
+        });
+      }
+    }
+  };
+
+  const validateGithubToken = async ({ orgPermission, githubOrgAccessToken }: TValidateGithubTokenDTO) => {
+    const { permission } = await permissionService.getOrgPermission(
+      orgPermission.type,
+      orgPermission.id,
+      orgPermission.orgId,
+      orgPermission.authMethod,
+      orgPermission.orgId
+    );
+
+    ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.GithubOrgSync);
+
+    const plan = await licenseService.getPlan(orgPermission.orgId);
+    if (!plan.githubOrgSync) {
+      throw new BadRequestError({
+        message:
+          "Failed to validate GitHub token due to plan restriction. Upgrade plan to use GitHub organization sync."
       });
     }
+
+    const config = await githubOrgSyncDAL.findOne({ orgId: orgPermission.orgId });
+    if (!config) {
+      throw new BadRequestError({ message: "GitHub organization sync is not configured" });
+    }
+
+    try {
+      const testOctokit = new OctokitRest({
+        auth: githubOrgAccessToken,
+        request: {
+          signal: AbortSignal.timeout(10000)
+        }
+      });
+
+      const { data: org } = await testOctokit.rest.orgs.get({
+        org: config.githubOrgName
+      });
+
+      const octokitGraphQL = new OctokitWithPlugin({
+        auth: githubOrgAccessToken,
+        request: {
+          signal: AbortSignal.timeout(10000)
+        }
+      });
+
+      await octokitGraphQL.graphql(`query($org: String!) { organization(login: $org) { id name } }`, {
+        org: config.githubOrgName
+      });
+
+      return {
+        valid: true,
+        organizationInfo: {
+          id: org.id,
+          login: org.login,
+          name: org.name || org.login,
+          publicRepos: org.public_repos,
+          privateRepos: org.owned_private_repos || 0
+        }
+      };
+    } catch (error) {
+      logger.error(error, `GitHub token validation failed for org ${config.githubOrgName}`);
+
+      const gitHubError = error as GitHubApiError;
+      const statusCode = gitHubError.status || gitHubError.response?.status;
+      if (statusCode) {
+        if (statusCode === 401) {
+          throw new BadRequestError({
+            message: "GitHub access token is invalid or expired."
+          });
+        }
+        if (statusCode === 403) {
+          throw new BadRequestError({
+            message:
+              "GitHub access token lacks required permissions. Required: 1) 'read:org' scope for organization teams, 2) Token owner must be an organization member with team visibility access, 3) Organization settings must allow team visibility. Check GitHub token scopes and organization member permissions."
+          });
+        }
+        if (statusCode === 404) {
+          throw new BadRequestError({
+            message: `Organization '${config.githubOrgName}' not found or access token does not have access to it.`
+          });
+        }
+      }
+
+      throw new BadRequestError({
+        message: `GitHub token validation failed: ${(error as Error).message}`
+      });
+    }
+  };
+
+  const syncAllTeams = async ({ orgPermission }: TSyncAllTeamsDTO): Promise<TSyncResult> => {
+    const { permission } = await permissionService.getOrgPermission(
+      orgPermission.type,
+      orgPermission.id,
+      orgPermission.orgId,
+      orgPermission.authMethod,
+      orgPermission.orgId
+    );
+
+    ForbiddenError.from(permission).throwUnlessCan(
+      OrgPermissionActions.Edit,
+      OrgPermissionSubjects.GithubOrgSyncManual
+    );
+
+    const plan = await licenseService.getPlan(orgPermission.orgId);
+    if (!plan.githubOrgSync) {
+      throw new BadRequestError({
+        message:
+          "Failed to sync all GitHub teams due to plan restriction. Upgrade plan to use GitHub organization sync."
+      });
+    }
+
+    const config = await githubOrgSyncDAL.findOne({ orgId: orgPermission.orgId });
+    if (!config || !config?.isActive) {
+      throw new BadRequestError({ message: "GitHub organization sync is not configured or not active" });
+    }
+
+    const { decryptor } = await kmsService.createCipherPairWithDataKey({
+      type: KmsDataKey.Organization,
+      orgId: orgPermission.orgId
+    });
+
+    if (!config.encryptedGithubOrgAccessToken) {
+      throw new BadRequestError({
+        message: "GitHub organization access token is required. Please set a token first."
+      });
+    }
+
+    const orgAccessToken = decryptor({ cipherTextBlob: config.encryptedGithubOrgAccessToken }).toString();
+
+    try {
+      const testOctokit = new OctokitRest({
+        auth: orgAccessToken,
+        request: {
+          signal: AbortSignal.timeout(10000)
+        }
+      });
+
+      await testOctokit.rest.orgs.get({
+        org: config.githubOrgName
+      });
+
+      await testOctokit.rest.users.getAuthenticated();
+    } catch (error) {
+      throw new BadRequestError({
+        message: "Stored GitHub access token is invalid or expired. Please set a new token."
+      });
+    }
+
+    const allMembers = await orgMembershipDAL.findOrgMembershipsWithUsersByOrgId(orgPermission.orgId);
+    const activeMembers = allMembers.filter(
+      (member) => member.status === "accepted" && member.isActive
+    ) as OrgMembershipWithUser[];
+
+    const startTime = Date.now();
+    const syncErrors: string[] = [];
+
+    const octokit = new OctokitWithPlugin({
+      auth: orgAccessToken,
+      request: {
+        signal: AbortSignal.timeout(30000)
+      }
+    });
+
+    const data = await retryWithBackoff(async () => {
+      return octokit.graphql
+        .paginate<{
+          organization: {
+            teams: {
+              totalCount: number;
+              edges: {
+                node: {
+                  name: string;
+                  description: string;
+                  members: {
+                    edges: {
+                      node: {
+                        login: string;
+                      };
+                    }[];
+                  };
+                };
+              }[];
+            };
+          };
+        }>(
+          `
+        query orgTeams($cursor: String, $org: String!) {
+          organization(login: $org) {
+            teams(first: 100, after: $cursor) {
+              totalCount
+              edges {
+                node {
+                  name
+                  description
+                  members(first: 100) {
+                    edges {
+                      node {
+                        login
+                      }
+                    }
+                  }
+                }
+              }
+              pageInfo {
+                hasNextPage
+                endCursor
+              }
+            }
+          }
+        }
+        `,
+          {
+            org: config.githubOrgName
+          }
+        )
+        .catch((err) => {
+          logger.error(err, "GitHub GraphQL error for batched team sync");
+
+          const gitHubError = err as GitHubApiError;
+          const statusCode = gitHubError.status || gitHubError.response?.status;
+          if (statusCode) {
+            if (statusCode === 401) {
+              throw new BadRequestError({
+                message: "GitHub access token is invalid or expired. Please provide a new token."
+              });
+            }
+            if (statusCode === 403) {
+              throw new BadRequestError({
+                message:
+                  "GitHub access token lacks required permissions for organization team sync. Required: 1) 'admin:org' scope, 2) Token owner must be organization owner or have team read permissions, 3) Organization settings must allow team visibility. Check token scopes and user role."
+              });
+            }
+            if (statusCode === 404) {
+              throw new BadRequestError({
+                message: `Organization ${config.githubOrgName} not found or access token does not have sufficient permissions to read it.`
+              });
+            }
+          }
+
+          if ((err as Error)?.message?.includes("Although you appear to have the correct authorization credential")) {
+            throw new BadRequestError({
+              message:
+                "Organization has restricted OAuth app access. Please check that: 1) Your organization has approved the Infisical OAuth application, 2) The token owner has sufficient organization permissions."
+            });
+          }
+          throw new BadRequestError({ message: `GitHub GraphQL query failed: ${(err as Error)?.message}` });
+        });
+    });
+
+    const {
+      organization: { teams }
+    } = data;
+
+    const userTeamMap = new Map<string, string[]>();
+    const allGithubUsernamesInTeams = new Set<string>();
+
+    teams?.edges?.forEach((teamEdge) => {
+      const teamName = teamEdge.node.name.toLowerCase();
+
+      teamEdge.node.members.edges.forEach((memberEdge) => {
+        const username = memberEdge.node.login.toLowerCase();
+        allGithubUsernamesInTeams.add(username);
+
+        if (!userTeamMap.has(username)) {
+          userTeamMap.set(username, []);
+        }
+        userTeamMap.get(username)!.push(teamName);
+      });
+    });
+
+    const allGithubTeamNames = Array.from(new Set(teams?.edges?.map((edge) => edge.node.name.toLowerCase()) || []));
+
+    const existingTeamsOnInfisical = await groupDAL.find({
+      orgId: orgPermission.orgId,
+      $in: { name: allGithubTeamNames }
+    });
+    const existingTeamsMap = groupBy(existingTeamsOnInfisical, (i) => i.name);
+
+    const teamsToCreate = allGithubTeamNames.filter((teamName) => !(teamName in existingTeamsMap));
+    const createdTeams = new Set<string>();
+    const updatedTeams = new Set<string>();
+    const totalRemovedMemberships = 0;
+
+    await groupDAL.transaction(async (tx) => {
+      if (teamsToCreate.length > 0) {
+        const newGroups = await groupDAL.insertMany(
+          teamsToCreate.map((teamName) => ({
+            name: teamName,
+            role: OrgMembershipRole.Member,
+            slug: teamName,
+            orgId: orgPermission.orgId
+          })),
+          tx
+        );
+
+        newGroups.forEach((group) => {
+          if (!existingTeamsMap[group.name]) {
+            existingTeamsMap[group.name] = [];
+          }
+          existingTeamsMap[group.name].push(group);
+          createdTeams.add(group.name);
+        });
+      }
+
+      const allTeams = [...Object.values(existingTeamsMap).flat()];
+
+      for (const team of allTeams) {
+        const teamName = team.name.toLowerCase();
+
+        const currentMemberships = (await userGroupMembershipDAL.findGroupMembershipsByGroupIdInOrg(
+          team.id,
+          orgPermission.orgId
+        )) as GroupMembership[];
+
+        const expectedUserIds = new Set<string>();
+        teams?.edges?.forEach((teamEdge) => {
+          if (teamEdge.node.name.toLowerCase() === teamName) {
+            teamEdge.node.members.edges.forEach((memberEdge) => {
+              const githubUsername = memberEdge.node.login.toLowerCase();
+
+              const matchingMember = activeMembers.find((member) => {
+                const email = member.user?.email || member.inviteEmail;
+                if (!email) return false;
+
+                const emailPrefix = email.split("@")[0].toLowerCase();
+                const emailDomain = email.split("@")[1].toLowerCase();
+
+                if (emailPrefix === githubUsername) {
+                  return true;
+                }
+                const domainName = emailDomain.split(".")[0];
+                if (githubUsername.endsWith(domainName) && githubUsername.length > domainName.length) {
+                  const baseUsername = githubUsername.slice(0, -domainName.length);
+                  if (emailPrefix === baseUsername) {
+                    return true;
+                  }
+                }
+                const emailSplitRegex = new RE2(/[._-]/);
+                const emailParts = emailPrefix.split(emailSplitRegex);
+                const longestEmailPart = emailParts.reduce((a, b) => (a.length > b.length ? a : b), "");
+                if (longestEmailPart.length >= 4 && githubUsername.includes(longestEmailPart)) {
+                  return true;
+                }
+                return false;
+              });
+
+              if (matchingMember?.user?.id) {
+                expectedUserIds.add(matchingMember.user.id);
+                logger.info(
+                  `Matched GitHub user ${githubUsername} to email ${matchingMember.user?.email || matchingMember.inviteEmail}`
+                );
+              }
+            });
+          }
+        });
+
+        const currentUserIds = new Set<string>();
+        currentMemberships.forEach((membership) => {
+          const activeMember = activeMembers.find((am) => am.id === membership.orgMembershipId);
+          if (activeMember?.user?.id) {
+            currentUserIds.add(activeMember.user.id);
+          }
+        });
+
+        const usersToAdd = Array.from(expectedUserIds).filter((userId) => !currentUserIds.has(userId));
+
+        const membershipsToRemove = currentMemberships.filter((membership) => {
+          const activeMember = activeMembers.find((am) => am.id === membership.orgMembershipId);
+          return activeMember?.user?.id && !expectedUserIds.has(activeMember.user.id);
+        });
+
+        if (usersToAdd.length > 0) {
+          await userGroupMembershipDAL.insertMany(
+            usersToAdd.map((userId) => ({
+              userId,
+              groupId: team.id
+            })),
+            tx
+          );
+          updatedTeams.add(teamName);
+        }
+
+        if (membershipsToRemove.length > 0) {
+          await userGroupMembershipDAL.delete(
+            {
+              $in: {
+                id: membershipsToRemove.map((m) => m.id)
+              }
+            },
+            tx
+          );
+          updatedTeams.add(teamName);
+        }
+      }
+    });
+
+    const syncDuration = Date.now() - startTime;
+
+    logger.info(
+      {
+        orgId: orgPermission.orgId,
+        createdTeams: createdTeams.size,
+        syncDuration
+      },
+      "GitHub team sync completed"
+    );
+
+    return {
+      totalUsers: activeMembers.length,
+      errors: syncErrors,
+      createdTeams: Array.from(createdTeams),
+      updatedTeams: Array.from(updatedTeams),
+      removedMemberships: totalRemovedMemberships,
+      syncDuration
+    };
   };
 
   return {
@@ -349,6 +821,8 @@ export const githubOrgSyncServiceFactory = ({
     updateGithubOrgSync,
     deleteGithubOrgSync,
     getGithubOrgSync,
-    syncUserGroups
+    syncUserGroups,
+    syncAllTeams,
+    validateGithubToken
   };
 };

backend/src/ee/services/github-org-sync/github-org-sync-types.ts

@@ -21,3 +21,21 @@ export interface TDeleteGithubOrgSyncDTO {
 export interface TGetGithubOrgSyncDTO {
   orgPermission: OrgServiceActor;
 }
+
+export interface TSyncAllTeamsDTO {
+  orgPermission: OrgServiceActor;
+}
+
+export interface TSyncResult {
+  totalUsers: number;
+  errors: string[];
+  createdTeams: string[];
+  updatedTeams: string[];
+  removedMemberships: number;
+  syncDuration: number;
+}
+
+export interface TValidateGithubTokenDTO {
+  orgPermission: OrgServiceActor;
+  githubOrgAccessToken: string;
+}

backend/src/ee/services/permission/default-roles.ts

@@ -2,6 +2,7 @@ import { AbilityBuilder, createMongoAbility, MongoAbility } from "@casl/ability"
 
 import {
   ProjectPermissionActions,
+  ProjectPermissionAuditLogsActions,
   ProjectPermissionCertificateActions,
   ProjectPermissionCmekActions,
   ProjectPermissionCommitsActions,
@@ -394,7 +395,7 @@ const buildMemberPermissionRules = () => {
   );
 
   can([ProjectPermissionActions.Read], ProjectPermissionSub.Role);
-  can([ProjectPermissionActions.Read], ProjectPermissionSub.AuditLogs);
+  can([ProjectPermissionAuditLogsActions.Read], ProjectPermissionSub.AuditLogs);
   can([ProjectPermissionActions.Read], ProjectPermissionSub.IpAllowList);
 
   // double check if all CRUD are needed for CA and Certificates
@@ -502,7 +503,7 @@ const buildViewerPermissionRules = () => {
   can(ProjectPermissionActions.Read, ProjectPermissionSub.Settings);
   can(ProjectPermissionActions.Read, ProjectPermissionSub.Environments);
   can(ProjectPermissionActions.Read, ProjectPermissionSub.Tags);
-  can(ProjectPermissionActions.Read, ProjectPermissionSub.AuditLogs);
+  can(ProjectPermissionAuditLogsActions.Read, ProjectPermissionSub.AuditLogs);
   can(ProjectPermissionActions.Read, ProjectPermissionSub.IpAllowList);
   can(ProjectPermissionActions.Read, ProjectPermissionSub.CertificateAuthorities);
   can(ProjectPermissionCertificateActions.Read, ProjectPermissionSub.Certificates);

backend/src/ee/services/permission/org-permission.ts

@@ -23,6 +23,10 @@ export enum OrgPermissionAppConnectionActions {
   Connect = "connect"
 }
 
+export enum OrgPermissionAuditLogsActions {
+  Read = "read"
+}
+
 export enum OrgPermissionKmipActions {
   Proxy = "proxy",
   Setup = "setup"
@@ -90,6 +94,7 @@ export enum OrgPermissionSubjects {
   Sso = "sso",
   Scim = "scim",
   GithubOrgSync = "github-org-sync",
+  GithubOrgSyncManual = "github-org-sync-manual",
   Ldap = "ldap",
   Groups = "groups",
   Billing = "billing",
@@ -119,13 +124,14 @@ export type OrgPermissionSet =
   | [OrgPermissionActions, OrgPermissionSubjects.Sso]
   | [OrgPermissionActions, OrgPermissionSubjects.Scim]
   | [OrgPermissionActions, OrgPermissionSubjects.GithubOrgSync]
+  | [OrgPermissionActions, OrgPermissionSubjects.GithubOrgSyncManual]
   | [OrgPermissionActions, OrgPermissionSubjects.Ldap]
   | [OrgPermissionGroupActions, OrgPermissionSubjects.Groups]
   | [OrgPermissionActions, OrgPermissionSubjects.SecretScanning]
   | [OrgPermissionBillingActions, OrgPermissionSubjects.Billing]
   | [OrgPermissionIdentityActions, OrgPermissionSubjects.Identity]
   | [OrgPermissionActions, OrgPermissionSubjects.Kms]
-  | [OrgPermissionActions, OrgPermissionSubjects.AuditLogs]
+  | [OrgPermissionAuditLogsActions, OrgPermissionSubjects.AuditLogs]
   | [OrgPermissionActions, OrgPermissionSubjects.ProjectTemplates]
   | [OrgPermissionGatewayActions, OrgPermissionSubjects.Gateway]
   | [
@@ -188,6 +194,10 @@ export const OrgPermissionSchema = z.discriminatedUnion("subject", [
     subject: z.literal(OrgPermissionSubjects.GithubOrgSync).describe("The entity this permission pertains to."),
     action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.")
   }),
+  z.object({
+    subject: z.literal(OrgPermissionSubjects.GithubOrgSyncManual).describe("The entity this permission pertains to."),
+    action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.")
+  }),
   z.object({
     subject: z.literal(OrgPermissionSubjects.Ldap).describe("The entity this permission pertains to."),
     action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.")
@@ -214,7 +224,9 @@ export const OrgPermissionSchema = z.discriminatedUnion("subject", [
   }),
   z.object({
     subject: z.literal(OrgPermissionSubjects.AuditLogs).describe("The entity this permission pertains to."),
-    action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.")
+    action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionAuditLogsActions).describe(
+      "Describe what action an entity can take."
+    )
   }),
   z.object({
     subject: z.literal(OrgPermissionSubjects.ProjectTemplates).describe("The entity this permission pertains to."),
@@ -309,6 +321,11 @@ const buildAdminPermission = () => {
   can(OrgPermissionActions.Edit, OrgPermissionSubjects.GithubOrgSync);
   can(OrgPermissionActions.Delete, OrgPermissionSubjects.GithubOrgSync);
 
+  can(OrgPermissionActions.Read, OrgPermissionSubjects.GithubOrgSyncManual);
+  can(OrgPermissionActions.Create, OrgPermissionSubjects.GithubOrgSyncManual);
+  can(OrgPermissionActions.Edit, OrgPermissionSubjects.GithubOrgSyncManual);
+  can(OrgPermissionActions.Delete, OrgPermissionSubjects.GithubOrgSyncManual);
+
   can(OrgPermissionActions.Read, OrgPermissionSubjects.Ldap);
   can(OrgPermissionActions.Create, OrgPermissionSubjects.Ldap);
   can(OrgPermissionActions.Edit, OrgPermissionSubjects.Ldap);
@@ -340,10 +357,7 @@ const buildAdminPermission = () => {
   can(OrgPermissionActions.Edit, OrgPermissionSubjects.Kms);
   can(OrgPermissionActions.Delete, OrgPermissionSubjects.Kms);
 
-  can(OrgPermissionActions.Read, OrgPermissionSubjects.AuditLogs);
-  can(OrgPermissionActions.Create, OrgPermissionSubjects.AuditLogs);
-  can(OrgPermissionActions.Edit, OrgPermissionSubjects.AuditLogs);
-  can(OrgPermissionActions.Delete, OrgPermissionSubjects.AuditLogs);
+  can(OrgPermissionAuditLogsActions.Read, OrgPermissionSubjects.AuditLogs);
 
   can(OrgPermissionActions.Read, OrgPermissionSubjects.ProjectTemplates);
   can(OrgPermissionActions.Create, OrgPermissionSubjects.ProjectTemplates);
@@ -416,7 +430,7 @@ const buildMemberPermission = () => {
   can(OrgPermissionIdentityActions.Edit, OrgPermissionSubjects.Identity);
   can(OrgPermissionIdentityActions.Delete, OrgPermissionSubjects.Identity);
 
-  can(OrgPermissionActions.Read, OrgPermissionSubjects.AuditLogs);
+  can(OrgPermissionAuditLogsActions.Read, OrgPermissionSubjects.AuditLogs);
 
   can(OrgPermissionAppConnectionActions.Connect, OrgPermissionSubjects.AppConnections);
   can(OrgPermissionGatewayActions.ListGateways, OrgPermissionSubjects.Gateway);

backend/src/ee/services/permission/project-permission.ts

@@ -164,6 +164,10 @@ export enum ProjectPermissionSecretEventActions {
   SubscribeImportMutations = "subscribe-on-import-mutations"
 }
 
+export enum ProjectPermissionAuditLogsActions {
+  Read = "read"
+}
+
 export enum ProjectPermissionSub {
   Role = "role",
   Member = "member",
@@ -304,7 +308,7 @@ export type ProjectPermissionSet =
   | [ProjectPermissionGroupActions, ProjectPermissionSub.Groups]
   | [ProjectPermissionActions, ProjectPermissionSub.Integrations]
   | [ProjectPermissionActions, ProjectPermissionSub.Webhooks]
-  | [ProjectPermissionActions, ProjectPermissionSub.AuditLogs]
+  | [ProjectPermissionAuditLogsActions, ProjectPermissionSub.AuditLogs]
   | [ProjectPermissionActions, ProjectPermissionSub.Environments]
   | [ProjectPermissionActions, ProjectPermissionSub.IpAllowList]
   | [ProjectPermissionActions, ProjectPermissionSub.Settings]
@@ -645,7 +649,7 @@ const GeneralPermissionSchema = [
   }),
   z.object({
     subject: z.literal(ProjectPermissionSub.AuditLogs).describe("The entity this permission pertains to."),
-    action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionActions).describe(
+    action: CASL_ACTION_SCHEMA_NATIVE_ENUM(ProjectPermissionAuditLogsActions).describe(
       "Describe what action an entity can take."
     )
   }),

backend/src/keystore/keystore.ts

@@ -41,6 +41,7 @@ export const KeyStorePrefixes = {
   SecretRotationLock: (rotationId: string) => `secret-rotation-v2-mutex-${rotationId}` as const,
   SecretScanningLock: (dataSourceId: string, resourceExternalId: string) =>
     `secret-scanning-v2-mutex-${dataSourceId}-${resourceExternalId}` as const,
+  IdentityLockoutLock: (lockoutKey: string) => `identity-lockout-lock-${lockoutKey}` as const,
   CaOrderCertificateForSubscriberLock: (subscriberId: string) =>
     `ca-order-certificate-for-subscriber-lock-${subscriberId}` as const,
   SecretSyncLastRunTimestamp: (syncId: string) => `secret-sync-last-run-${syncId}` as const,

backend/src/lib/api-docs/constants.ts

@@ -2174,7 +2174,9 @@ export const CertificateAuthorities = {
       directoryUrl: `The directory URL for the ACME Certificate Authority.`,
       accountEmail: `The email address for the ACME Certificate Authority.`,
       provider: `The DNS provider for the ACME Certificate Authority.`,
-      hostedZoneId: `The hosted zone ID for the ACME Certificate Authority.`
+      hostedZoneId: `The hosted zone ID for the ACME Certificate Authority.`,
+      eabKid: `The External Account Binding (EAB) Key ID for the ACME Certificate Authority. Required if the ACME provider uses EAB.`,
+      eabHmacKey: `The External Account Binding (EAB) HMAC key for the ACME Certificate Authority. Required if the ACME provider uses EAB.`
     },
     INTERNAL: {
       type: "The type of CA to create.",

backend/src/lib/retry/index.ts

@@ -0,0 +1,43 @@
+/* eslint-disable no-await-in-loop */
+interface GitHubApiError extends Error {
+  status?: number;
+  response?: {
+    status?: number;
+    headers?: {
+      "x-ratelimit-reset"?: string;
+    };
+  };
+}
+
+const delay = (ms: number) =>
+  new Promise<void>((resolve) => {
+    setTimeout(() => resolve(), ms);
+  });
+
+export const retryWithBackoff = async <T>(fn: () => Promise<T>, maxRetries = 3, baseDelay = 1000): Promise<T> => {
+  let lastError: Error;
+
+  for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error as Error;
+      const gitHubError = error as GitHubApiError;
+      const statusCode = gitHubError.status || gitHubError.response?.status;
+      if (statusCode === 403) {
+        const rateLimitReset = gitHubError.response?.headers?.["x-ratelimit-reset"];
+        if (rateLimitReset) {
+          const resetTime = parseInt(rateLimitReset, 10) * 1000;
+          const waitTime = Math.max(resetTime - Date.now(), baseDelay);
+          await delay(Math.min(waitTime, 60000));
+        } else {
+          await delay(baseDelay * 2 ** attempt);
+        }
+      } else if (attempt < maxRetries) {
+        await delay(baseDelay * 2 ** attempt);
+      }
+    }
+  }
+
+  throw lastError!;
+};

backend/src/server/routes/index.ts

@@ -680,7 +680,8 @@ export const registerRoutes = async (
     kmsService,
     permissionService,
     groupDAL,
-    userGroupMembershipDAL
+    userGroupMembershipDAL,
+    orgMembershipDAL
   });
 
   const ldapService = ldapConfigServiceFactory({
@@ -1747,7 +1748,8 @@ export const registerRoutes = async (
   const migrationService = externalMigrationServiceFactory({
     externalMigrationQueue,
     userDAL,
-    permissionService
+    permissionService,
+    gatewayService
   });
 
   const externalGroupOrgRoleMappingService = externalGroupOrgRoleMappingServiceFactory({

backend/src/server/routes/v1/dashboard-router.ts

@@ -703,6 +703,9 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
       // prevent older projects from accessing endpoint
       if (!shouldUseSecretV2Bridge) throw new BadRequestError({ message: "Project version not supported" });
 
+      // verify folder exists and user has project permission
+      await server.services.folder.getFolderByPath({ projectId, environment, secretPath }, req.permission);
+
       const tags = req.query.tags?.split(",") ?? [];
 
       let remainingLimit = limit;

backend/src/server/routes/v3/external-migration-router.ts

@@ -66,7 +66,8 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider
         vaultAccessToken: z.string(),
         vaultNamespace: z.string().trim().optional(),
         vaultUrl: z.string(),
-        mappingType: z.nativeEnum(VaultMappingType)
+        mappingType: z.nativeEnum(VaultMappingType),
+        gatewayId: z.string().optional()
       })
     },
     onRequest: verifyAuth([AuthMode.JWT]),

backend/src/server/routes/v3/secret-router.ts

@@ -419,6 +419,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
         200: z.object({
           secret: secretRawSchema.extend({
             secretValueHidden: z.boolean(),
+            secretPath: z.string(),
             tags: SanitizedTagSchema.array().optional(),
             secretMetadata: ResourceMetadataSchema.optional()
           })

backend/src/services/app-connection/app-connection-service.ts

@@ -600,7 +600,7 @@ export const appConnectionServiceFactory = ({
     azureClientSecrets: azureClientSecretsConnectionService(connectAppConnectionById, appConnectionDAL, kmsService),
     azureDevOps: azureDevOpsConnectionService(connectAppConnectionById, appConnectionDAL, kmsService),
     auth0: auth0ConnectionService(connectAppConnectionById, appConnectionDAL, kmsService),
-    hcvault: hcVaultConnectionService(connectAppConnectionById),
+    hcvault: hcVaultConnectionService(connectAppConnectionById, gatewayService),
     windmill: windmillConnectionService(connectAppConnectionById),
     teamcity: teamcityConnectionService(connectAppConnectionById),
     oci: ociConnectionService(connectAppConnectionById, licenseService),

backend/src/services/app-connection/auth0/auth0-connection-fns.ts

@@ -91,7 +91,7 @@ export const validateAuth0ConnectionCredentials = async ({ credentials }: TAuth0
     };
   } catch (e: unknown) {
     throw new BadRequestError({
-      message: (e as Error).message ?? `Unable to validate connection: verify credentials`
+      message: (e as Error).message ?? "Unable to validate connection: verify credentials"
     });
   }
 };

backend/src/services/app-connection/azure-app-configuration/azure-app-configuration-connection-fns.ts

@@ -70,7 +70,7 @@ export const validateAzureAppConfigurationConnectionCredentials = async (
           tokenError = e;
         } else {
           throw new BadRequestError({
-            message: `Unable to validate connection: verify credentials`
+            message: "Unable to validate connection: verify credentials"
           });
         }
       }

backend/src/services/app-connection/azure-client-secrets/azure-client-secrets-connection-fns.ts

@@ -186,7 +186,7 @@ export const validateAzureClientSecretsConnectionCredentials = async (config: TA
           tokenError = e;
         } else {
           throw new BadRequestError({
-            message: `Unable to validate connection: verify credentials`
+            message: "Unable to validate connection: verify credentials"
           });
         }
       }

backend/src/services/app-connection/azure-devops/azure-devops-fns.ts

@@ -204,7 +204,7 @@ export const validateAzureDevOpsConnectionCredentials = async (config: TAzureDev
           tokenError = e;
         } else {
           throw new BadRequestError({
-            message: `Unable to validate connection: verify credentials`
+            message: "Unable to validate connection: verify credentials"
           });
         }
       }

backend/src/services/app-connection/azure-key-vault/azure-key-vault-connection-fns.ts

@@ -186,7 +186,7 @@ export const validateAzureKeyVaultConnectionCredentials = async (config: TAzureK
           tokenError = e;
         } else {
           throw new BadRequestError({
-            message: `Unable to validate connection: verify credentials`
+            message: "Unable to validate connection: verify credentials"
           });
         }
       }

backend/src/services/app-connection/camunda/camunda-connection-fns.ts

@@ -82,7 +82,7 @@ export const validateCamundaConnectionCredentials = async (appConnection: TCamun
     };
   } catch (e: unknown) {
     throw new BadRequestError({
-      message: `Unable to validate connection: verify credentials`
+      message: "Unable to validate connection: verify credentials"
     });
   }
 };

backend/src/services/app-connection/databricks/databricks-connection-fns.ts

@@ -89,7 +89,7 @@ export const validateDatabricksConnectionCredentials = async (appConnection: TDa
     };
   } catch (e: unknown) {
     throw new BadRequestError({
-      message: `Unable to validate connection: verify credentials`
+      message: "Unable to validate connection: verify credentials"
     });
   }
 };

backend/src/services/app-connection/github-radar/github-radar-connection-fns.ts

@@ -114,7 +114,7 @@ export const validateGitHubRadarConnectionCredentials = async (config: TGitHubRa
     }
 
     throw new BadRequestError({
-      message: `Unable to validate connection: verify credentials`
+      message: "Unable to validate connection: verify credentials"
     });
   }
 

backend/src/services/app-connection/github/github-connection-fns.ts

@@ -447,7 +447,7 @@ export const validateGitHubConnectionCredentials = async (
     }
 
     throw new BadRequestError({
-      message: `Unable to validate connection: verify credentials`
+      message: "Unable to validate connection: verify credentials"
     });
   }
 

backend/src/services/app-connection/hc-vault/hc-vault-connection-fns.ts

@@ -1,18 +1,18 @@
-import { AxiosError } from "axios";
+import { AxiosError, AxiosRequestConfig, AxiosResponse } from "axios";
+import https from "https";
 
+import { verifyHostInputValidity } from "@app/ee/services/dynamic-secret/dynamic-secret-fns";
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { request } from "@app/lib/config/request";
 import { BadRequestError } from "@app/lib/errors";
 import { removeTrailingSlash } from "@app/lib/fn";
+import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway";
+import { logger } from "@app/lib/logger";
 import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
 import { AppConnection } from "@app/services/app-connection/app-connection-enums";
 
 import { HCVaultConnectionMethod } from "./hc-vault-connection-enums";
-import {
-  THCVaultConnection,
-  THCVaultConnectionConfig,
-  THCVaultMountResponse,
-  TValidateHCVaultConnectionCredentials
-} from "./hc-vault-connection-types";
+import { THCVaultConnection, THCVaultConnectionConfig, THCVaultMountResponse } from "./hc-vault-connection-types";
 
 export const getHCVaultInstanceUrl = async (config: THCVaultConnectionConfig) => {
   const instanceUrl = removeTrailingSlash(config.credentials.instanceUrl);
@@ -37,7 +37,78 @@ type TokenRespData = {
   };
 };
 
-export const getHCVaultAccessToken = async (connection: TValidateHCVaultConnectionCredentials) => {
+export const requestWithHCVaultGateway = async <T>(
+  appConnection: { gatewayId?: string | null },
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
+  requestConfig: AxiosRequestConfig
+): Promise<AxiosResponse<T>> => {
+  const { gatewayId } = appConnection;
+
+  // If gateway isn't set up, don't proxy request
+  if (!gatewayId) {
+    return request.request(requestConfig);
+  }
+
+  const url = new URL(requestConfig.url as string);
+
+  await blockLocalAndPrivateIpAddresses(url.toString());
+
+  const [targetHost] = await verifyHostInputValidity(url.hostname, true);
+  const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(gatewayId);
+  const [relayHost, relayPort] = relayDetails.relayAddress.split(":");
+
+  return withGatewayProxy(
+    async (proxyPort) => {
+      const httpsAgent = new https.Agent({
+        servername: targetHost
+      });
+
+      url.protocol = "https:";
+      url.host = `localhost:${proxyPort}`;
+
+      const finalRequestConfig: AxiosRequestConfig = {
+        ...requestConfig,
+        url: url.toString(),
+        httpsAgent,
+        headers: {
+          ...requestConfig.headers,
+          Host: targetHost
+        }
+      };
+
+      try {
+        return await request.request(finalRequestConfig);
+      } catch (error) {
+        if (error instanceof AxiosError) {
+          logger.error(
+            { message: error.message, data: (error.response as undefined | { data: unknown })?.data },
+            "Error during HashiCorp Vault gateway request:"
+          );
+        }
+        throw error;
+      }
+    },
+    {
+      protocol: GatewayProxyProtocol.Tcp,
+      targetHost,
+      targetPort: url.port ? Number(url.port) : 8200, // 8200 is the default port for Vault self-hosted/dedicated
+      relayHost,
+      relayPort: Number(relayPort),
+      identityId: relayDetails.identityId,
+      orgId: relayDetails.orgId,
+      tlsOptions: {
+        ca: relayDetails.certChain,
+        cert: relayDetails.certificate,
+        key: relayDetails.privateKey.toString()
+      }
+    }
+  );
+};
+
+export const getHCVaultAccessToken = async (
+  connection: THCVaultConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
   // Return access token directly if not using AppRole method
   if (connection.method !== HCVaultConnectionMethod.AppRole) {
     return connection.credentials.accessToken;
@@ -46,16 +117,16 @@ export const getHCVaultAccessToken = async (connection: TValidateHCVaultConnecti
   // Generate temporary token for AppRole method
   try {
     const { instanceUrl, roleId, secretId } = connection.credentials;
-    const tokenResp = await request.post<TokenRespData>(
-      `${removeTrailingSlash(instanceUrl)}/v1/auth/approle/login`,
-      { role_id: roleId, secret_id: secretId },
-      {
-        headers: {
-          "Content-Type": "application/json",
-          ...(connection.credentials.namespace ? { "X-Vault-Namespace": connection.credentials.namespace } : {})
-        }
-      }
-    );
+
+    const tokenResp = await requestWithHCVaultGateway<TokenRespData>(connection, gatewayService, {
+      url: `${removeTrailingSlash(instanceUrl)}/v1/auth/approle/login`,
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        ...(connection.credentials.namespace ? { "X-Vault-Namespace": connection.credentials.namespace } : {})
+      },
+      data: { role_id: roleId, secret_id: secretId }
+    });
 
     if (tokenResp.status !== 200) {
       throw new BadRequestError({
@@ -71,38 +142,55 @@ export const getHCVaultAccessToken = async (connection: TValidateHCVaultConnecti
   }
 };
 
-export const validateHCVaultConnectionCredentials = async (config: THCVaultConnectionConfig) => {
-  const instanceUrl = await getHCVaultInstanceUrl(config);
+export const validateHCVaultConnectionCredentials = async (
+  connection: THCVaultConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
+  const instanceUrl = await getHCVaultInstanceUrl(connection);
 
   try {
-    const accessToken = await getHCVaultAccessToken(config);
+    const accessToken = await getHCVaultAccessToken(connection, gatewayService);
 
     // Verify token
-    await request.get(`${instanceUrl}/v1/auth/token/lookup-self`, {
+    await requestWithHCVaultGateway(connection, gatewayService, {
+      url: `${instanceUrl}/v1/auth/token/lookup-self`,
+      method: "GET",
       headers: { "X-Vault-Token": accessToken }
     });
 
-    return config.credentials;
+    return connection.credentials;
   } catch (error: unknown) {
+    logger.error(error, "Unable to verify HC Vault connection");
+
     if (error instanceof AxiosError) {
       throw new BadRequestError({
         message: `Failed to validate credentials: ${error.message || "Unknown error"}`
       });
     }
+
+    if (error instanceof BadRequestError) {
+      throw error;
+    }
+
     throw new BadRequestError({
       message: "Unable to validate connection: verify credentials"
     });
   }
 };
 
-export const listHCVaultMounts = async (appConnection: THCVaultConnection) => {
-  const instanceUrl = await getHCVaultInstanceUrl(appConnection);
-  const accessToken = await getHCVaultAccessToken(appConnection);
+export const listHCVaultMounts = async (
+  connection: THCVaultConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
+  const instanceUrl = await getHCVaultInstanceUrl(connection);
+  const accessToken = await getHCVaultAccessToken(connection, gatewayService);
 
-  const { data } = await request.get<THCVaultMountResponse>(`${instanceUrl}/v1/sys/mounts`, {
+  const { data } = await requestWithHCVaultGateway<THCVaultMountResponse>(connection, gatewayService, {
+    url: `${instanceUrl}/v1/sys/mounts`,
+    method: "GET",
     headers: {
       "X-Vault-Token": accessToken,
-      ...(appConnection.credentials.namespace ? { "X-Vault-Namespace": appConnection.credentials.namespace } : {})
+      ...(connection.credentials.namespace ? { "X-Vault-Namespace": connection.credentials.namespace } : {})
     }
   });
 

backend/src/services/app-connection/hc-vault/hc-vault-connection-schemas.ts

@@ -55,11 +55,18 @@ export const HCVaultConnectionSchema = z.intersection(
 export const SanitizedHCVaultConnectionSchema = z.discriminatedUnion("method", [
   BaseHCVaultConnectionSchema.extend({
     method: z.literal(HCVaultConnectionMethod.AccessToken),
-    credentials: HCVaultConnectionAccessTokenCredentialsSchema.pick({})
+    credentials: HCVaultConnectionAccessTokenCredentialsSchema.pick({
+      namespace: true,
+      instanceUrl: true
+    })
   }),
   BaseHCVaultConnectionSchema.extend({
     method: z.literal(HCVaultConnectionMethod.AppRole),
-    credentials: HCVaultConnectionAppRoleCredentialsSchema.pick({})
+    credentials: HCVaultConnectionAppRoleCredentialsSchema.pick({
+      namespace: true,
+      instanceUrl: true,
+      roleId: true
+    })
   })
 ]);
 
@@ -81,7 +88,7 @@ export const ValidateHCVaultConnectionCredentialsSchema = z.discriminatedUnion("
 ]);
 
 export const CreateHCVaultConnectionSchema = ValidateHCVaultConnectionCredentialsSchema.and(
-  GenericCreateAppConnectionFieldsSchema(AppConnection.HCVault)
+  GenericCreateAppConnectionFieldsSchema(AppConnection.HCVault, { supportsGateways: true })
 );
 
 export const UpdateHCVaultConnectionSchema = z
@@ -91,7 +98,7 @@ export const UpdateHCVaultConnectionSchema = z
       .optional()
       .describe(AppConnections.UPDATE(AppConnection.HCVault).credentials)
   })
-  .and(GenericUpdateAppConnectionFieldsSchema(AppConnection.HCVault));
+  .and(GenericUpdateAppConnectionFieldsSchema(AppConnection.HCVault, { supportsGateways: true }));
 
 export const HCVaultConnectionListItemSchema = z.object({
   name: z.literal("HCVault"),

backend/src/services/app-connection/hc-vault/hc-vault-connection-service.ts

@@ -1,3 +1,4 @@
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { logger } from "@app/lib/logger";
 import { OrgServiceActor } from "@app/lib/types";
 
@@ -11,12 +12,15 @@ type TGetAppConnectionFunc = (
   actor: OrgServiceActor
 ) => Promise<THCVaultConnection>;
 
-export const hcVaultConnectionService = (getAppConnection: TGetAppConnectionFunc) => {
+export const hcVaultConnectionService = (
+  getAppConnection: TGetAppConnectionFunc,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
   const listMounts = async (connectionId: string, actor: OrgServiceActor) => {
     const appConnection = await getAppConnection(AppConnection.HCVault, connectionId, actor);
 
     try {
-      const mounts = await listHCVaultMounts(appConnection);
+      const mounts = await listHCVaultMounts(appConnection, gatewayService);
       return mounts;
     } catch (error) {
       logger.error(error, "Failed to establish connection with Hashicorp Vault");

backend/src/services/auth/auth-login-service.ts

@@ -453,10 +453,14 @@ export const authLoginServiceFactory = ({
 
     const selectedOrg = await orgDAL.findById(organizationId);
 
-    // Check if authEnforced is true, if that's the case, throw an error
-    if (selectedOrg.authEnforced) {
+    // Check if authEnforced is true and the current auth method is not an enforced method
+    if (
+      selectedOrg.authEnforced &&
+      !isAuthMethodSaml(decodedToken.authMethod) &&
+      decodedToken.authMethod !== AuthMethod.OIDC
+    ) {
       throw new BadRequestError({
-        message: "Authentication is required by your organization before you can log in."
+        message: "Login with the auth method required by your organization."
       });
     }
 

backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts

@@ -64,6 +64,8 @@ type DBConfigurationColumn = {
   directoryUrl: string;
   accountEmail: string;
   hostedZoneId: string;
+  eabKid?: string;
+  eabHmacKey?: string;
 };
 
 export const castDbEntryToAcmeCertificateAuthority = (
@@ -89,7 +91,9 @@ export const castDbEntryToAcmeCertificateAuthority = (
         hostedZoneId: dbConfigurationCol.hostedZoneId
       },
       directoryUrl: dbConfigurationCol.directoryUrl,
-      accountEmail: dbConfigurationCol.accountEmail
+      accountEmail: dbConfigurationCol.accountEmail,
+      eabKid: dbConfigurationCol.eabKid,
+      eabHmacKey: dbConfigurationCol.eabHmacKey
     },
     status: ca.status as CaStatus
   };
@@ -128,7 +132,7 @@ export const AcmeCertificateAuthorityFns = ({
       });
     }
 
-    const { dnsAppConnectionId, directoryUrl, accountEmail, dnsProviderConfig } = configuration;
+    const { dnsAppConnectionId, directoryUrl, accountEmail, dnsProviderConfig, eabKid, eabHmacKey } = configuration;
     const appConnection = await appConnectionDAL.findById(dnsAppConnectionId);
 
     if (!appConnection) {
@@ -171,7 +175,9 @@ export const AcmeCertificateAuthorityFns = ({
               directoryUrl,
               accountEmail,
               dnsProvider: dnsProviderConfig.provider,
-              hostedZoneId: dnsProviderConfig.hostedZoneId
+              hostedZoneId: dnsProviderConfig.hostedZoneId,
+              eabKid,
+              eabHmacKey
             }
           },
           tx
@@ -214,7 +220,7 @@ export const AcmeCertificateAuthorityFns = ({
   }) => {
     const updatedCa = await certificateAuthorityDAL.transaction(async (tx) => {
       if (configuration) {
-        const { dnsAppConnectionId, directoryUrl, accountEmail, dnsProviderConfig } = configuration;
+        const { dnsAppConnectionId, directoryUrl, accountEmail, dnsProviderConfig, eabKid, eabHmacKey } = configuration;
         const appConnection = await appConnectionDAL.findById(dnsAppConnectionId);
 
         if (!appConnection) {
@@ -254,7 +260,9 @@ export const AcmeCertificateAuthorityFns = ({
               directoryUrl,
               accountEmail,
               dnsProvider: dnsProviderConfig.provider,
-              hostedZoneId: dnsProviderConfig.hostedZoneId
+              hostedZoneId: dnsProviderConfig.hostedZoneId,
+              eabKid,
+              eabHmacKey
             }
           },
           tx
@@ -354,10 +362,19 @@ export const AcmeCertificateAuthorityFns = ({
 
     await blockLocalAndPrivateIpAddresses(acmeCa.configuration.directoryUrl);
 
-    const acmeClient = new acme.Client({
+    const acmeClientOptions: acme.ClientOptions = {
       directoryUrl: acmeCa.configuration.directoryUrl,
       accountKey
-    });
+    };
+
+    if (acmeCa.configuration.eabKid && acmeCa.configuration.eabHmacKey) {
+      acmeClientOptions.externalAccountBinding = {
+        kid: acmeCa.configuration.eabKid,
+        hmacKey: acmeCa.configuration.eabHmacKey
+      };
+    }
+
+    const acmeClient = new acme.Client(acmeClientOptions);
 
     const alg = keyAlgorithmToAlgCfg(CertKeyAlgorithm.RSA_2048);
 

backend/src/services/certificate-authority/acme/acme-certificate-authority-schemas.ts

@@ -18,7 +18,9 @@ export const AcmeCertificateAuthorityConfigurationSchema = z.object({
     hostedZoneId: z.string().trim().min(1).describe(CertificateAuthorities.CONFIGURATIONS.ACME.hostedZoneId)
   }),
   directoryUrl: z.string().url().trim().min(1).describe(CertificateAuthorities.CONFIGURATIONS.ACME.directoryUrl),
-  accountEmail: z.string().trim().min(1).describe(CertificateAuthorities.CONFIGURATIONS.ACME.accountEmail)
+  accountEmail: z.string().trim().min(1).describe(CertificateAuthorities.CONFIGURATIONS.ACME.accountEmail),
+  eabKid: z.string().trim().max(64).optional().describe(CertificateAuthorities.CONFIGURATIONS.ACME.eabKid),
+  eabHmacKey: z.string().trim().max(512).optional().describe(CertificateAuthorities.CONFIGURATIONS.ACME.eabHmacKey)
 });
 
 export const AcmeCertificateAuthorityCredentialsSchema = z.object({

backend/src/services/external-migration/external-migration-fns/vault.ts

@@ -1,12 +1,21 @@
+import https from "node:https";
+
 import axios, { AxiosInstance } from "axios";
 import { v4 as uuidv4 } from "uuid";
 
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { BadRequestError } from "@app/lib/errors";
+import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway";
 import { logger } from "@app/lib/logger";
 import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
 
 import { InfisicalImportData, VaultMappingType } from "../external-migration-types";
 
+enum KvVersion {
+  V1 = "1",
+  V2 = "2"
+}
+
 type VaultData = {
   namespace: string;
   mount: string;
@@ -14,7 +23,42 @@ type VaultData = {
   secretData: Record<string, string>;
 };
 
-const vaultFactory = () => {
+const vaultFactory = (gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">) => {
+  const $gatewayProxyWrapper = async <T>(
+    inputs: {
+      gatewayId: string;
+      targetHost?: string;
+      targetPort?: number;
+    },
+    gatewayCallback: (host: string, port: number, httpsAgent?: https.Agent) => Promise<T>
+  ): Promise<T> => {
+    const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(inputs.gatewayId);
+    const [relayHost, relayPort] = relayDetails.relayAddress.split(":");
+
+    const callbackResult = await withGatewayProxy(
+      async (port, httpsAgent) => {
+        const res = await gatewayCallback("http://localhost", port, httpsAgent);
+        return res;
+      },
+      {
+        protocol: GatewayProxyProtocol.Http,
+        targetHost: inputs.targetHost,
+        targetPort: inputs.targetPort,
+        relayHost,
+        relayPort: Number(relayPort),
+        identityId: relayDetails.identityId,
+        orgId: relayDetails.orgId,
+        tlsOptions: {
+          ca: relayDetails.certChain,
+          cert: relayDetails.certificate,
+          key: relayDetails.privateKey.toString()
+        }
+      }
+    );
+
+    return callbackResult;
+  };
+
   const getMounts = async (request: AxiosInstance) => {
     const response = await request
       .get<{
@@ -31,11 +75,24 @@ const vaultFactory = () => {
 
   const getPaths = async (
     request: AxiosInstance,
-    { mountPath, secretPath = "" }: { mountPath: string; secretPath?: string }
+    { mountPath, secretPath = "" }: { mountPath: string; secretPath?: string },
+    kvVersion: KvVersion
   ) => {
     try {
-      // For KV v2: /v1/{mount}/metadata/{path}?list=true
-      const path = secretPath ? `${mountPath}/metadata/${secretPath}` : `${mountPath}/metadata`;
+      if (kvVersion === KvVersion.V2) {
+        // For KV v2: /v1/{mount}/metadata/{path}?list=true
+        const path = secretPath ? `${mountPath}/metadata/${secretPath}` : `${mountPath}/metadata`;
+        const response = await request.get<{
+          data: {
+            keys: string[];
+          };
+        }>(`/v1/${path}?list=true`);
+
+        return response.data.data.keys;
+      }
+
+      // kv version v1: /v1/{mount}?list=true
+      const path = secretPath ? `${mountPath}/${secretPath}` : mountPath;
       const response = await request.get<{
         data: {
           keys: string[];
@@ -56,21 +113,42 @@ const vaultFactory = () => {
 
   const getSecrets = async (
     request: AxiosInstance,
-    { mountPath, secretPath }: { mountPath: string; secretPath: string }
+    { mountPath, secretPath }: { mountPath: string; secretPath: string },
+    kvVersion: KvVersion
   ) => {
-    // For KV v2: /v1/{mount}/data/{path}
+    if (kvVersion === KvVersion.V2) {
+      // For KV v2: /v1/{mount}/data/{path}
+      const response = await request
+        .get<{
+          data: {
+            data: Record<string, string>; // KV v2 has nested data structure
+            metadata: {
+              created_time: string;
+              deletion_time: string;
+              destroyed: boolean;
+              version: number;
+            };
+          };
+        }>(`/v1/${mountPath}/data/${secretPath}`)
+        .catch((err) => {
+          if (axios.isAxiosError(err)) {
+            logger.error(err.response?.data, "External migration: Failed to get Vault secret");
+          }
+          throw err;
+        });
+
+      return response.data.data.data;
+    }
+
+    // kv version v1
+
     const response = await request
       .get<{
-        data: {
-          data: Record<string, string>; // KV v2 has nested data structure
-          metadata: {
-            created_time: string;
-            deletion_time: string;
-            destroyed: boolean;
-            version: number;
-          };
-        };
-      }>(`/v1/${mountPath}/data/${secretPath}`)
+        data: Record<string, string>; // KV v1 has flat data structure
+        lease_duration: number;
+        lease_id: string;
+        renewable: boolean;
+      }>(`/v1/${mountPath}/${secretPath}`)
       .catch((err) => {
         if (axios.isAxiosError(err)) {
           logger.error(err.response?.data, "External migration: Failed to get Vault secret");
@@ -78,7 +156,7 @@ const vaultFactory = () => {
         throw err;
       });
 
-    return response.data.data.data;
+    return response.data.data;
   };
 
   // helper function to check if a mount is KV v2 (will be useful if we add support for Vault KV v1)
@@ -89,9 +167,10 @@ const vaultFactory = () => {
   const recursivelyGetAllPaths = async (
     request: AxiosInstance,
     mountPath: string,
+    kvVersion: KvVersion,
     currentPath: string = ""
   ): Promise<string[]> => {
-    const paths = await getPaths(request, { mountPath, secretPath: currentPath });
+    const paths = await getPaths(request, { mountPath, secretPath: currentPath }, kvVersion);
 
     if (paths === null || paths.length === 0) {
       return [];
@@ -105,7 +184,7 @@ const vaultFactory = () => {
 
       if (path.endsWith("/")) {
         // it's a folder so we recurse into it
-        const subSecrets = await recursivelyGetAllPaths(request, mountPath, fullItemPath);
+        const subSecrets = await recursivelyGetAllPaths(request, mountPath, kvVersion, fullItemPath);
         allSecrets.push(...subSecrets);
       } else {
         // it's a secret so we add it to our results
@@ -119,60 +198,93 @@ const vaultFactory = () => {
   async function collectVaultData({
     baseUrl,
     namespace,
-    accessToken
+    accessToken,
+    gatewayId
   }: {
     baseUrl: string;
     namespace?: string;
     accessToken: string;
+    gatewayId?: string;
   }): Promise<VaultData[]> {
-    const request = axios.create({
-      baseURL: baseUrl,
-      headers: {
-        "X-Vault-Token": accessToken,
-        ...(namespace ? { "X-Vault-Namespace": namespace } : {})
+    const getData = async (host: string, port?: number, httpsAgent?: https.Agent) => {
+      const allData: VaultData[] = [];
+
+      const request = axios.create({
+        baseURL: port ? `${host}:${port}` : host,
+        headers: {
+          "X-Vault-Token": accessToken,
+          ...(namespace ? { "X-Vault-Namespace": namespace } : {})
+        },
+        httpsAgent
+      });
+
+      // Get all mounts in this namespace
+      const mounts = await getMounts(request);
+
+      for (const mount of Object.keys(mounts)) {
+        if (!mount.endsWith("/")) {
+          delete mounts[mount];
+        }
       }
-    });
 
-    const allData: VaultData[] = [];
+      for await (const [mountPath, mountInfo] of Object.entries(mounts)) {
+        // skip non-KV mounts
+        if (!mountInfo.type.startsWith("kv")) {
+          // eslint-disable-next-line no-continue
+          continue;
+        }
 
-    // Get all mounts in this namespace
-    const mounts = await getMounts(request);
+        const kvVersion = mountInfo.options?.version === "2" ? KvVersion.V2 : KvVersion.V1;
 
-    for (const mount of Object.keys(mounts)) {
-      if (!mount.endsWith("/")) {
-        delete mounts[mount];
+        // get all paths in this mount
+        const paths = await recursivelyGetAllPaths(request, `${mountPath.replace(/\/$/, "")}`, kvVersion);
+
+        const cleanMountPath = mountPath.replace(/\/$/, "");
+
+        for await (const secretPath of paths) {
+          // get the actual secret data
+          const secretData = await getSecrets(
+            request,
+            {
+              mountPath: cleanMountPath,
+              secretPath: secretPath.replace(`${cleanMountPath}/`, "")
+            },
+            kvVersion
+          );
+
+          allData.push({
+            namespace: namespace || "",
+            mount: mountPath.replace(/\/$/, ""),
+            path: secretPath.replace(`${cleanMountPath}/`, ""),
+            secretData
+          });
+        }
       }
+
+      return allData;
+    };
+
+    let data;
+
+    if (gatewayId) {
+      const url = new URL(baseUrl);
+
+      const { port, protocol, hostname } = url;
+      const cleanedProtocol = protocol.slice(0, -1);
+
+      data = await $gatewayProxyWrapper(
+        {
+          gatewayId,
+          targetHost: `${cleanedProtocol}://${hostname}`,
+          targetPort: port ? Number(port) : 8200 // 8200, default port for Vault self-hosted/dedicated
+        },
+        getData
+      );
+    } else {
+      data = await getData(baseUrl);
     }
 
-    for await (const [mountPath, mountInfo] of Object.entries(mounts)) {
-      // skip non-KV mounts
-      if (!mountInfo.type.startsWith("kv")) {
-        // eslint-disable-next-line no-continue
-        continue;
-      }
-
-      // get all paths in this mount
-      const paths = await recursivelyGetAllPaths(request, `${mountPath.replace(/\/$/, "")}`);
-
-      const cleanMountPath = mountPath.replace(/\/$/, "");
-
-      for await (const secretPath of paths) {
-        // get the actual secret data
-        const secretData = await getSecrets(request, {
-          mountPath: cleanMountPath,
-          secretPath: secretPath.replace(`${cleanMountPath}/`, "")
-        });
-
-        allData.push({
-          namespace: namespace || "",
-          mount: mountPath.replace(/\/$/, ""),
-          path: secretPath.replace(`${cleanMountPath}/`, ""),
-          secretData
-        });
-      }
-    }
-
-    return allData;
+    return data;
   }
 
   return {
@@ -296,17 +408,22 @@ export const transformToInfisicalFormatNamespaceToProjects = (
   };
 };
 
-export const importVaultDataFn = async ({
-  vaultAccessToken,
-  vaultNamespace,
-  vaultUrl,
-  mappingType
-}: {
-  vaultAccessToken: string;
-  vaultNamespace?: string;
-  vaultUrl: string;
-  mappingType: VaultMappingType;
-}) => {
+export const importVaultDataFn = async (
+  {
+    vaultAccessToken,
+    vaultNamespace,
+    vaultUrl,
+    mappingType,
+    gatewayId
+  }: {
+    vaultAccessToken: string;
+    vaultNamespace?: string;
+    vaultUrl: string;
+    mappingType: VaultMappingType;
+    gatewayId?: string;
+  },
+  { gatewayService }: { gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId"> }
+) => {
   await blockLocalAndPrivateIpAddresses(vaultUrl);
 
   if (mappingType === VaultMappingType.Namespace && !vaultNamespace) {
@@ -315,12 +432,13 @@ export const importVaultDataFn = async ({
     });
   }
 
-  const vaultApi = vaultFactory();
+  const vaultApi = vaultFactory(gatewayService);
 
   const vaultData = await vaultApi.collectVaultData({
     accessToken: vaultAccessToken,
     baseUrl: vaultUrl,
-    namespace: vaultNamespace
+    namespace: vaultNamespace,
+    gatewayId
   });
 
   const infisicalData = transformToInfisicalFormatNamespaceToProjects(vaultData, mappingType);

backend/src/services/external-migration/external-migration-service.ts

@@ -1,4 +1,5 @@
 import { OrgMembershipRole } from "@app/db/schemas";
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";
 import { crypto } from "@app/lib/crypto/cryptography";
 import { BadRequestError, ForbiddenRequestError } from "@app/lib/errors";
@@ -12,6 +13,7 @@ type TExternalMigrationServiceFactoryDep = {
   permissionService: TPermissionServiceFactory;
   externalMigrationQueue: TExternalMigrationQueueFactory;
   userDAL: Pick<TUserDALFactory, "findById">;
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
 };
 
 export type TExternalMigrationServiceFactory = ReturnType<typeof externalMigrationServiceFactory>;
@@ -19,7 +21,8 @@ export type TExternalMigrationServiceFactory = ReturnType<typeof externalMigrati
 export const externalMigrationServiceFactory = ({
   permissionService,
   externalMigrationQueue,
-  userDAL
+  userDAL,
+  gatewayService
 }: TExternalMigrationServiceFactoryDep) => {
   const importEnvKeyData = async ({
     decryptionKey,
@@ -72,6 +75,7 @@ export const externalMigrationServiceFactory = ({
     vaultNamespace,
     mappingType,
     vaultUrl,
+    gatewayId,
     actor,
     actorId,
     actorOrgId,
@@ -91,12 +95,18 @@ export const externalMigrationServiceFactory = ({
 
     const user = await userDAL.findById(actorId);
 
-    const vaultData = await importVaultDataFn({
-      vaultAccessToken,
-      vaultNamespace,
-      vaultUrl,
-      mappingType
-    });
+    const vaultData = await importVaultDataFn(
+      {
+        vaultAccessToken,
+        vaultNamespace,
+        vaultUrl,
+        mappingType,
+        gatewayId
+      },
+      {
+        gatewayService
+      }
+    );
 
     const stringifiedJson = JSON.stringify({
       data: vaultData,

backend/src/services/external-migration/external-migration-types.ts

@@ -31,6 +31,7 @@ export type TImportVaultDataDTO = {
   vaultNamespace?: string;
   mappingType: VaultMappingType;
   vaultUrl: string;
+  gatewayId?: string;
 } & Omit<TOrgPermission, "orgId">;
 
 export type TImportInfisicalDataCreate = {

backend/src/services/identity-ua/identity-ua-service.ts

@@ -8,11 +8,18 @@ import {
   validatePrivilegeChangeOperation
 } from "@app/ee/services/permission/permission-fns";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";
-import { PgSqlLock, TKeyStoreFactory } from "@app/keystore/keystore";
+import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
 import { getConfig } from "@app/lib/config/env";
 import { crypto } from "@app/lib/crypto/cryptography";
-import { BadRequestError, NotFoundError, PermissionBoundaryError, UnauthorizedError } from "@app/lib/errors";
+import {
+  BadRequestError,
+  NotFoundError,
+  PermissionBoundaryError,
+  RateLimitError,
+  UnauthorizedError
+} from "@app/lib/errors";
 import { checkIPAgainstBlocklist, extractIPDetails, isValidIpOrCidr, TIp } from "@app/lib/ip";
+import { logger } from "@app/lib/logger";
 
 import { ActorType, AuthTokenType } from "../auth/auth-type";
 import { TIdentityOrgDALFactory } from "../identity/identity-org-dal";
@@ -40,7 +47,10 @@ type TIdentityUaServiceFactoryDep = {
   identityOrgMembershipDAL: TIdentityOrgDALFactory;
   permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
   licenseService: Pick<TLicenseServiceFactory, "getPlan">;
-  keyStore: Pick<TKeyStoreFactory, "setItemWithExpiry" | "getItem" | "deleteItem" | "getKeysByPattern" | "deleteItems">;
+  keyStore: Pick<
+    TKeyStoreFactory,
+    "setItemWithExpiry" | "getItem" | "deleteItem" | "getKeysByPattern" | "deleteItems" | "acquireLock"
+  >;
 };
 
 export type TIdentityUaServiceFactory = ReturnType<typeof identityUaServiceFactory>;
@@ -74,17 +84,21 @@ export const identityUaServiceFactory = ({
 
     const LOCKOUT_KEY = `lockout:identity:${identityUa.identityId}:${IdentityAuthMethod.UNIVERSAL_AUTH}:${clientId}`;
 
-    const identityMembershipOrg = await identityOrgMembershipDAL.findOne({ identityId: identityUa.identityId });
-    if (!identityMembershipOrg) {
-      throw new UnauthorizedError({
-        message: "Invalid credentials"
+    let lock: Awaited<ReturnType<typeof keyStore.acquireLock>>;
+    try {
+      lock = await keyStore.acquireLock([KeyStorePrefixes.IdentityLockoutLock(LOCKOUT_KEY)], 500, {
+        retryCount: 3,
+        retryDelay: 300,
+        retryJitter: 100
       });
+    } catch (e) {
+      logger.info(
+        `identity login failed to acquire lock [identityId=${identityUa.identityId}] [authMethod=${IdentityAuthMethod.UNIVERSAL_AUTH}]`
+      );
+      throw new RateLimitError({ message: "Rate limit exceeded" });
     }
 
-    const identityTx = await identityUaDAL.transaction(async (tx) => {
-      await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.IdentityLogin(identityUa.identityId, clientId)]);
-
-      // Lockout Check
+    try {
       const lockoutRaw = await keyStore.getItem(LOCKOUT_KEY);
 
       let lockout: LockoutObject | undefined;
@@ -98,6 +112,13 @@ export const identityUaServiceFactory = ({
         });
       }
 
+      const identityMembershipOrg = await identityOrgMembershipDAL.findOne({ identityId: identityUa.identityId });
+      if (!identityMembershipOrg) {
+        throw new UnauthorizedError({
+          message: "Invalid credentials"
+        });
+      }
+
       const clientSecretPrefix = clientSecret.slice(0, 4);
       const clientSecretInfo = await identityUaClientSecretDAL.find({
         identityUAId: identityUa.id,
@@ -159,7 +180,7 @@ export const identityUaServiceFactory = ({
         }
       }
 
-      if (clientSecretNumUsesLimit > 0 && clientSecretNumUses === clientSecretNumUsesLimit) {
+      if (clientSecretNumUsesLimit > 0 && clientSecretNumUses >= clientSecretNumUsesLimit) {
         // number of times client secret can be used for
         // a login operation reached
         await identityUaClientSecretDAL.updateById(validClientSecretInfo.id, {
@@ -183,57 +204,61 @@ export const identityUaServiceFactory = ({
               accessTokenMaxTTL: 1000000000
             };
 
-      const uaClientSecretDoc = await identityUaClientSecretDAL.incrementUsage(validClientSecretInfo.id, tx);
-      await identityOrgMembershipDAL.updateById(
-        identityMembershipOrg.id,
-        {
-          lastLoginAuthMethod: IdentityAuthMethod.UNIVERSAL_AUTH,
-          lastLoginTime: new Date()
-        },
-        tx
-      );
-      const newToken = await identityAccessTokenDAL.create(
+      const identityAccessToken = await identityUaDAL.transaction(async (tx) => {
+        const uaClientSecretDoc = await identityUaClientSecretDAL.incrementUsage(validClientSecretInfo!.id, tx);
+        await identityOrgMembershipDAL.updateById(
+          identityMembershipOrg.id,
+          {
+            lastLoginAuthMethod: IdentityAuthMethod.UNIVERSAL_AUTH,
+            lastLoginTime: new Date()
+          },
+          tx
+        );
+        const newToken = await identityAccessTokenDAL.create(
+          {
+            identityId: identityUa.identityId,
+            isAccessTokenRevoked: false,
+            identityUAClientSecretId: uaClientSecretDoc.id,
+            accessTokenNumUses: 0,
+            accessTokenNumUsesLimit: identityUa.accessTokenNumUsesLimit,
+            accessTokenPeriod: identityUa.accessTokenPeriod,
+            authMethod: IdentityAuthMethod.UNIVERSAL_AUTH,
+            ...accessTokenTTLParams
+          },
+          tx
+        );
+
+        return newToken;
+      });
+
+      const appCfg = getConfig();
+      const accessToken = crypto.jwt().sign(
         {
           identityId: identityUa.identityId,
-          isAccessTokenRevoked: false,
-          identityUAClientSecretId: uaClientSecretDoc.id,
-          accessTokenNumUses: 0,
-          accessTokenNumUsesLimit: identityUa.accessTokenNumUsesLimit,
-          accessTokenPeriod: identityUa.accessTokenPeriod,
-          authMethod: IdentityAuthMethod.UNIVERSAL_AUTH,
-          ...accessTokenTTLParams
-        },
-        tx
+          clientSecretId: validClientSecretInfo.id,
+          identityAccessTokenId: identityAccessToken.id,
+          authTokenType: AuthTokenType.IDENTITY_ACCESS_TOKEN
+        } as TIdentityAccessTokenJwtPayload,
+        appCfg.AUTH_SECRET,
+        // akhilmhdh: for non-expiry tokens you should not even set the value, including undefined. Even for undefined jsonwebtoken throws error
+        Number(identityAccessToken.accessTokenTTL) === 0
+          ? undefined
+          : {
+              expiresIn: Number(identityAccessToken.accessTokenTTL)
+            }
       );
 
-      return { newToken, validClientSecretInfo, accessTokenTTLParams };
-    });
-
-    const appCfg = getConfig();
-    const accessToken = crypto.jwt().sign(
-      {
-        identityId: identityUa.identityId,
-        clientSecretId: identityTx.validClientSecretInfo.id,
-        identityAccessTokenId: identityTx.newToken.id,
-        authTokenType: AuthTokenType.IDENTITY_ACCESS_TOKEN
-      } as TIdentityAccessTokenJwtPayload,
-      appCfg.AUTH_SECRET,
-      // akhilmhdh: for non-expiry tokens you should not even set the value, including undefined. Even for undefined jsonwebtoken throws error
-      Number(identityTx.newToken.accessTokenTTL) === 0
-        ? undefined
-        : {
-            expiresIn: Number(identityTx.newToken.accessTokenTTL)
-          }
-    );
-
-    return {
-      accessToken,
-      identityUa,
-      validClientSecretInfo: identityTx.validClientSecretInfo,
-      identityAccessToken: identityTx.newToken,
-      identityMembershipOrg,
-      ...identityTx.accessTokenTTLParams
-    };
+      return {
+        accessToken,
+        identityUa,
+        validClientSecretInfo,
+        identityAccessToken,
+        identityMembershipOrg,
+        ...accessTokenTTLParams
+      };
+    } finally {
+      await lock.release();
+    }
   };
 
   const attachUniversalAuth = async ({

backend/src/services/org-membership/org-membership-dal.ts

@@ -153,10 +153,64 @@ export const orgMembershipDALFactory = (db: TDbClient) => {
     }
   };
 
+  const findOrgMembershipsWithUsersByOrgId = async (orgId: string) => {
+    try {
+      const members = await db
+        .replicaNode()(TableName.OrgMembership)
+        .where(`${TableName.OrgMembership}.orgId`, orgId)
+        .join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`)
+        .leftJoin<TUserEncryptionKeys>(
+          TableName.UserEncryptionKey,
+          `${TableName.UserEncryptionKey}.userId`,
+          `${TableName.Users}.id`
+        )
+        .leftJoin(TableName.IdentityMetadata, (queryBuilder) => {
+          void queryBuilder
+            .on(`${TableName.OrgMembership}.userId`, `${TableName.IdentityMetadata}.userId`)
+            .andOn(`${TableName.OrgMembership}.orgId`, `${TableName.IdentityMetadata}.orgId`);
+        })
+        .select(
+          db.ref("id").withSchema(TableName.OrgMembership),
+          db.ref("inviteEmail").withSchema(TableName.OrgMembership),
+          db.ref("orgId").withSchema(TableName.OrgMembership),
+          db.ref("role").withSchema(TableName.OrgMembership),
+          db.ref("roleId").withSchema(TableName.OrgMembership),
+          db.ref("status").withSchema(TableName.OrgMembership),
+          db.ref("isActive").withSchema(TableName.OrgMembership),
+          db.ref("email").withSchema(TableName.Users),
+          db.ref("username").withSchema(TableName.Users),
+          db.ref("firstName").withSchema(TableName.Users),
+          db.ref("lastName").withSchema(TableName.Users),
+          db.ref("isEmailVerified").withSchema(TableName.Users),
+          db.ref("id").withSchema(TableName.Users).as("userId")
+        )
+        .where({ isGhost: false });
+
+      return members.map((member) => ({
+        id: member.id,
+        orgId: member.orgId,
+        role: member.role,
+        status: member.status,
+        isActive: member.isActive,
+        inviteEmail: member.inviteEmail,
+        user: {
+          id: member.userId,
+          email: member.email,
+          username: member.username,
+          firstName: member.firstName,
+          lastName: member.lastName
+        }
+      }));
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find org memberships with users by org id" });
+    }
+  };
+
   return {
     ...orgMembershipOrm,
     findOrgMembershipById,
     findRecentInvitedMemberships,
-    updateLastInvitedAtByIds
+    updateLastInvitedAtByIds,
+    findOrgMembershipsWithUsersByOrgId
   };
 };

backend/src/services/secret-folder/secret-folder-service.ts

@@ -30,6 +30,7 @@ import {
   TDeleteFolderDTO,
   TDeleteManyFoldersDTO,
   TGetFolderByIdDTO,
+  TGetFolderByPathDTO,
   TGetFolderDTO,
   TGetFoldersDeepByEnvsDTO,
   TUpdateFolderDTO,
@@ -1398,6 +1399,31 @@ export const secretFolderServiceFactory = ({
     };
   };
 
+  const getFolderByPath = async (
+    { projectId, environment, secretPath }: TGetFolderByPathDTO,
+    actor: OrgServiceActor
+  ) => {
+    // folder check is allowed to be read by anyone
+    // permission is to check if user has access
+    await permissionService.getProjectPermission({
+      actor: actor.type,
+      actorId: actor.id,
+      projectId,
+      actorAuthMethod: actor.authMethod,
+      actorOrgId: actor.orgId,
+      actionProjectType: ActionProjectType.SecretManager
+    });
+
+    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
+
+    if (!folder)
+      throw new NotFoundError({
+        message: `Could not find folder with path "${secretPath}" in environment "${environment}" for project with ID "${projectId}"`
+      });
+
+    return folder;
+  };
+
   return {
     createFolder,
     updateFolder,
@@ -1405,6 +1431,7 @@ export const secretFolderServiceFactory = ({
     deleteFolder,
     getFolders,
     getFolderById,
+    getFolderByPath,
     getProjectFolderCount,
     getFoldersMultiEnv,
     getFoldersDeepByEnvs,

backend/src/services/secret-folder/secret-folder-types.ts

@@ -91,3 +91,9 @@ export type TDeleteManyFoldersDTO = {
     idOrName: string;
   }>;
 };
+
+export type TGetFolderByPathDTO = {
+  projectId: string;
+  environment: string;
+  secretPath: string;
+};

backend/src/services/secret-sync/hc-vault/hc-vault-sync-fns.ts

@@ -1,9 +1,13 @@
 import { isAxiosError } from "axios";
 
-import { request } from "@app/lib/config/request";
+import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
 import { removeTrailingSlash } from "@app/lib/fn";
-import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
-import { getHCVaultAccessToken, getHCVaultInstanceUrl } from "@app/services/app-connection/hc-vault";
+import {
+  getHCVaultAccessToken,
+  getHCVaultInstanceUrl,
+  requestWithHCVaultGateway,
+  THCVaultConnection
+} from "@app/services/app-connection/hc-vault";
 import {
   THCVaultListVariables,
   THCVaultListVariablesResponse,
@@ -14,19 +18,20 @@ import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
 import { matchesSchema } from "@app/services/secret-sync/secret-sync-fns";
 import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
 
-const listHCVaultVariables = async ({ instanceUrl, namespace, mount, accessToken, path }: THCVaultListVariables) => {
-  await blockLocalAndPrivateIpAddresses(instanceUrl);
-
+const listHCVaultVariables = async (
+  { instanceUrl, namespace, mount, accessToken, path }: THCVaultListVariables,
+  connection: THCVaultConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
   try {
-    const { data } = await request.get<THCVaultListVariablesResponse>(
-      `${instanceUrl}/v1/${removeTrailingSlash(mount)}/data/${path}`,
-      {
-        headers: {
-          "X-Vault-Token": accessToken,
-          ...(namespace ? { "X-Vault-Namespace": namespace } : {})
-        }
+    const { data } = await requestWithHCVaultGateway<THCVaultListVariablesResponse>(connection, gatewayService, {
+      url: `${instanceUrl}/v1/${removeTrailingSlash(mount)}/data/${path}`,
+      method: "GET",
+      headers: {
+        "X-Vault-Token": accessToken,
+        ...(namespace ? { "X-Vault-Namespace": namespace } : {})
       }
-    );
+    });
 
     return data.data.data;
   } catch (error: unknown) {
@@ -39,33 +44,29 @@ const listHCVaultVariables = async ({ instanceUrl, namespace, mount, accessToken
 };
 
 // Hashicorp Vault updates all variables in one batch. This is to respect their versioning
-const updateHCVaultVariables = async ({
-  path,
-  instanceUrl,
-  namespace,
-  accessToken,
-  mount,
-  data
-}: TPostHCVaultVariable) => {
-  await blockLocalAndPrivateIpAddresses(instanceUrl);
-
-  return request.post(
-    `${instanceUrl}/v1/${removeTrailingSlash(mount)}/data/${path}`,
-    {
-      data
+const updateHCVaultVariables = async (
+  { path, instanceUrl, namespace, accessToken, mount, data }: TPostHCVaultVariable,
+  connection: THCVaultConnection,
+  gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+) => {
+  return requestWithHCVaultGateway(connection, gatewayService, {
+    url: `${instanceUrl}/v1/${removeTrailingSlash(mount)}/data/${path}`,
+    method: "POST",
+    headers: {
+      "X-Vault-Token": accessToken,
+      ...(namespace ? { "X-Vault-Namespace": namespace } : {}),
+      "Content-Type": "application/json"
     },
-    {
-      headers: {
-        "X-Vault-Token": accessToken,
-        ...(namespace ? { "X-Vault-Namespace": namespace } : {}),
-        "Content-Type": "application/json"
-      }
-    }
-  );
+    data: { data }
+  });
 };
 
 export const HCVaultSyncFns = {
-  syncSecrets: async (secretSync: THCVaultSyncWithCredentials, secretMap: TSecretMap) => {
+  syncSecrets: async (
+    secretSync: THCVaultSyncWithCredentials,
+    secretMap: TSecretMap,
+    gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+  ) => {
     const {
       connection,
       environment,
@@ -74,16 +75,20 @@ export const HCVaultSyncFns = {
     } = secretSync;
 
     const { namespace } = connection.credentials;
-    const accessToken = await getHCVaultAccessToken(connection);
+    const accessToken = await getHCVaultAccessToken(connection, gatewayService);
     const instanceUrl = await getHCVaultInstanceUrl(connection);
 
-    const variables = await listHCVaultVariables({
-      instanceUrl,
-      accessToken,
-      namespace,
-      mount,
-      path
-    });
+    const variables = await listHCVaultVariables(
+      {
+        instanceUrl,
+        accessToken,
+        namespace,
+        mount,
+        path
+      },
+      connection,
+      gatewayService
+    );
     let tainted = false;
 
     for (const entry of Object.entries(secretMap)) {
@@ -110,24 +115,36 @@ export const HCVaultSyncFns = {
     if (!tainted) return;
 
     try {
-      await updateHCVaultVariables({ accessToken, instanceUrl, namespace, mount, path, data: variables });
+      await updateHCVaultVariables(
+        { accessToken, instanceUrl, namespace, mount, path, data: variables },
+        connection,
+        gatewayService
+      );
     } catch (error) {
       throw new SecretSyncError({
         error
       });
     }
   },
-  removeSecrets: async (secretSync: THCVaultSyncWithCredentials, secretMap: TSecretMap) => {
+  removeSecrets: async (
+    secretSync: THCVaultSyncWithCredentials,
+    secretMap: TSecretMap,
+    gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+  ) => {
     const {
       connection,
       destinationConfig: { mount, path }
     } = secretSync;
 
     const { namespace } = connection.credentials;
-    const accessToken = await getHCVaultAccessToken(connection);
+    const accessToken = await getHCVaultAccessToken(connection, gatewayService);
     const instanceUrl = await getHCVaultInstanceUrl(connection);
 
-    const variables = await listHCVaultVariables({ instanceUrl, namespace, accessToken, mount, path });
+    const variables = await listHCVaultVariables(
+      { instanceUrl, namespace, accessToken, mount, path },
+      connection,
+      gatewayService
+    );
 
     for await (const [key] of Object.entries(variables)) {
       if (key in secretMap) {
@@ -136,30 +153,41 @@ export const HCVaultSyncFns = {
     }
 
     try {
-      await updateHCVaultVariables({ accessToken, instanceUrl, namespace, mount, path, data: variables });
+      await updateHCVaultVariables(
+        { accessToken, instanceUrl, namespace, mount, path, data: variables },
+        connection,
+        gatewayService
+      );
     } catch (error) {
       throw new SecretSyncError({
         error
       });
     }
   },
-  getSecrets: async (secretSync: THCVaultSyncWithCredentials) => {
+  getSecrets: async (
+    secretSync: THCVaultSyncWithCredentials,
+    gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
+  ) => {
     const {
       connection,
       destinationConfig: { mount, path }
     } = secretSync;
 
     const { namespace } = connection.credentials;
-    const accessToken = await getHCVaultAccessToken(connection);
+    const accessToken = await getHCVaultAccessToken(connection, gatewayService);
     const instanceUrl = await getHCVaultInstanceUrl(connection);
 
-    const variables = await listHCVaultVariables({
-      instanceUrl,
-      namespace,
-      accessToken,
-      mount,
-      path
-    });
+    const variables = await listHCVaultVariables(
+      {
+        instanceUrl,
+        namespace,
+        accessToken,
+        mount,
+        path
+      },
+      connection,
+      gatewayService
+    );
 
     return Object.fromEntries(Object.entries(variables).map(([key, value]) => [key, { value }]));
   }

backend/src/services/secret-sync/secret-sync-fns.ts

@@ -244,7 +244,7 @@ export const SecretSyncFns = {
       case SecretSync.Windmill:
         return WindmillSyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.HCVault:
-        return HCVaultSyncFns.syncSecrets(secretSync, schemaSecretMap);
+        return HCVaultSyncFns.syncSecrets(secretSync, schemaSecretMap, gatewayService);
       case SecretSync.TeamCity:
         return TeamCitySyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.OCIVault:
@@ -283,7 +283,7 @@ export const SecretSyncFns = {
   },
   getSecrets: async (
     secretSync: TSecretSyncWithCredentials,
-    { kmsService, appConnectionDAL }: TSyncSecretDeps
+    { kmsService, appConnectionDAL, gatewayService }: TSyncSecretDeps
   ): Promise<TSecretMap> => {
     let secretMap: TSecretMap;
     switch (secretSync.destination) {
@@ -341,7 +341,7 @@ export const SecretSyncFns = {
         secretMap = await WindmillSyncFns.getSecrets(secretSync);
         break;
       case SecretSync.HCVault:
-        secretMap = await HCVaultSyncFns.getSecrets(secretSync);
+        secretMap = await HCVaultSyncFns.getSecrets(secretSync, gatewayService);
         break;
       case SecretSync.TeamCity:
         secretMap = await TeamCitySyncFns.getSecrets(secretSync);
@@ -451,7 +451,7 @@ export const SecretSyncFns = {
       case SecretSync.Windmill:
         return WindmillSyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.HCVault:
-        return HCVaultSyncFns.removeSecrets(secretSync, schemaSecretMap);
+        return HCVaultSyncFns.removeSecrets(secretSync, schemaSecretMap, gatewayService);
       case SecretSync.TeamCity:
         return TeamCitySyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.OCIVault: