Improvements on github bulk sync

backend/src/ee/routes/v1/github-org-sync-router.ts

@@ -135,9 +135,6 @@ export const registerGithubOrgSyncRouter = async (server: FastifyZodProvider) =>
     },
     onRequest: verifyAuth([AuthMode.JWT]),
     schema: {
-      body: z.object({
-        githubOrgAccessToken: z.string().trim().max(1000).optional()
-      }),
       response: {
         200: z.object({
           syncedUsersCount: z.number(),
@@ -152,8 +149,7 @@ export const registerGithubOrgSyncRouter = async (server: FastifyZodProvider) =>
     },
     handler: async (req) => {
       const result = await server.services.githubOrgSync.syncAllTeams({
-        orgPermission: req.permission,
+        orgPermission: req.permission
-        githubOrgAccessToken: req.body.githubOrgAccessToken
       });
 
       return {
@@ -167,40 +163,4 @@ export const registerGithubOrgSyncRouter = async (server: FastifyZodProvider) =>
       };
     }
   });
-
-  server.route({
-    url: "/validate-token",
-    method: "POST",
-    config: {
-      rateLimit: writeLimit
-    },
-    onRequest: verifyAuth([AuthMode.JWT]),
-    schema: {
-      body: z.object({
-        githubOrgAccessToken: z.string().trim().min(1, "GitHub access token is required").max(1000)
-      }),
-      response: {
-        200: z.object({
-          valid: z.boolean(),
-          organizationInfo: z
-            .object({
-              id: z.number(),
-              login: z.string(),
-              name: z.string(),
-              publicRepos: z.number().optional(),
-              privateRepos: z.number().optional()
-            })
-            .optional()
-        })
-      }
-    },
-    handler: async (req) => {
-      const result = await server.services.githubOrgSync.validateGithubToken({
-        orgPermission: req.permission,
-        githubOrgAccessToken: req.body.githubOrgAccessToken
-      });
-
-      return result;
-    }
-  });
 };

backend/src/ee/services/github-org-sync/github-org-sync-service.ts

@@ -4,12 +4,13 @@ import { ForbiddenError } from "@casl/ability";
 import { Octokit } from "@octokit/core";
 import { paginateGraphql } from "@octokit/plugin-paginate-graphql";
 import { Octokit as OctokitRest } from "@octokit/rest";
+import RE2 from "re2";
 
 import { OrgMembershipRole } from "@app/db/schemas";
 import { BadRequestError, NotFoundError } from "@app/lib/errors";
 import { groupBy } from "@app/lib/fn";
 import { logger } from "@app/lib/logger";
-import { TIdentityMetadataDALFactory } from "@app/services/identity/identity-metadata-dal";
+import { retryWithBackoff } from "@app/lib/retry";
 import { TKmsServiceFactory } from "@app/services/kms/kms-service";
 import { KmsDataKey } from "@app/services/kms/kms-types";
 import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal";
@@ -42,18 +43,45 @@ interface GitHubApiError extends Error {
   };
 }
 
+interface OrgMembershipWithUser {
+  id: string;
+  orgId: string;
+  role: string;
+  status: string;
+  isActive: boolean;
+  inviteEmail: string | null;
+  user: {
+    id: string;
+    email: string;
+    username: string | null;
+    firstName: string | null;
+    lastName: string | null;
+  } | null;
+}
+
+interface GroupMembership {
+  id: string;
+  groupId: string;
+  groupName: string;
+  orgMembershipId: string;
+  firstName: string | null;
+  lastName: string | null;
+}
+
 type TGithubOrgSyncServiceFactoryDep = {
   githubOrgSyncDAL: TGithubOrgSyncDALFactory;
   permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
   kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
   userGroupMembershipDAL: Pick<
     TUserGroupMembershipDALFactory,
-    "findGroupMembershipsByUserIdInOrg" | "insertMany" | "delete"
+    "findGroupMembershipsByUserIdInOrg" | "findGroupMembershipsByGroupIdInOrg" | "insertMany" | "delete"
   >;
   groupDAL: Pick<TGroupDALFactory, "insertMany" | "transaction" | "find">;
   licenseService: Pick<TLicenseServiceFactory, "getPlan">;
-  orgMembershipDAL: Pick<TOrgMembershipDALFactory, "find">;
+  orgMembershipDAL: Pick<
-  identityMetadataDAL: TIdentityMetadataDALFactory;
+    TOrgMembershipDALFactory,
+    "find" | "findOrgMembershipById" | "findOrgMembershipsWithUsersByOrgId"
+  >;
 };
 
 export type TGithubOrgSyncServiceFactory = ReturnType<typeof githubOrgSyncServiceFactory>;
@@ -65,8 +93,7 @@ export const githubOrgSyncServiceFactory = ({
   userGroupMembershipDAL,
   groupDAL,
   licenseService,
-  orgMembershipDAL,
+  orgMembershipDAL
-  identityMetadataDAL
 }: TGithubOrgSyncServiceFactoryDep) => {
   const createGithubOrgSync = async ({
     githubOrgName,
@@ -444,7 +471,8 @@ export const githubOrgSyncServiceFactory = ({
         }
         if (statusCode === 403) {
           throw new BadRequestError({
-            message: "GitHub access token lacks required permissions. Ensure it has 'read:org' and 'read:user' scopes."
+            message:
+              "GitHub access token lacks required permissions. Required: 1) 'read:org' scope for organization teams, 2) Token owner must be an organization member with team visibility access, 3) Organization settings must allow team visibility. Check GitHub token scopes and organization member permissions."
           });
         }
         if (statusCode === 404) {
@@ -460,7 +488,7 @@ export const githubOrgSyncServiceFactory = ({
     }
   };
 
-  const syncAllTeams = async ({ orgPermission, githubOrgAccessToken }: TSyncAllTeamsDTO): Promise<TSyncResult> => {
+  const syncAllTeams = async ({ orgPermission }: TSyncAllTeamsDTO): Promise<TSyncResult> => {
     const { permission } = await permissionService.getOrgPermission(
       orgPermission.type,
       orgPermission.id,
@@ -469,7 +497,10 @@ export const githubOrgSyncServiceFactory = ({
       orgPermission.orgId
     );
 
-    ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.GithubOrgSync);
+    ForbiddenError.from(permission).throwUnlessCan(
+      OrgPermissionActions.Edit,
+      OrgPermissionSubjects.GithubOrgSyncManual
+    );
 
     const plan = await licenseService.getPlan(orgPermission.orgId);
     if (!plan.githubOrgSync) {
@@ -484,27 +515,19 @@ export const githubOrgSyncServiceFactory = ({
       throw new BadRequestError({ message: "GitHub organization sync is not configured or not active" });
     }
 
-    const { encryptor, decryptor } = await kmsService.createCipherPairWithDataKey({
+    const { decryptor } = await kmsService.createCipherPairWithDataKey({
       type: KmsDataKey.Organization,
       orgId: orgPermission.orgId
     });
 
-    let orgAccessToken: string;
+    if (!config.encryptedGithubOrgAccessToken) {
-    let shouldUpdateStoredToken = false;
-
-    // If a new token is provided, use it and update the stored token
-    if (githubOrgAccessToken) {
-      orgAccessToken = githubOrgAccessToken;
-      shouldUpdateStoredToken = true;
-    } else if (config.encryptedGithubOrgAccessToken) {
-      // Use the stored token
-      orgAccessToken = decryptor({ cipherTextBlob: config.encryptedGithubOrgAccessToken }).toString();
-    } else {
       throw new BadRequestError({
-        message: "GitHub organization access token is required for bulk sync. Please provide a token."
+        message: "GitHub organization access token is required. Please set a token first."
       });
     }
 
+    const orgAccessToken = decryptor({ cipherTextBlob: config.encryptedGithubOrgAccessToken }).toString();
+
     try {
       const testOctokit = new OctokitRest({
         auth: orgAccessToken,
@@ -517,81 +540,21 @@ export const githubOrgSyncServiceFactory = ({
         org: config.githubOrgName
       });
 
-      if (shouldUpdateStoredToken) {
+      await testOctokit.rest.users.getAuthenticated();
-        await githubOrgSyncDAL.updateById(config.id, {
-          encryptedGithubOrgAccessToken: encryptor({ plainText: Buffer.from(orgAccessToken) }).cipherTextBlob
-        });
-      }
     } catch (error) {
-      if (!githubOrgAccessToken && config.encryptedGithubOrgAccessToken) {
-        throw new BadRequestError({
-          message: "Stored GitHub access token is invalid or expired. Please provide a new token."
-        });
-      }
       throw new BadRequestError({
-        message: `Invalid GitHub access token or insufficient permissions: ${(error as Error).message}`
+        message: "Stored GitHub access token is invalid or expired. Please set a new token."
       });
     }
 
-    // Get all organization members
+    const allMembers = await orgMembershipDAL.findOrgMembershipsWithUsersByOrgId(orgPermission.orgId);
-    const orgMembers = await orgMembershipDAL.find({ orgId: orgPermission.orgId });
+    const activeMembers = allMembers.filter(
-    const activeMembers = orgMembers.filter((member) => member.status === "accepted" && member.isActive);
+      (member) => member.status === "accepted" && member.isActive
-
+    ) as OrgMembershipWithUser[];
-    // Get GitHub usernames from metadata for all users
-    const userMetadata = await identityMetadataDAL.find({
-      orgId: orgPermission.orgId,
-      key: "github_username"
-    });
-
-    const githubUsernameMap = new Map<string, string>();
-    userMetadata.forEach((meta) => {
-      if (meta.userId) {
-        githubUsernameMap.set(meta.userId, meta.value);
-      }
-    });
 
     const startTime = Date.now();
     let syncedUsersCount = 0;
     const syncErrors: string[] = [];
-    const createdTeams = new Set<string>();
-    const updatedTeams = new Set<string>();
-    let totalRemovedMemberships = 0;
-
-    const delay = (ms: number) =>
-      new Promise<void>((resolve) => {
-        setTimeout(() => resolve(), ms);
-      });
-
-    const retryWithBackoff = async <T>(fn: () => Promise<T>, maxRetries = 3, baseDelay = 1000): Promise<T> => {
-      let lastError: Error;
-
-      for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
-        try {
-          return await fn();
-        } catch (error) {
-          lastError = error as Error;
-          const gitHubError = error as GitHubApiError;
-          const statusCode = gitHubError.status || gitHubError.response?.status;
-          if (statusCode === 403) {
-            const rateLimitReset = gitHubError.response?.headers?.["x-ratelimit-reset"];
-            if (rateLimitReset) {
-              const resetTime = parseInt(rateLimitReset, 10) * 1000;
-              const waitTime = Math.max(resetTime - Date.now(), baseDelay);
-              logger.warn(`Rate limit hit, waiting ${waitTime}ms until reset`);
-              await delay(Math.min(waitTime, 60000)); // Cap at 1 minute
-            } else {
-              await delay(baseDelay * 2 ** attempt);
-            }
-          } else if (attempt < maxRetries) {
-            await delay(baseDelay * 2 ** attempt);
-          }
-        }
-      }
-
-      throw lastError!;
-    };
-
-    const RATE_LIMIT_DELAY = 150;
 
     const octokit = new OctokitWithPlugin({
       auth: orgAccessToken,
@@ -600,29 +563,44 @@ export const githubOrgSyncServiceFactory = ({
       }
     });
 
-    const syncUserGroupsWithStoredUsername = async (
+    const data = await retryWithBackoff(async () => {
-      orgId: string,
+      return octokit.graphql
-      userId: string,
-      githubUsername: string,
-      githubOrgName: string,
-      octokitInstance: InstanceType<typeof OctokitWithPlugin>
-    ): Promise<{ createdTeams: string[]; updatedTeams: string[]; removedMemberships: number } | null> => {
-      const infisicalUserGroups = await userGroupMembershipDAL.findGroupMembershipsByUserIdInOrg(userId, orgId);
-      const infisicalUserGroupSet = new Set(infisicalUserGroups.map((el) => el.groupName));
-
-      const data = await octokitInstance.graphql
         .paginate<{
-          organization: { teams: { totalCount: number; edges: { node: { name: string; description: string } }[] } };
+          organization: {
+            teams: {
+              totalCount: number;
+              edges: {
+                node: {
+                  name: string;
+                  description: string;
+                  members: {
+                    edges: {
+                      node: {
+                        login: string;
+                      };
+                    }[];
+                  };
+                };
+              }[];
+            };
+          };
         }>(
           `
-        query orgTeams($cursor: String,$org: String!, $username: String!){
+        query orgTeams($cursor: String, $org: String!) {
           organization(login: $org) {
-            teams(first: 100, userLogins: [$username], after: $cursor) {
+            teams(first: 100, after: $cursor) {
               totalCount
               edges {
                 node {
                   name
                   description
+                  members(first: 100) {
+                    edges {
+                      node {
+                        login
+                      }
+                    }
+                  }
                 }
               }
               pageInfo {
@@ -634,12 +612,11 @@ export const githubOrgSyncServiceFactory = ({
         }
         `,
           {
-            org: githubOrgName,
+            org: config.githubOrgName
-            username: githubUsername
           }
         )
         .catch((err) => {
-          logger.error(err, `GitHub GraphQL error for user ${githubUsername}`);
+          logger.error(err, "GitHub GraphQL error for batched team sync");
 
           const gitHubError = err as GitHubApiError;
           const statusCode = gitHubError.status || gitHubError.response?.status;
@@ -652,12 +629,12 @@ export const githubOrgSyncServiceFactory = ({
             if (statusCode === 403) {
               throw new BadRequestError({
                 message:
-                  "GitHub access token lacks required permissions. Please ensure the token has 'read:org' and 'read:user' scopes."
+                  "GitHub access token lacks required permissions for organization team sync. Required: 1) 'admin:org' scope, 2) Token owner must be organization owner or have team read permissions, 3) Organization settings must allow team visibility. Check token scopes and user role."
               });
             }
             if (statusCode === 404) {
               throw new BadRequestError({
-                message: `Organization ${config.githubOrgName} not found or user ${githubUsername} is not a member.`
+                message: `Organization ${config.githubOrgName} not found or access token does not have sufficient permissions to read it.`
               });
             }
           }
@@ -665,155 +642,169 @@ export const githubOrgSyncServiceFactory = ({
           if ((err as Error)?.message?.includes("Although you appear to have the correct authorization credential")) {
             throw new BadRequestError({
               message:
-                "Please check your organization have approved Infisical Oauth application. For more info: https://infisical.com/docs/documentation/platform/github-org-sync#troubleshooting"
+                "Organization has restricted OAuth app access. Please check that: 1) Your organization has approved the Infisical OAuth application, 2) The token owner has sufficient organization permissions."
             });
           }
-          throw new BadRequestError({ message: (err as Error)?.message });
+          throw new BadRequestError({ message: `GitHub GraphQL query failed: ${(err as Error)?.message}` });
         });
+    });
 
-      const {
+    const {
-        organization: { teams }
+      organization: { teams }
-      } = data;
+    } = data;
-      const githubUserTeams = teams?.edges?.map((el) => el.node.name.toLowerCase()) || [];
-      const githubUserTeamSet = new Set(githubUserTeams);
-      const githubUserTeamOnInfisical = await groupDAL.find({ orgId, $in: { name: githubUserTeams } });
-      const githubUserTeamOnInfisicalGroupByName = groupBy(githubUserTeamOnInfisical, (i) => i.name);
 
-      const newTeams = githubUserTeams.filter(
+    const userTeamMap = new Map<string, string[]>();
-        (el) => !infisicalUserGroupSet.has(el) && !(el in githubUserTeamOnInfisicalGroupByName)
+    const allGithubUsernamesInTeams = new Set<string>();
-      );
-      const updateTeams = githubUserTeams.filter(
-        (el) => !infisicalUserGroupSet.has(el) && el in githubUserTeamOnInfisicalGroupByName
-      );
-      const removeFromTeams = infisicalUserGroups.filter((el) => !githubUserTeamSet.has(el.groupName));
 
-      if (newTeams.length || updateTeams.length || removeFromTeams.length) {
+    teams?.edges?.forEach((teamEdge) => {
-        const result = {
+      const teamName = teamEdge.node.name.toLowerCase();
-          createdTeams: [] as string[],
-          updatedTeams: [] as string[],
-          removedMemberships: 0
-        };
 
-        try {
+      teamEdge.node.members.edges.forEach((memberEdge) => {
-          if (newTeams.length) {
+        const username = memberEdge.node.login.toLowerCase();
-            await groupDAL.transaction(async (tx) => {
+        allGithubUsernamesInTeams.add(username);
-              logger.info({ userId, githubUsername, newTeams, orgId }, "Creating new teams for user");
 
-              const newGroups = await groupDAL.insertMany(
+        if (!userTeamMap.has(username)) {
-                newTeams.map((newGroupName) => ({
+          userTeamMap.set(username, []);
-                  name: newGroupName,
-                  role: OrgMembershipRole.Member,
-                  slug: newGroupName,
-                  orgId
-                })),
-                tx
-              );
-
-              await userGroupMembershipDAL.insertMany(
-                newGroups.map((el) => ({
-                  groupId: el.id,
-                  userId
-                })),
-                tx
-              );
-            });
-
-            result.createdTeams = newTeams;
-          }
-
-          if (updateTeams.length) {
-            await groupDAL.transaction(async (tx) => {
-              logger.info({ userId, githubUsername, updateTeams, orgId }, "Adding user to existing teams");
-
-              await userGroupMembershipDAL.insertMany(
-                updateTeams.map((el) => ({
-                  groupId: githubUserTeamOnInfisicalGroupByName[el][0].id,
-                  userId
-                })),
-                tx
-              );
-            });
-
-            result.updatedTeams = updateTeams;
-          }
-
-          if (removeFromTeams.length) {
-            await groupDAL.transaction(async (tx) => {
-              logger.info(
-                { userId, githubUsername, removeFromTeams: removeFromTeams.map((t) => t.groupName), orgId },
-                "Removing user from teams"
-              );
-
-              await userGroupMembershipDAL.delete(
-                { userId, $in: { groupId: removeFromTeams.map((el) => el.groupId) } },
-                tx
-              );
-            });
-
-            result.removedMemberships = removeFromTeams.length;
-          }
-
-          return result;
-        } catch (error) {
-          logger.error(error, `Failed to update team memberships for user ${userId} (${githubUsername})`);
-          throw error;
         }
+        userTeamMap.get(username)!.push(teamName);
+      });
+    });
+
+    const allGithubTeamNames = Array.from(new Set(teams?.edges?.map((edge) => edge.node.name.toLowerCase()) || []));
+
+    const existingTeamsOnInfisical = await groupDAL.find({
+      orgId: orgPermission.orgId,
+      $in: { name: allGithubTeamNames }
+    });
+    const existingTeamsMap = groupBy(existingTeamsOnInfisical, (i) => i.name);
+
+    const teamsToCreate = allGithubTeamNames.filter((teamName) => !(teamName in existingTeamsMap));
+    const createdTeams = new Set<string>();
+    const updatedTeams = new Set<string>();
+    const totalRemovedMemberships = 0;
+
+    syncedUsersCount = allGithubUsernamesInTeams.size;
+
+    await groupDAL.transaction(async (tx) => {
+      if (teamsToCreate.length > 0) {
+        const newGroups = await groupDAL.insertMany(
+          teamsToCreate.map((teamName) => ({
+            name: teamName,
+            role: OrgMembershipRole.Member,
+            slug: teamName,
+            orgId: orgPermission.orgId
+          })),
+          tx
+        );
+
+        newGroups.forEach((group) => {
+          if (!existingTeamsMap[group.name]) {
+            existingTeamsMap[group.name] = [];
+          }
+          existingTeamsMap[group.name].push(group);
+          createdTeams.add(group.name);
+        });
       }
 
-      return null;
+      const allTeams = [...Object.values(existingTeamsMap).flat()];
-    };
 
-    for (const member of activeMembers) {
+      for (const team of allTeams) {
-      try {
+        const teamName = team.name.toLowerCase();
-        if (!member.userId) {
-          // eslint-disable-next-line no-continue
-          continue;
-        }
 
-        const githubUsername = githubUsernameMap.get(member.userId);
+        const currentMemberships = (await userGroupMembershipDAL.findGroupMembershipsByGroupIdInOrg(
+          team.id,
+          orgPermission.orgId
+        )) as GroupMembership[];
 
-        if (!githubUsername) {
+        const expectedUserIds = new Set<string>();
-          // eslint-disable-next-line no-continue
+        teams?.edges?.forEach((teamEdge) => {
-          continue;
+          if (teamEdge.node.name.toLowerCase() === teamName) {
-        }
+            teamEdge.node.members.edges.forEach((memberEdge) => {
+              const githubUsername = memberEdge.node.login.toLowerCase();
 
-        const syncResult = await retryWithBackoff(async () => {
+              const matchingMember = activeMembers.find((member) => {
-          return syncUserGroupsWithStoredUsername(
+                const email = member.user?.email || member.inviteEmail;
-            orgPermission.orgId,
+                if (!email) return false;
-            member.userId!,
+
-            githubUsername,
+                const emailPrefix = email.split("@")[0].toLowerCase();
-            config.githubOrgName,
+                const emailDomain = email.split("@")[1].toLowerCase();
-            octokit
+
+                if (emailPrefix === githubUsername) {
+                  return true;
+                }
+                const domainName = emailDomain.split(".")[0];
+                if (githubUsername.endsWith(domainName) && githubUsername.length > domainName.length) {
+                  const baseUsername = githubUsername.slice(0, -domainName.length);
+                  if (emailPrefix === baseUsername) {
+                    return true;
+                  }
+                }
+                const emailSplitRegex = new RE2(/[._-]/);
+                const emailParts = emailPrefix.split(emailSplitRegex);
+                const longestEmailPart = emailParts.reduce((a, b) => (a.length > b.length ? a : b), "");
+                if (longestEmailPart.length >= 4 && githubUsername.includes(longestEmailPart)) {
+                  return true;
+                }
+                return false;
+              });
+
+              if (matchingMember?.user?.id) {
+                expectedUserIds.add(matchingMember.user.id);
+                logger.info(
+                  `Matched GitHub user ${githubUsername} to email ${matchingMember.user?.email || matchingMember.inviteEmail}`
+                );
+              }
+            });
+          }
+        });
+
+        const currentUserIds = new Set<string>();
+        currentMemberships.forEach((membership) => {
+          const activeMember = activeMembers.find((am) => am.id === membership.orgMembershipId);
+          if (activeMember?.user?.id) {
+            currentUserIds.add(activeMember.user.id);
+          }
+        });
+
+        const usersToAdd = Array.from(expectedUserIds).filter((userId) => !currentUserIds.has(userId));
+
+        const membershipsToRemove = currentMemberships.filter((membership) => {
+          const activeMember = activeMembers.find((am) => am.id === membership.orgMembershipId);
+          return activeMember?.user?.id && !expectedUserIds.has(activeMember.user.id);
+        });
+
+        if (usersToAdd.length > 0) {
+          await userGroupMembershipDAL.insertMany(
+            usersToAdd.map((userId) => ({
+              userId,
+              groupId: team.id
+            })),
+            tx
           );
-        });
+          updatedTeams.add(teamName);
-
-        if (syncResult) {
-          syncResult.createdTeams.forEach((team) => createdTeams.add(team));
-          syncResult.updatedTeams.forEach((team) => updatedTeams.add(team));
-          totalRemovedMemberships += syncResult.removedMemberships;
         }
 
-        syncedUsersCount += 1;
+        if (membershipsToRemove.length > 0) {
-
+          await userGroupMembershipDAL.delete(
-        await delay(RATE_LIMIT_DELAY);
+            {
-      } catch (error) {
+              $in: {
-        logger.error(error, `Failed to sync teams for user ${member.userId || "unknown"}`);
+                id: membershipsToRemove.map((m) => m.id)
-        syncErrors.push(`User ${member.userId || "unknown"}: ${(error as Error).message}`);
+              }
+            },
+            tx
+          );
+          updatedTeams.add(teamName);
+        }
       }
-    }
+    });
 
     const syncDuration = Date.now() - startTime;
 
     logger.info(
       {
         orgId: orgPermission.orgId,
-        syncedUsersCount,
-        totalUsers: activeMembers.length,
         createdTeams: createdTeams.size,
-        updatedTeams: updatedTeams.size,
+        syncDuration
-        removedMemberships: totalRemovedMemberships,
-        syncDuration,
-        errorCount: syncErrors.length
       },
       "GitHub team sync completed"
     );

backend/src/ee/services/github-org-sync/github-org-sync-types.ts

@@ -24,7 +24,6 @@ export interface TGetGithubOrgSyncDTO {
 
 export interface TSyncAllTeamsDTO {
   orgPermission: OrgServiceActor;
-  githubOrgAccessToken?: string;
 }
 
 export interface TSyncResult {

backend/src/ee/services/license/license-fns.ts

@@ -18,51 +18,51 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({
   environmentsUsed: 0,
   identityLimit: null,
   identitiesUsed: 0,
-  dynamicSecret: false,
+  dynamicSecret: true,
   secretVersioning: true,
-  pitRecovery: false,
+  pitRecovery: true,
-  ipAllowlisting: false,
+  ipAllowlisting: true,
-  rbac: false,
+  rbac: true,
-  githubOrgSync: false,
+  githubOrgSync: true,
-  customRateLimits: false,
+  customRateLimits: true,
-  customAlerts: false,
+  customAlerts: true,
-  secretAccessInsights: false,
+  secretAccessInsights: true,
-  auditLogs: false,
+  auditLogs: true,
   auditLogsRetentionDays: 0,
-  auditLogStreams: false,
+  auditLogStreams: true,
   auditLogStreamLimit: 3,
-  samlSSO: false,
+  samlSSO: true,
-  enforceGoogleSSO: false,
+  enforceGoogleSSO: true,
-  hsm: false,
+  hsm: true,
-  oidcSSO: false,
+  oidcSSO: true,
-  scim: false,
+  scim: true,
-  ldap: false,
+  ldap: true,
-  groups: false,
+  groups: true,
   status: null,
   trial_end: null,
   has_used_trial: true,
-  secretApproval: false,
+  secretApproval: true,
-  secretRotation: false,
+  secretRotation: true,
-  caCrl: false,
+  caCrl: true,
-  instanceUserManagement: false,
+  instanceUserManagement: true,
-  externalKms: false,
+  externalKms: true,
   rateLimits: {
     readLimit: 60,
     writeLimit: 200,
     secretsLimit: 40
   },
-  pkiEst: false,
+  pkiEst: true,
-  enforceMfa: false,
+  enforceMfa: true,
-  projectTemplates: false,
+  projectTemplates: true,
-  kmip: false,
+  kmip: true,
-  gateway: false,
+  gateway: true,
-  sshHostGroups: false,
+  sshHostGroups: true,
-  secretScanning: false,
+  secretScanning: true,
-  enterpriseSecretSyncs: false,
+  enterpriseSecretSyncs: true,
-  enterpriseAppConnections: false,
+  enterpriseAppConnections: true,
-  fips: false,
+  fips: true,
-  eventSubscriptions: false,
+  eventSubscriptions: true,
-  machineIdentityAuthTemplates: false
+  machineIdentityAuthTemplates: true
 });
 
 export const setupLicenseRequestWithStore = (

backend/src/ee/services/permission/org-permission.ts

@@ -90,6 +90,7 @@ export enum OrgPermissionSubjects {
   Sso = "sso",
   Scim = "scim",
   GithubOrgSync = "github-org-sync",
+  GithubOrgSyncManual = "github-org-sync-manual",
   Ldap = "ldap",
   Groups = "groups",
   Billing = "billing",
@@ -119,6 +120,7 @@ export type OrgPermissionSet =
   | [OrgPermissionActions, OrgPermissionSubjects.Sso]
   | [OrgPermissionActions, OrgPermissionSubjects.Scim]
   | [OrgPermissionActions, OrgPermissionSubjects.GithubOrgSync]
+  | [OrgPermissionActions, OrgPermissionSubjects.GithubOrgSyncManual]
   | [OrgPermissionActions, OrgPermissionSubjects.Ldap]
   | [OrgPermissionGroupActions, OrgPermissionSubjects.Groups]
   | [OrgPermissionActions, OrgPermissionSubjects.SecretScanning]
@@ -188,6 +190,10 @@ export const OrgPermissionSchema = z.discriminatedUnion("subject", [
     subject: z.literal(OrgPermissionSubjects.GithubOrgSync).describe("The entity this permission pertains to."),
     action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.")
   }),
+  z.object({
+    subject: z.literal(OrgPermissionSubjects.GithubOrgSyncManual).describe("The entity this permission pertains to."),
+    action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.")
+  }),
   z.object({
     subject: z.literal(OrgPermissionSubjects.Ldap).describe("The entity this permission pertains to."),
     action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.")
@@ -309,6 +315,11 @@ const buildAdminPermission = () => {
   can(OrgPermissionActions.Edit, OrgPermissionSubjects.GithubOrgSync);
   can(OrgPermissionActions.Delete, OrgPermissionSubjects.GithubOrgSync);
 
+  can(OrgPermissionActions.Read, OrgPermissionSubjects.GithubOrgSyncManual);
+  can(OrgPermissionActions.Create, OrgPermissionSubjects.GithubOrgSyncManual);
+  can(OrgPermissionActions.Edit, OrgPermissionSubjects.GithubOrgSyncManual);
+  can(OrgPermissionActions.Delete, OrgPermissionSubjects.GithubOrgSyncManual);
+
   can(OrgPermissionActions.Read, OrgPermissionSubjects.Ldap);
   can(OrgPermissionActions.Create, OrgPermissionSubjects.Ldap);
   can(OrgPermissionActions.Edit, OrgPermissionSubjects.Ldap);

backend/src/lib/retry/index.ts

@@ -0,0 +1,43 @@
+/* eslint-disable no-await-in-loop */
+interface GitHubApiError extends Error {
+  status?: number;
+  response?: {
+    status?: number;
+    headers?: {
+      "x-ratelimit-reset"?: string;
+    };
+  };
+}
+
+const delay = (ms: number) =>
+  new Promise<void>((resolve) => {
+    setTimeout(() => resolve(), ms);
+  });
+
+export const retryWithBackoff = async <T>(fn: () => Promise<T>, maxRetries = 3, baseDelay = 1000): Promise<T> => {
+  let lastError: Error;
+
+  for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error as Error;
+      const gitHubError = error as GitHubApiError;
+      const statusCode = gitHubError.status || gitHubError.response?.status;
+      if (statusCode === 403) {
+        const rateLimitReset = gitHubError.response?.headers?.["x-ratelimit-reset"];
+        if (rateLimitReset) {
+          const resetTime = parseInt(rateLimitReset, 10) * 1000;
+          const waitTime = Math.max(resetTime - Date.now(), baseDelay);
+          await delay(Math.min(waitTime, 60000));
+        } else {
+          await delay(baseDelay * 2 ** attempt);
+        }
+      } else if (attempt < maxRetries) {
+        await delay(baseDelay * 2 ** attempt);
+      }
+    }
+  }
+
+  throw lastError!;
+};

backend/src/server/routes/index.ts

@@ -681,8 +681,7 @@ export const registerRoutes = async (
     permissionService,
     groupDAL,
     userGroupMembershipDAL,
-    orgMembershipDAL,
+    orgMembershipDAL
-    identityMetadataDAL
   });
 
   const ldapService = ldapConfigServiceFactory({

backend/src/services/org-membership/org-membership-dal.ts

@@ -153,10 +153,64 @@ export const orgMembershipDALFactory = (db: TDbClient) => {
     }
   };
 
+  const findOrgMembershipsWithUsersByOrgId = async (orgId: string) => {
+    try {
+      const members = await db
+        .replicaNode()(TableName.OrgMembership)
+        .where(`${TableName.OrgMembership}.orgId`, orgId)
+        .join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`)
+        .leftJoin<TUserEncryptionKeys>(
+          TableName.UserEncryptionKey,
+          `${TableName.UserEncryptionKey}.userId`,
+          `${TableName.Users}.id`
+        )
+        .leftJoin(TableName.IdentityMetadata, (queryBuilder) => {
+          void queryBuilder
+            .on(`${TableName.OrgMembership}.userId`, `${TableName.IdentityMetadata}.userId`)
+            .andOn(`${TableName.OrgMembership}.orgId`, `${TableName.IdentityMetadata}.orgId`);
+        })
+        .select(
+          db.ref("id").withSchema(TableName.OrgMembership),
+          db.ref("inviteEmail").withSchema(TableName.OrgMembership),
+          db.ref("orgId").withSchema(TableName.OrgMembership),
+          db.ref("role").withSchema(TableName.OrgMembership),
+          db.ref("roleId").withSchema(TableName.OrgMembership),
+          db.ref("status").withSchema(TableName.OrgMembership),
+          db.ref("isActive").withSchema(TableName.OrgMembership),
+          db.ref("email").withSchema(TableName.Users),
+          db.ref("username").withSchema(TableName.Users),
+          db.ref("firstName").withSchema(TableName.Users),
+          db.ref("lastName").withSchema(TableName.Users),
+          db.ref("isEmailVerified").withSchema(TableName.Users),
+          db.ref("id").withSchema(TableName.Users).as("userId")
+        )
+        .where({ isGhost: false });
+
+      return members.map((member) => ({
+        id: member.id,
+        orgId: member.orgId,
+        role: member.role,
+        status: member.status,
+        isActive: member.isActive,
+        inviteEmail: member.inviteEmail,
+        user: {
+          id: member.userId,
+          email: member.email,
+          username: member.username,
+          firstName: member.firstName,
+          lastName: member.lastName
+        }
+      }));
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find org memberships with users by org id" });
+    }
+  };
+
   return {
     ...orgMembershipOrm,
     findOrgMembershipById,
     findRecentInvitedMemberships,
-    updateLastInvitedAtByIds
+    updateLastInvitedAtByIds,
+    findOrgMembershipsWithUsersByOrgId
   };
 };

frontend/src/context/OrgPermissionContext/types.ts

@@ -52,6 +52,7 @@ export enum OrgPermissionSubjects {
   Gateway = "gateway",
   SecretShare = "secret-share",
   GithubOrgSync = "github-org-sync",
+  GithubOrgSyncManual = "github-org-sync-manual",
   MachineIdentityAuthTemplate = "machine-identity-auth-template"
 }
 
@@ -111,6 +112,7 @@ export type OrgPermissionSet =
   | [OrgPermissionActions, OrgPermissionSubjects.IncidentAccount]
   | [OrgPermissionActions, OrgPermissionSubjects.Scim]
   | [OrgPermissionActions, OrgPermissionSubjects.GithubOrgSync]
+  | [OrgPermissionActions, OrgPermissionSubjects.GithubOrgSyncManual]
   | [OrgPermissionActions, OrgPermissionSubjects.Sso]
   | [OrgPermissionActions, OrgPermissionSubjects.Ldap]
   | [OrgPermissionGroupActions, OrgPermissionSubjects.Groups]

frontend/src/hooks/api/githubOrgSyncConfig/index.tsx

@@ -2,7 +2,6 @@ export {
   useCreateGithubSyncOrgConfig,
   useDeleteGithubSyncOrgConfig,
   useSyncAllGithubTeams,
-  useUpdateGithubSyncOrgConfig,
+  useUpdateGithubSyncOrgConfig
-  useValidateGithubToken
 } from "./mutations";
 export { githubOrgSyncConfigQueryKeys } from "./queries";

frontend/src/hooks/api/githubOrgSyncConfig/mutations.tsx

@@ -43,11 +43,7 @@ export const useDeleteGithubSyncOrgConfig = () => {
 
 export const useSyncAllGithubTeams = () => {
   return useMutation({
-    mutationFn: async ({
+    mutationFn: async (): Promise<{
-      githubOrgAccessToken
-    }: {
-      githubOrgAccessToken?: string;
-    } = {}): Promise<{
       syncedUsersCount: number;
       totalUsers: number;
       errors: string[];
@@ -56,33 +52,7 @@ export const useSyncAllGithubTeams = () => {
       removedMemberships: number;
       syncDuration: number;
     }> => {
-      const response = await apiRequest.post("/api/v1/github-org-sync-config/sync-all-teams", {
+      const response = await apiRequest.post("/api/v1/github-org-sync-config/sync-all-teams");
-        githubOrgAccessToken
-      });
-      return response.data;
-    }
-  });
-};
-
-export const useValidateGithubToken = () => {
-  return useMutation({
-    mutationFn: async ({
-      githubOrgAccessToken
-    }: {
-      githubOrgAccessToken: string;
-    }): Promise<{
-      valid: boolean;
-      organizationInfo?: {
-        id: number;
-        login: string;
-        name: string;
-        publicRepos?: number;
-        privateRepos?: number;
-      };
-    }> => {
-      const response = await apiRequest.post("/api/v1/github-org-sync-config/validate-token", {
-        githubOrgAccessToken
-      });
       return response.data;
     }
   });

frontend/src/pages/organization/SettingsPage/components/OrgProvisioningTab/OrgGithubSyncSection.tsx

@@ -20,8 +20,7 @@ import { OrgPermissionActions, OrgPermissionSubjects, useSubscription } from "@a
 import {
   githubOrgSyncConfigQueryKeys,
   useSyncAllGithubTeams,
-  useUpdateGithubSyncOrgConfig,
+  useUpdateGithubSyncOrgConfig
-  useValidateGithubToken
 } from "@app/hooks/api";
 import { usePopUp } from "@app/hooks/usePopUp";
 
@@ -33,11 +32,10 @@ export const OrgGithubSyncSection = () => {
     "upgradePlan",
     "githubOrgSyncConfig",
     "deleteGithubOrgSyncConfig",
-    "syncAllTeamsToken"
+    "setAccessToken"
   ] as const);
 
   const [accessToken, setAccessToken] = useState("");
-  const [isValidatingToken, setIsValidatingToken] = useState(false);
   const [tokenValidationResult, setTokenValidationResult] = useState<{
     valid: boolean;
     organizationInfo?: {
@@ -57,16 +55,13 @@ export const OrgGithubSyncSection = () => {
 
   const updateGithubSyncOrgConfig = useUpdateGithubSyncOrgConfig();
   const syncAllTeamsMutation = useSyncAllGithubTeams();
-  const validateGithubTokenMutation = useValidateGithubToken();
 
   const isPending = subscription.githubOrgSync && githubOrgSyncConfig.isPending;
   const data = !isPending && !githubOrgSyncConfig?.isError ? githubOrgSyncConfig?.data : undefined;
 
-  const handleBulkSync = async (token?: string) => {
+  const handleBulkSync = async () => {
     try {
-      const result = await syncAllTeamsMutation.mutateAsync({
+      const result = await syncAllTeamsMutation.mutateAsync();
-        githubOrgAccessToken: token
-      });
       let message = `Successfully synced teams for ${result.syncedUsersCount} user${result.syncedUsersCount === 1 ? "" : "s"}`;
 
       const details = [];
@@ -102,9 +97,6 @@ export const OrgGithubSyncSection = () => {
         });
         console.warn("Sync errors:", result.errors);
       }
-
-      setAccessToken("");
-      handlePopUpToggle("syncAllTeamsToken", false);
     } catch (error) {
       const errorMessage =
         (error as any)?.response?.data?.message || (error as Error)?.message || "Unknown error";
@@ -113,11 +105,12 @@ export const OrgGithubSyncSection = () => {
         errorMessage.includes("token") &&
         (errorMessage.includes("required") ||
           errorMessage.includes("invalid") ||
-          errorMessage.includes("expired"))
+          errorMessage.includes("expired") ||
+          errorMessage.includes("set a token first"))
       ) {
-        handlePopUpOpen("syncAllTeamsToken");
+        handlePopUpOpen("setAccessToken");
         createNotification({
-          text: errorMessage,
+          text: "Please provide a GitHub access token to continue with the sync",
           type: "error"
         });
       } else {
@@ -129,48 +122,7 @@ export const OrgGithubSyncSection = () => {
     }
   };
 
-  const validateToken = async () => {
+  const handleSetAccessToken = async () => {
-    if (!accessToken.trim()) {
-      createNotification({
-        text: "Please enter a GitHub access token",
-        type: "error"
-      });
-      return false;
-    }
-
-    setIsValidatingToken(true);
-    try {
-      const result = await validateGithubTokenMutation.mutateAsync({
-        githubOrgAccessToken: accessToken.trim()
-      });
-
-      setTokenValidationResult(result);
-
-      if (result.valid && result.organizationInfo) {
-        createNotification({
-          text: `Token validated successfully for organization: ${result.organizationInfo.name}`,
-          type: "success"
-        });
-      }
-
-      return result.valid;
-    } catch (error) {
-      const errorMessage =
-        (error as any)?.response?.data?.message ||
-        (error as Error)?.message ||
-        "Token validation failed";
-      createNotification({
-        text: errorMessage,
-        type: "error"
-      });
-      setTokenValidationResult({ valid: false });
-      return false;
-    } finally {
-      setIsValidatingToken(false);
-    }
-  };
-
-  const handleSyncWithToken = async () => {
     if (!accessToken.trim()) {
       createNotification({
         text: "Please enter a GitHub access token",
@@ -179,15 +131,30 @@ export const OrgGithubSyncSection = () => {
       return;
     }
 
-    let isTokenValid = tokenValidationResult?.valid ?? false;
+    try {
-    if (!isTokenValid) {
+      await updateGithubSyncOrgConfig.mutateAsync({
-      isTokenValid = await validateToken();
+        githubOrgAccessToken: accessToken.trim()
-      if (!isTokenValid) {
+      });
-        return;
-      }
-    }
 
-    await handleBulkSync(accessToken.trim());
+      createNotification({
+        text: "GitHub access token set successfully. Starting sync...",
+        type: "success"
+      });
+
+      setAccessToken("");
+      handlePopUpToggle("setAccessToken", false);
+
+      // Automatically trigger sync after token is set
+      await handleBulkSync();
+    } catch (error) {
+      const errorMessage =
+        (error as any)?.response?.data?.message || (error as Error)?.message || "Unknown error";
+
+      createNotification({
+        text: `Failed to set GitHub access token: ${errorMessage}`,
+        type: "error"
+      });
+    }
   };
 
   return (
@@ -252,7 +219,10 @@ export const OrgGithubSyncSection = () => {
         <div className="py-4">
           <div className="mb-2 flex items-center justify-between">
             <h2 className="text-md text-mineshaft-100">Sync Now</h2>
-            <OrgPermissionCan I={OrgPermissionActions.Edit} a={OrgPermissionSubjects.GithubOrgSync}>
+            <OrgPermissionCan
+              I={OrgPermissionActions.Edit}
+              a={OrgPermissionSubjects.GithubOrgSyncManual}
+            >
               {(isAllowed) => (
                 <Button
                   onClick={() => handleBulkSync()}
@@ -296,9 +266,9 @@ export const OrgGithubSyncSection = () => {
         text="You can use GitHub Organization Plan if you switch to Infisical's Enterprise plan."
       />
       <Modal
-        isOpen={popUp?.syncAllTeamsToken?.isOpen}
+        isOpen={popUp?.setAccessToken?.isOpen}
         onOpenChange={(isOpen) => {
-          handlePopUpToggle("syncAllTeamsToken", isOpen);
+          handlePopUpToggle("setAccessToken", isOpen);
           if (!isOpen) {
             setAccessToken("");
             setTokenValidationResult(null);
@@ -307,7 +277,7 @@ export const OrgGithubSyncSection = () => {
       >
         <ModalContent
           title="GitHub Access Token Required"
-          subTitle="Provide a GitHub access token with organization and team access permissions"
+          subTitle="Provide a GitHub access token to sync teams from your GitHub organization"
         >
           <div className="space-y-4">
             <FormControl
@@ -340,22 +310,11 @@ export const OrgGithubSyncSection = () => {
               </div>
             </FormControl>
             <div className="flex justify-between">
-              <Button
-                colorSchema="secondary"
-                variant="outline_bg"
-                onClick={validateToken}
-                isLoading={isValidatingToken}
-                isDisabled={
-                  !accessToken.trim() || isValidatingToken || syncAllTeamsMutation.isPending
-                }
-              >
-                Validate Token
-              </Button>
               <div className="flex space-x-2">
                 <Button
                   colorSchema="secondary"
                   onClick={() => {
-                    handlePopUpToggle("syncAllTeamsToken", false);
+                    handlePopUpToggle("setAccessToken", false);
                     setAccessToken("");
                     setTokenValidationResult(null);
                   }}
@@ -364,11 +323,15 @@ export const OrgGithubSyncSection = () => {
                 </Button>
                 <Button
                   colorSchema="primary"
-                  onClick={handleSyncWithToken}
+                  onClick={handleSetAccessToken}
-                  isLoading={syncAllTeamsMutation.isPending}
+                  isLoading={updateGithubSyncOrgConfig.isPending || syncAllTeamsMutation.isPending}
-                  isDisabled={!accessToken.trim() || syncAllTeamsMutation.isPending}
+                  isDisabled={
+                    !accessToken.trim() ||
+                    updateGithubSyncOrgConfig.isPending ||
+                    syncAllTeamsMutation.isPending
+                  }
                 >
-                  Sync Teams
+                  Set Token & Sync
                 </Button>
               </div>
             </div>