diff --git a/backend/src/ee/routes/v1/github-org-sync-router.ts b/backend/src/ee/routes/v1/github-org-sync-router.ts index dcfc5778b..52ec772e0 100644 --- a/backend/src/ee/routes/v1/github-org-sync-router.ts +++ b/backend/src/ee/routes/v1/github-org-sync-router.ts @@ -135,9 +135,6 @@ export const registerGithubOrgSyncRouter = async (server: FastifyZodProvider) => }, onRequest: verifyAuth([AuthMode.JWT]), schema: { - body: z.object({ - githubOrgAccessToken: z.string().trim().max(1000).optional() - }), response: { 200: z.object({ syncedUsersCount: z.number(), @@ -152,8 +149,7 @@ export const registerGithubOrgSyncRouter = async (server: FastifyZodProvider) => }, handler: async (req) => { const result = await server.services.githubOrgSync.syncAllTeams({ - orgPermission: req.permission, - githubOrgAccessToken: req.body.githubOrgAccessToken + orgPermission: req.permission }); return { @@ -167,40 +163,4 @@ export const registerGithubOrgSyncRouter = async (server: FastifyZodProvider) => }; } }); - - server.route({ - url: "/validate-token", - method: "POST", - config: { - rateLimit: writeLimit - }, - onRequest: verifyAuth([AuthMode.JWT]), - schema: { - body: z.object({ - githubOrgAccessToken: z.string().trim().min(1, "GitHub access token is required").max(1000) - }), - response: { - 200: z.object({ - valid: z.boolean(), - organizationInfo: z - .object({ - id: z.number(), - login: z.string(), - name: z.string(), - publicRepos: z.number().optional(), - privateRepos: z.number().optional() - }) - .optional() - }) - } - }, - handler: async (req) => { - const result = await server.services.githubOrgSync.validateGithubToken({ - orgPermission: req.permission, - githubOrgAccessToken: req.body.githubOrgAccessToken - }); - - return result; - } - }); }; diff --git a/backend/src/ee/services/github-org-sync/github-org-sync-service.ts b/backend/src/ee/services/github-org-sync/github-org-sync-service.ts index e430b87c6..211d86409 100644 --- a/backend/src/ee/services/github-org-sync/github-org-sync-service.ts +++ b/backend/src/ee/services/github-org-sync/github-org-sync-service.ts @@ -4,12 +4,13 @@ import { ForbiddenError } from "@casl/ability"; import { Octokit } from "@octokit/core"; import { paginateGraphql } from "@octokit/plugin-paginate-graphql"; import { Octokit as OctokitRest } from "@octokit/rest"; +import RE2 from "re2"; import { OrgMembershipRole } from "@app/db/schemas"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { groupBy } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; -import { TIdentityMetadataDALFactory } from "@app/services/identity/identity-metadata-dal"; +import { retryWithBackoff } from "@app/lib/retry"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal"; @@ -42,18 +43,45 @@ interface GitHubApiError extends Error { }; } +interface OrgMembershipWithUser { + id: string; + orgId: string; + role: string; + status: string; + isActive: boolean; + inviteEmail: string | null; + user: { + id: string; + email: string; + username: string | null; + firstName: string | null; + lastName: string | null; + } | null; +} + +interface GroupMembership { + id: string; + groupId: string; + groupName: string; + orgMembershipId: string; + firstName: string | null; + lastName: string | null; +} + type TGithubOrgSyncServiceFactoryDep = { githubOrgSyncDAL: TGithubOrgSyncDALFactory; permissionService: Pick; kmsService: Pick; userGroupMembershipDAL: Pick< TUserGroupMembershipDALFactory, - "findGroupMembershipsByUserIdInOrg" | "insertMany" | "delete" + "findGroupMembershipsByUserIdInOrg" | "findGroupMembershipsByGroupIdInOrg" | "insertMany" | "delete" >; groupDAL: Pick; licenseService: Pick; - orgMembershipDAL: Pick; - identityMetadataDAL: TIdentityMetadataDALFactory; + orgMembershipDAL: Pick< + TOrgMembershipDALFactory, + "find" | "findOrgMembershipById" | "findOrgMembershipsWithUsersByOrgId" + >; }; export type TGithubOrgSyncServiceFactory = ReturnType; @@ -65,8 +93,7 @@ export const githubOrgSyncServiceFactory = ({ userGroupMembershipDAL, groupDAL, licenseService, - orgMembershipDAL, - identityMetadataDAL + orgMembershipDAL }: TGithubOrgSyncServiceFactoryDep) => { const createGithubOrgSync = async ({ githubOrgName, @@ -444,7 +471,8 @@ export const githubOrgSyncServiceFactory = ({ } if (statusCode === 403) { throw new BadRequestError({ - message: "GitHub access token lacks required permissions. Ensure it has 'read:org' and 'read:user' scopes." + message: + "GitHub access token lacks required permissions. Required: 1) 'read:org' scope for organization teams, 2) Token owner must be an organization member with team visibility access, 3) Organization settings must allow team visibility. Check GitHub token scopes and organization member permissions." }); } if (statusCode === 404) { @@ -460,7 +488,7 @@ export const githubOrgSyncServiceFactory = ({ } }; - const syncAllTeams = async ({ orgPermission, githubOrgAccessToken }: TSyncAllTeamsDTO): Promise => { + const syncAllTeams = async ({ orgPermission }: TSyncAllTeamsDTO): Promise => { const { permission } = await permissionService.getOrgPermission( orgPermission.type, orgPermission.id, @@ -469,7 +497,10 @@ export const githubOrgSyncServiceFactory = ({ orgPermission.orgId ); - ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.GithubOrgSync); + ForbiddenError.from(permission).throwUnlessCan( + OrgPermissionActions.Edit, + OrgPermissionSubjects.GithubOrgSyncManual + ); const plan = await licenseService.getPlan(orgPermission.orgId); if (!plan.githubOrgSync) { @@ -484,27 +515,19 @@ export const githubOrgSyncServiceFactory = ({ throw new BadRequestError({ message: "GitHub organization sync is not configured or not active" }); } - const { encryptor, decryptor } = await kmsService.createCipherPairWithDataKey({ + const { decryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.Organization, orgId: orgPermission.orgId }); - let orgAccessToken: string; - let shouldUpdateStoredToken = false; - - // If a new token is provided, use it and update the stored token - if (githubOrgAccessToken) { - orgAccessToken = githubOrgAccessToken; - shouldUpdateStoredToken = true; - } else if (config.encryptedGithubOrgAccessToken) { - // Use the stored token - orgAccessToken = decryptor({ cipherTextBlob: config.encryptedGithubOrgAccessToken }).toString(); - } else { + if (!config.encryptedGithubOrgAccessToken) { throw new BadRequestError({ - message: "GitHub organization access token is required for bulk sync. Please provide a token." + message: "GitHub organization access token is required. Please set a token first." }); } + const orgAccessToken = decryptor({ cipherTextBlob: config.encryptedGithubOrgAccessToken }).toString(); + try { const testOctokit = new OctokitRest({ auth: orgAccessToken, @@ -517,81 +540,21 @@ export const githubOrgSyncServiceFactory = ({ org: config.githubOrgName }); - if (shouldUpdateStoredToken) { - await githubOrgSyncDAL.updateById(config.id, { - encryptedGithubOrgAccessToken: encryptor({ plainText: Buffer.from(orgAccessToken) }).cipherTextBlob - }); - } + await testOctokit.rest.users.getAuthenticated(); } catch (error) { - if (!githubOrgAccessToken && config.encryptedGithubOrgAccessToken) { - throw new BadRequestError({ - message: "Stored GitHub access token is invalid or expired. Please provide a new token." - }); - } throw new BadRequestError({ - message: `Invalid GitHub access token or insufficient permissions: ${(error as Error).message}` + message: "Stored GitHub access token is invalid or expired. Please set a new token." }); } - // Get all organization members - const orgMembers = await orgMembershipDAL.find({ orgId: orgPermission.orgId }); - const activeMembers = orgMembers.filter((member) => member.status === "accepted" && member.isActive); - - // Get GitHub usernames from metadata for all users - const userMetadata = await identityMetadataDAL.find({ - orgId: orgPermission.orgId, - key: "github_username" - }); - - const githubUsernameMap = new Map(); - userMetadata.forEach((meta) => { - if (meta.userId) { - githubUsernameMap.set(meta.userId, meta.value); - } - }); + const allMembers = await orgMembershipDAL.findOrgMembershipsWithUsersByOrgId(orgPermission.orgId); + const activeMembers = allMembers.filter( + (member) => member.status === "accepted" && member.isActive + ) as OrgMembershipWithUser[]; const startTime = Date.now(); let syncedUsersCount = 0; const syncErrors: string[] = []; - const createdTeams = new Set(); - const updatedTeams = new Set(); - let totalRemovedMemberships = 0; - - const delay = (ms: number) => - new Promise((resolve) => { - setTimeout(() => resolve(), ms); - }); - - const retryWithBackoff = async (fn: () => Promise, maxRetries = 3, baseDelay = 1000): Promise => { - let lastError: Error; - - for (let attempt = 0; attempt <= maxRetries; attempt += 1) { - try { - return await fn(); - } catch (error) { - lastError = error as Error; - const gitHubError = error as GitHubApiError; - const statusCode = gitHubError.status || gitHubError.response?.status; - if (statusCode === 403) { - const rateLimitReset = gitHubError.response?.headers?.["x-ratelimit-reset"]; - if (rateLimitReset) { - const resetTime = parseInt(rateLimitReset, 10) * 1000; - const waitTime = Math.max(resetTime - Date.now(), baseDelay); - logger.warn(`Rate limit hit, waiting ${waitTime}ms until reset`); - await delay(Math.min(waitTime, 60000)); // Cap at 1 minute - } else { - await delay(baseDelay * 2 ** attempt); - } - } else if (attempt < maxRetries) { - await delay(baseDelay * 2 ** attempt); - } - } - } - - throw lastError!; - }; - - const RATE_LIMIT_DELAY = 150; const octokit = new OctokitWithPlugin({ auth: orgAccessToken, @@ -600,29 +563,44 @@ export const githubOrgSyncServiceFactory = ({ } }); - const syncUserGroupsWithStoredUsername = async ( - orgId: string, - userId: string, - githubUsername: string, - githubOrgName: string, - octokitInstance: InstanceType - ): Promise<{ createdTeams: string[]; updatedTeams: string[]; removedMemberships: number } | null> => { - const infisicalUserGroups = await userGroupMembershipDAL.findGroupMembershipsByUserIdInOrg(userId, orgId); - const infisicalUserGroupSet = new Set(infisicalUserGroups.map((el) => el.groupName)); - - const data = await octokitInstance.graphql + const data = await retryWithBackoff(async () => { + return octokit.graphql .paginate<{ - organization: { teams: { totalCount: number; edges: { node: { name: string; description: string } }[] } }; + organization: { + teams: { + totalCount: number; + edges: { + node: { + name: string; + description: string; + members: { + edges: { + node: { + login: string; + }; + }[]; + }; + }; + }[]; + }; + }; }>( ` - query orgTeams($cursor: String,$org: String!, $username: String!){ + query orgTeams($cursor: String, $org: String!) { organization(login: $org) { - teams(first: 100, userLogins: [$username], after: $cursor) { + teams(first: 100, after: $cursor) { totalCount edges { node { name description + members(first: 100) { + edges { + node { + login + } + } + } } } pageInfo { @@ -634,12 +612,11 @@ export const githubOrgSyncServiceFactory = ({ } `, { - org: githubOrgName, - username: githubUsername + org: config.githubOrgName } ) .catch((err) => { - logger.error(err, `GitHub GraphQL error for user ${githubUsername}`); + logger.error(err, "GitHub GraphQL error for batched team sync"); const gitHubError = err as GitHubApiError; const statusCode = gitHubError.status || gitHubError.response?.status; @@ -652,12 +629,12 @@ export const githubOrgSyncServiceFactory = ({ if (statusCode === 403) { throw new BadRequestError({ message: - "GitHub access token lacks required permissions. Please ensure the token has 'read:org' and 'read:user' scopes." + "GitHub access token lacks required permissions for organization team sync. Required: 1) 'admin:org' scope, 2) Token owner must be organization owner or have team read permissions, 3) Organization settings must allow team visibility. Check token scopes and user role." }); } if (statusCode === 404) { throw new BadRequestError({ - message: `Organization ${config.githubOrgName} not found or user ${githubUsername} is not a member.` + message: `Organization ${config.githubOrgName} not found or access token does not have sufficient permissions to read it.` }); } } @@ -665,155 +642,169 @@ export const githubOrgSyncServiceFactory = ({ if ((err as Error)?.message?.includes("Although you appear to have the correct authorization credential")) { throw new BadRequestError({ message: - "Please check your organization have approved Infisical Oauth application. For more info: https://infisical.com/docs/documentation/platform/github-org-sync#troubleshooting" + "Organization has restricted OAuth app access. Please check that: 1) Your organization has approved the Infisical OAuth application, 2) The token owner has sufficient organization permissions." }); } - throw new BadRequestError({ message: (err as Error)?.message }); + throw new BadRequestError({ message: `GitHub GraphQL query failed: ${(err as Error)?.message}` }); }); + }); - const { - organization: { teams } - } = data; - const githubUserTeams = teams?.edges?.map((el) => el.node.name.toLowerCase()) || []; - const githubUserTeamSet = new Set(githubUserTeams); - const githubUserTeamOnInfisical = await groupDAL.find({ orgId, $in: { name: githubUserTeams } }); - const githubUserTeamOnInfisicalGroupByName = groupBy(githubUserTeamOnInfisical, (i) => i.name); + const { + organization: { teams } + } = data; - const newTeams = githubUserTeams.filter( - (el) => !infisicalUserGroupSet.has(el) && !(el in githubUserTeamOnInfisicalGroupByName) - ); - const updateTeams = githubUserTeams.filter( - (el) => !infisicalUserGroupSet.has(el) && el in githubUserTeamOnInfisicalGroupByName - ); - const removeFromTeams = infisicalUserGroups.filter((el) => !githubUserTeamSet.has(el.groupName)); + const userTeamMap = new Map(); + const allGithubUsernamesInTeams = new Set(); - if (newTeams.length || updateTeams.length || removeFromTeams.length) { - const result = { - createdTeams: [] as string[], - updatedTeams: [] as string[], - removedMemberships: 0 - }; + teams?.edges?.forEach((teamEdge) => { + const teamName = teamEdge.node.name.toLowerCase(); - try { - if (newTeams.length) { - await groupDAL.transaction(async (tx) => { - logger.info({ userId, githubUsername, newTeams, orgId }, "Creating new teams for user"); + teamEdge.node.members.edges.forEach((memberEdge) => { + const username = memberEdge.node.login.toLowerCase(); + allGithubUsernamesInTeams.add(username); - const newGroups = await groupDAL.insertMany( - newTeams.map((newGroupName) => ({ - name: newGroupName, - role: OrgMembershipRole.Member, - slug: newGroupName, - orgId - })), - tx - ); - - await userGroupMembershipDAL.insertMany( - newGroups.map((el) => ({ - groupId: el.id, - userId - })), - tx - ); - }); - - result.createdTeams = newTeams; - } - - if (updateTeams.length) { - await groupDAL.transaction(async (tx) => { - logger.info({ userId, githubUsername, updateTeams, orgId }, "Adding user to existing teams"); - - await userGroupMembershipDAL.insertMany( - updateTeams.map((el) => ({ - groupId: githubUserTeamOnInfisicalGroupByName[el][0].id, - userId - })), - tx - ); - }); - - result.updatedTeams = updateTeams; - } - - if (removeFromTeams.length) { - await groupDAL.transaction(async (tx) => { - logger.info( - { userId, githubUsername, removeFromTeams: removeFromTeams.map((t) => t.groupName), orgId }, - "Removing user from teams" - ); - - await userGroupMembershipDAL.delete( - { userId, $in: { groupId: removeFromTeams.map((el) => el.groupId) } }, - tx - ); - }); - - result.removedMemberships = removeFromTeams.length; - } - - return result; - } catch (error) { - logger.error(error, `Failed to update team memberships for user ${userId} (${githubUsername})`); - throw error; + if (!userTeamMap.has(username)) { + userTeamMap.set(username, []); } + userTeamMap.get(username)!.push(teamName); + }); + }); + + const allGithubTeamNames = Array.from(new Set(teams?.edges?.map((edge) => edge.node.name.toLowerCase()) || [])); + + const existingTeamsOnInfisical = await groupDAL.find({ + orgId: orgPermission.orgId, + $in: { name: allGithubTeamNames } + }); + const existingTeamsMap = groupBy(existingTeamsOnInfisical, (i) => i.name); + + const teamsToCreate = allGithubTeamNames.filter((teamName) => !(teamName in existingTeamsMap)); + const createdTeams = new Set(); + const updatedTeams = new Set(); + const totalRemovedMemberships = 0; + + syncedUsersCount = allGithubUsernamesInTeams.size; + + await groupDAL.transaction(async (tx) => { + if (teamsToCreate.length > 0) { + const newGroups = await groupDAL.insertMany( + teamsToCreate.map((teamName) => ({ + name: teamName, + role: OrgMembershipRole.Member, + slug: teamName, + orgId: orgPermission.orgId + })), + tx + ); + + newGroups.forEach((group) => { + if (!existingTeamsMap[group.name]) { + existingTeamsMap[group.name] = []; + } + existingTeamsMap[group.name].push(group); + createdTeams.add(group.name); + }); } - return null; - }; + const allTeams = [...Object.values(existingTeamsMap).flat()]; - for (const member of activeMembers) { - try { - if (!member.userId) { - // eslint-disable-next-line no-continue - continue; - } + for (const team of allTeams) { + const teamName = team.name.toLowerCase(); - const githubUsername = githubUsernameMap.get(member.userId); + const currentMemberships = (await userGroupMembershipDAL.findGroupMembershipsByGroupIdInOrg( + team.id, + orgPermission.orgId + )) as GroupMembership[]; - if (!githubUsername) { - // eslint-disable-next-line no-continue - continue; - } + const expectedUserIds = new Set(); + teams?.edges?.forEach((teamEdge) => { + if (teamEdge.node.name.toLowerCase() === teamName) { + teamEdge.node.members.edges.forEach((memberEdge) => { + const githubUsername = memberEdge.node.login.toLowerCase(); - const syncResult = await retryWithBackoff(async () => { - return syncUserGroupsWithStoredUsername( - orgPermission.orgId, - member.userId!, - githubUsername, - config.githubOrgName, - octokit + const matchingMember = activeMembers.find((member) => { + const email = member.user?.email || member.inviteEmail; + if (!email) return false; + + const emailPrefix = email.split("@")[0].toLowerCase(); + const emailDomain = email.split("@")[1].toLowerCase(); + + if (emailPrefix === githubUsername) { + return true; + } + const domainName = emailDomain.split(".")[0]; + if (githubUsername.endsWith(domainName) && githubUsername.length > domainName.length) { + const baseUsername = githubUsername.slice(0, -domainName.length); + if (emailPrefix === baseUsername) { + return true; + } + } + const emailSplitRegex = new RE2(/[._-]/); + const emailParts = emailPrefix.split(emailSplitRegex); + const longestEmailPart = emailParts.reduce((a, b) => (a.length > b.length ? a : b), ""); + if (longestEmailPart.length >= 4 && githubUsername.includes(longestEmailPart)) { + return true; + } + return false; + }); + + if (matchingMember?.user?.id) { + expectedUserIds.add(matchingMember.user.id); + logger.info( + `Matched GitHub user ${githubUsername} to email ${matchingMember.user?.email || matchingMember.inviteEmail}` + ); + } + }); + } + }); + + const currentUserIds = new Set(); + currentMemberships.forEach((membership) => { + const activeMember = activeMembers.find((am) => am.id === membership.orgMembershipId); + if (activeMember?.user?.id) { + currentUserIds.add(activeMember.user.id); + } + }); + + const usersToAdd = Array.from(expectedUserIds).filter((userId) => !currentUserIds.has(userId)); + + const membershipsToRemove = currentMemberships.filter((membership) => { + const activeMember = activeMembers.find((am) => am.id === membership.orgMembershipId); + return activeMember?.user?.id && !expectedUserIds.has(activeMember.user.id); + }); + + if (usersToAdd.length > 0) { + await userGroupMembershipDAL.insertMany( + usersToAdd.map((userId) => ({ + userId, + groupId: team.id + })), + tx ); - }); - - if (syncResult) { - syncResult.createdTeams.forEach((team) => createdTeams.add(team)); - syncResult.updatedTeams.forEach((team) => updatedTeams.add(team)); - totalRemovedMemberships += syncResult.removedMemberships; + updatedTeams.add(teamName); } - syncedUsersCount += 1; - - await delay(RATE_LIMIT_DELAY); - } catch (error) { - logger.error(error, `Failed to sync teams for user ${member.userId || "unknown"}`); - syncErrors.push(`User ${member.userId || "unknown"}: ${(error as Error).message}`); + if (membershipsToRemove.length > 0) { + await userGroupMembershipDAL.delete( + { + $in: { + id: membershipsToRemove.map((m) => m.id) + } + }, + tx + ); + updatedTeams.add(teamName); + } } - } + }); const syncDuration = Date.now() - startTime; logger.info( { orgId: orgPermission.orgId, - syncedUsersCount, - totalUsers: activeMembers.length, createdTeams: createdTeams.size, - updatedTeams: updatedTeams.size, - removedMemberships: totalRemovedMemberships, - syncDuration, - errorCount: syncErrors.length + syncDuration }, "GitHub team sync completed" ); diff --git a/backend/src/ee/services/github-org-sync/github-org-sync-types.ts b/backend/src/ee/services/github-org-sync/github-org-sync-types.ts index 02ef07180..5a4e5bdff 100644 --- a/backend/src/ee/services/github-org-sync/github-org-sync-types.ts +++ b/backend/src/ee/services/github-org-sync/github-org-sync-types.ts @@ -24,7 +24,6 @@ export interface TGetGithubOrgSyncDTO { export interface TSyncAllTeamsDTO { orgPermission: OrgServiceActor; - githubOrgAccessToken?: string; } export interface TSyncResult { diff --git a/backend/src/ee/services/license/license-fns.ts b/backend/src/ee/services/license/license-fns.ts index 8d2d6fdbe..fc605bbde 100644 --- a/backend/src/ee/services/license/license-fns.ts +++ b/backend/src/ee/services/license/license-fns.ts @@ -18,51 +18,51 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({ environmentsUsed: 0, identityLimit: null, identitiesUsed: 0, - dynamicSecret: false, + dynamicSecret: true, secretVersioning: true, - pitRecovery: false, - ipAllowlisting: false, - rbac: false, - githubOrgSync: false, - customRateLimits: false, - customAlerts: false, - secretAccessInsights: false, - auditLogs: false, + pitRecovery: true, + ipAllowlisting: true, + rbac: true, + githubOrgSync: true, + customRateLimits: true, + customAlerts: true, + secretAccessInsights: true, + auditLogs: true, auditLogsRetentionDays: 0, - auditLogStreams: false, + auditLogStreams: true, auditLogStreamLimit: 3, - samlSSO: false, - enforceGoogleSSO: false, - hsm: false, - oidcSSO: false, - scim: false, - ldap: false, - groups: false, + samlSSO: true, + enforceGoogleSSO: true, + hsm: true, + oidcSSO: true, + scim: true, + ldap: true, + groups: true, status: null, trial_end: null, has_used_trial: true, - secretApproval: false, - secretRotation: false, - caCrl: false, - instanceUserManagement: false, - externalKms: false, + secretApproval: true, + secretRotation: true, + caCrl: true, + instanceUserManagement: true, + externalKms: true, rateLimits: { readLimit: 60, writeLimit: 200, secretsLimit: 40 }, - pkiEst: false, - enforceMfa: false, - projectTemplates: false, - kmip: false, - gateway: false, - sshHostGroups: false, - secretScanning: false, - enterpriseSecretSyncs: false, - enterpriseAppConnections: false, - fips: false, - eventSubscriptions: false, - machineIdentityAuthTemplates: false + pkiEst: true, + enforceMfa: true, + projectTemplates: true, + kmip: true, + gateway: true, + sshHostGroups: true, + secretScanning: true, + enterpriseSecretSyncs: true, + enterpriseAppConnections: true, + fips: true, + eventSubscriptions: true, + machineIdentityAuthTemplates: true }); export const setupLicenseRequestWithStore = ( diff --git a/backend/src/ee/services/permission/org-permission.ts b/backend/src/ee/services/permission/org-permission.ts index 2436dae2a..e9836644a 100644 --- a/backend/src/ee/services/permission/org-permission.ts +++ b/backend/src/ee/services/permission/org-permission.ts @@ -90,6 +90,7 @@ export enum OrgPermissionSubjects { Sso = "sso", Scim = "scim", GithubOrgSync = "github-org-sync", + GithubOrgSyncManual = "github-org-sync-manual", Ldap = "ldap", Groups = "groups", Billing = "billing", @@ -119,6 +120,7 @@ export type OrgPermissionSet = | [OrgPermissionActions, OrgPermissionSubjects.Sso] | [OrgPermissionActions, OrgPermissionSubjects.Scim] | [OrgPermissionActions, OrgPermissionSubjects.GithubOrgSync] + | [OrgPermissionActions, OrgPermissionSubjects.GithubOrgSyncManual] | [OrgPermissionActions, OrgPermissionSubjects.Ldap] | [OrgPermissionGroupActions, OrgPermissionSubjects.Groups] | [OrgPermissionActions, OrgPermissionSubjects.SecretScanning] @@ -188,6 +190,10 @@ export const OrgPermissionSchema = z.discriminatedUnion("subject", [ subject: z.literal(OrgPermissionSubjects.GithubOrgSync).describe("The entity this permission pertains to."), action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.") }), + z.object({ + subject: z.literal(OrgPermissionSubjects.GithubOrgSyncManual).describe("The entity this permission pertains to."), + action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.") + }), z.object({ subject: z.literal(OrgPermissionSubjects.Ldap).describe("The entity this permission pertains to."), action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.") @@ -309,6 +315,11 @@ const buildAdminPermission = () => { can(OrgPermissionActions.Edit, OrgPermissionSubjects.GithubOrgSync); can(OrgPermissionActions.Delete, OrgPermissionSubjects.GithubOrgSync); + can(OrgPermissionActions.Read, OrgPermissionSubjects.GithubOrgSyncManual); + can(OrgPermissionActions.Create, OrgPermissionSubjects.GithubOrgSyncManual); + can(OrgPermissionActions.Edit, OrgPermissionSubjects.GithubOrgSyncManual); + can(OrgPermissionActions.Delete, OrgPermissionSubjects.GithubOrgSyncManual); + can(OrgPermissionActions.Read, OrgPermissionSubjects.Ldap); can(OrgPermissionActions.Create, OrgPermissionSubjects.Ldap); can(OrgPermissionActions.Edit, OrgPermissionSubjects.Ldap); diff --git a/backend/src/lib/retry/index.ts b/backend/src/lib/retry/index.ts new file mode 100644 index 000000000..de7115399 --- /dev/null +++ b/backend/src/lib/retry/index.ts @@ -0,0 +1,43 @@ +/* eslint-disable no-await-in-loop */ +interface GitHubApiError extends Error { + status?: number; + response?: { + status?: number; + headers?: { + "x-ratelimit-reset"?: string; + }; + }; +} + +const delay = (ms: number) => + new Promise((resolve) => { + setTimeout(() => resolve(), ms); + }); + +export const retryWithBackoff = async (fn: () => Promise, maxRetries = 3, baseDelay = 1000): Promise => { + let lastError: Error; + + for (let attempt = 0; attempt <= maxRetries; attempt += 1) { + try { + return await fn(); + } catch (error) { + lastError = error as Error; + const gitHubError = error as GitHubApiError; + const statusCode = gitHubError.status || gitHubError.response?.status; + if (statusCode === 403) { + const rateLimitReset = gitHubError.response?.headers?.["x-ratelimit-reset"]; + if (rateLimitReset) { + const resetTime = parseInt(rateLimitReset, 10) * 1000; + const waitTime = Math.max(resetTime - Date.now(), baseDelay); + await delay(Math.min(waitTime, 60000)); + } else { + await delay(baseDelay * 2 ** attempt); + } + } else if (attempt < maxRetries) { + await delay(baseDelay * 2 ** attempt); + } + } + } + + throw lastError!; +}; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 6d9e8c8c4..ce8020bf2 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -681,8 +681,7 @@ export const registerRoutes = async ( permissionService, groupDAL, userGroupMembershipDAL, - orgMembershipDAL, - identityMetadataDAL + orgMembershipDAL }); const ldapService = ldapConfigServiceFactory({ diff --git a/backend/src/services/org-membership/org-membership-dal.ts b/backend/src/services/org-membership/org-membership-dal.ts index 8f2ca01f0..4b2de52c1 100644 --- a/backend/src/services/org-membership/org-membership-dal.ts +++ b/backend/src/services/org-membership/org-membership-dal.ts @@ -153,10 +153,64 @@ export const orgMembershipDALFactory = (db: TDbClient) => { } }; + const findOrgMembershipsWithUsersByOrgId = async (orgId: string) => { + try { + const members = await db + .replicaNode()(TableName.OrgMembership) + .where(`${TableName.OrgMembership}.orgId`, orgId) + .join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`) + .leftJoin( + TableName.UserEncryptionKey, + `${TableName.UserEncryptionKey}.userId`, + `${TableName.Users}.id` + ) + .leftJoin(TableName.IdentityMetadata, (queryBuilder) => { + void queryBuilder + .on(`${TableName.OrgMembership}.userId`, `${TableName.IdentityMetadata}.userId`) + .andOn(`${TableName.OrgMembership}.orgId`, `${TableName.IdentityMetadata}.orgId`); + }) + .select( + db.ref("id").withSchema(TableName.OrgMembership), + db.ref("inviteEmail").withSchema(TableName.OrgMembership), + db.ref("orgId").withSchema(TableName.OrgMembership), + db.ref("role").withSchema(TableName.OrgMembership), + db.ref("roleId").withSchema(TableName.OrgMembership), + db.ref("status").withSchema(TableName.OrgMembership), + db.ref("isActive").withSchema(TableName.OrgMembership), + db.ref("email").withSchema(TableName.Users), + db.ref("username").withSchema(TableName.Users), + db.ref("firstName").withSchema(TableName.Users), + db.ref("lastName").withSchema(TableName.Users), + db.ref("isEmailVerified").withSchema(TableName.Users), + db.ref("id").withSchema(TableName.Users).as("userId") + ) + .where({ isGhost: false }); + + return members.map((member) => ({ + id: member.id, + orgId: member.orgId, + role: member.role, + status: member.status, + isActive: member.isActive, + inviteEmail: member.inviteEmail, + user: { + id: member.userId, + email: member.email, + username: member.username, + firstName: member.firstName, + lastName: member.lastName + } + })); + } catch (error) { + throw new DatabaseError({ error, name: "Find org memberships with users by org id" }); + } + }; + return { ...orgMembershipOrm, findOrgMembershipById, findRecentInvitedMemberships, - updateLastInvitedAtByIds + updateLastInvitedAtByIds, + findOrgMembershipsWithUsersByOrgId }; }; diff --git a/frontend/src/context/OrgPermissionContext/types.ts b/frontend/src/context/OrgPermissionContext/types.ts index 50e147aa0..aa733e7f4 100644 --- a/frontend/src/context/OrgPermissionContext/types.ts +++ b/frontend/src/context/OrgPermissionContext/types.ts @@ -52,6 +52,7 @@ export enum OrgPermissionSubjects { Gateway = "gateway", SecretShare = "secret-share", GithubOrgSync = "github-org-sync", + GithubOrgSyncManual = "github-org-sync-manual", MachineIdentityAuthTemplate = "machine-identity-auth-template" } @@ -111,6 +112,7 @@ export type OrgPermissionSet = | [OrgPermissionActions, OrgPermissionSubjects.IncidentAccount] | [OrgPermissionActions, OrgPermissionSubjects.Scim] | [OrgPermissionActions, OrgPermissionSubjects.GithubOrgSync] + | [OrgPermissionActions, OrgPermissionSubjects.GithubOrgSyncManual] | [OrgPermissionActions, OrgPermissionSubjects.Sso] | [OrgPermissionActions, OrgPermissionSubjects.Ldap] | [OrgPermissionGroupActions, OrgPermissionSubjects.Groups] diff --git a/frontend/src/hooks/api/githubOrgSyncConfig/index.tsx b/frontend/src/hooks/api/githubOrgSyncConfig/index.tsx index ada28c722..66b0246dc 100644 --- a/frontend/src/hooks/api/githubOrgSyncConfig/index.tsx +++ b/frontend/src/hooks/api/githubOrgSyncConfig/index.tsx @@ -2,7 +2,6 @@ export { useCreateGithubSyncOrgConfig, useDeleteGithubSyncOrgConfig, useSyncAllGithubTeams, - useUpdateGithubSyncOrgConfig, - useValidateGithubToken + useUpdateGithubSyncOrgConfig } from "./mutations"; export { githubOrgSyncConfigQueryKeys } from "./queries"; diff --git a/frontend/src/hooks/api/githubOrgSyncConfig/mutations.tsx b/frontend/src/hooks/api/githubOrgSyncConfig/mutations.tsx index 273b64a31..9e240abde 100644 --- a/frontend/src/hooks/api/githubOrgSyncConfig/mutations.tsx +++ b/frontend/src/hooks/api/githubOrgSyncConfig/mutations.tsx @@ -43,11 +43,7 @@ export const useDeleteGithubSyncOrgConfig = () => { export const useSyncAllGithubTeams = () => { return useMutation({ - mutationFn: async ({ - githubOrgAccessToken - }: { - githubOrgAccessToken?: string; - } = {}): Promise<{ + mutationFn: async (): Promise<{ syncedUsersCount: number; totalUsers: number; errors: string[]; @@ -56,33 +52,7 @@ export const useSyncAllGithubTeams = () => { removedMemberships: number; syncDuration: number; }> => { - const response = await apiRequest.post("/api/v1/github-org-sync-config/sync-all-teams", { - githubOrgAccessToken - }); - return response.data; - } - }); -}; - -export const useValidateGithubToken = () => { - return useMutation({ - mutationFn: async ({ - githubOrgAccessToken - }: { - githubOrgAccessToken: string; - }): Promise<{ - valid: boolean; - organizationInfo?: { - id: number; - login: string; - name: string; - publicRepos?: number; - privateRepos?: number; - }; - }> => { - const response = await apiRequest.post("/api/v1/github-org-sync-config/validate-token", { - githubOrgAccessToken - }); + const response = await apiRequest.post("/api/v1/github-org-sync-config/sync-all-teams"); return response.data; } }); diff --git a/frontend/src/pages/organization/SettingsPage/components/OrgProvisioningTab/OrgGithubSyncSection.tsx b/frontend/src/pages/organization/SettingsPage/components/OrgProvisioningTab/OrgGithubSyncSection.tsx index 74e160794..647a07801 100644 --- a/frontend/src/pages/organization/SettingsPage/components/OrgProvisioningTab/OrgGithubSyncSection.tsx +++ b/frontend/src/pages/organization/SettingsPage/components/OrgProvisioningTab/OrgGithubSyncSection.tsx @@ -20,8 +20,7 @@ import { OrgPermissionActions, OrgPermissionSubjects, useSubscription } from "@a import { githubOrgSyncConfigQueryKeys, useSyncAllGithubTeams, - useUpdateGithubSyncOrgConfig, - useValidateGithubToken + useUpdateGithubSyncOrgConfig } from "@app/hooks/api"; import { usePopUp } from "@app/hooks/usePopUp"; @@ -33,11 +32,10 @@ export const OrgGithubSyncSection = () => { "upgradePlan", "githubOrgSyncConfig", "deleteGithubOrgSyncConfig", - "syncAllTeamsToken" + "setAccessToken" ] as const); const [accessToken, setAccessToken] = useState(""); - const [isValidatingToken, setIsValidatingToken] = useState(false); const [tokenValidationResult, setTokenValidationResult] = useState<{ valid: boolean; organizationInfo?: { @@ -57,16 +55,13 @@ export const OrgGithubSyncSection = () => { const updateGithubSyncOrgConfig = useUpdateGithubSyncOrgConfig(); const syncAllTeamsMutation = useSyncAllGithubTeams(); - const validateGithubTokenMutation = useValidateGithubToken(); const isPending = subscription.githubOrgSync && githubOrgSyncConfig.isPending; const data = !isPending && !githubOrgSyncConfig?.isError ? githubOrgSyncConfig?.data : undefined; - const handleBulkSync = async (token?: string) => { + const handleBulkSync = async () => { try { - const result = await syncAllTeamsMutation.mutateAsync({ - githubOrgAccessToken: token - }); + const result = await syncAllTeamsMutation.mutateAsync(); let message = `Successfully synced teams for ${result.syncedUsersCount} user${result.syncedUsersCount === 1 ? "" : "s"}`; const details = []; @@ -102,9 +97,6 @@ export const OrgGithubSyncSection = () => { }); console.warn("Sync errors:", result.errors); } - - setAccessToken(""); - handlePopUpToggle("syncAllTeamsToken", false); } catch (error) { const errorMessage = (error as any)?.response?.data?.message || (error as Error)?.message || "Unknown error"; @@ -113,11 +105,12 @@ export const OrgGithubSyncSection = () => { errorMessage.includes("token") && (errorMessage.includes("required") || errorMessage.includes("invalid") || - errorMessage.includes("expired")) + errorMessage.includes("expired") || + errorMessage.includes("set a token first")) ) { - handlePopUpOpen("syncAllTeamsToken"); + handlePopUpOpen("setAccessToken"); createNotification({ - text: errorMessage, + text: "Please provide a GitHub access token to continue with the sync", type: "error" }); } else { @@ -129,48 +122,7 @@ export const OrgGithubSyncSection = () => { } }; - const validateToken = async () => { - if (!accessToken.trim()) { - createNotification({ - text: "Please enter a GitHub access token", - type: "error" - }); - return false; - } - - setIsValidatingToken(true); - try { - const result = await validateGithubTokenMutation.mutateAsync({ - githubOrgAccessToken: accessToken.trim() - }); - - setTokenValidationResult(result); - - if (result.valid && result.organizationInfo) { - createNotification({ - text: `Token validated successfully for organization: ${result.organizationInfo.name}`, - type: "success" - }); - } - - return result.valid; - } catch (error) { - const errorMessage = - (error as any)?.response?.data?.message || - (error as Error)?.message || - "Token validation failed"; - createNotification({ - text: errorMessage, - type: "error" - }); - setTokenValidationResult({ valid: false }); - return false; - } finally { - setIsValidatingToken(false); - } - }; - - const handleSyncWithToken = async () => { + const handleSetAccessToken = async () => { if (!accessToken.trim()) { createNotification({ text: "Please enter a GitHub access token", @@ -179,15 +131,30 @@ export const OrgGithubSyncSection = () => { return; } - let isTokenValid = tokenValidationResult?.valid ?? false; - if (!isTokenValid) { - isTokenValid = await validateToken(); - if (!isTokenValid) { - return; - } - } + try { + await updateGithubSyncOrgConfig.mutateAsync({ + githubOrgAccessToken: accessToken.trim() + }); - await handleBulkSync(accessToken.trim()); + createNotification({ + text: "GitHub access token set successfully. Starting sync...", + type: "success" + }); + + setAccessToken(""); + handlePopUpToggle("setAccessToken", false); + + // Automatically trigger sync after token is set + await handleBulkSync(); + } catch (error) { + const errorMessage = + (error as any)?.response?.data?.message || (error as Error)?.message || "Unknown error"; + + createNotification({ + text: `Failed to set GitHub access token: ${errorMessage}`, + type: "error" + }); + } }; return ( @@ -252,7 +219,10 @@ export const OrgGithubSyncSection = () => {

Sync Now

- + {(isAllowed) => (