Add Github Bulk Team Sync

This commit is contained in:
Carlos Monastyrski
2025-08-28 01:13:25 -03:00
parent af2f21fe93
commit 7d74dce82b
7 changed files with 913 additions and 8 deletions
@@ -126,4 +126,81 @@ export const registerGithubOrgSyncRouter = async (server: FastifyZodProvider) =>
return { githubOrgSyncConfig };
}
});
server.route({
url: "/sync-all-teams",
method: "POST",
config: {
rateLimit: writeLimit
},
onRequest: verifyAuth([AuthMode.JWT]),
schema: {
body: z.object({
githubOrgAccessToken: z.string().trim().max(1000).optional()
}),
response: {
200: z.object({
syncedUsersCount: z.number(),
skippedUsersCount: z.number(),
totalUsers: z.number(),
errors: z.array(z.string()),
createdTeams: z.array(z.string()),
updatedTeams: z.array(z.string()),
removedMemberships: z.number(),
syncDuration: z.number()
})
}
},
handler: async (req) => {
const result = await server.services.githubOrgSync.syncAllTeams({
orgPermission: req.permission,
githubOrgAccessToken: req.body.githubOrgAccessToken
});
return {
syncedUsersCount: result.syncedUsersCount,
skippedUsersCount: result.skippedUsersCount,
totalUsers: result.totalUsers,
errors: result.errors,
createdTeams: result.createdTeams,
updatedTeams: result.updatedTeams,
removedMemberships: result.removedMemberships,
syncDuration: result.syncDuration
};
}
});
server.route({
url: "/validate-token",
method: "POST",
config: {
rateLimit: writeLimit
},
onRequest: verifyAuth([AuthMode.JWT]),
schema: {
body: z.object({
githubOrgAccessToken: z.string().trim().min(1, "GitHub access token is required")
}),
response: {
200: z.object({
valid: z.boolean(),
organizationInfo: z.object({
id: z.number(),
login: z.string(),
name: z.string(),
publicRepos: z.number().optional(),
privateRepos: z.number().optional()
}).optional()
})
}
},
handler: async (req) => {
const result = await server.services.githubOrgSync.validateGithubToken({
orgPermission: req.permission,
githubOrgAccessToken: req.body.githubOrgAccessToken
});
return result;
}
});
};
@@ -1,3 +1,5 @@
/* eslint-disable @typescript-eslint/return-await */
/* eslint-disable no-await-in-loop */
import { ForbiddenError } from "@casl/ability";
import { Octokit } from "@octokit/core";
import { paginateGraphql } from "@octokit/plugin-paginate-graphql";
@@ -7,8 +9,10 @@ import { OrgMembershipRole } from "@app/db/schemas";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { groupBy } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { TIdentityMetadataDALFactory } from "@app/services/identity/identity-metadata-dal";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { KmsDataKey } from "@app/services/kms/kms-types";
import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal";
import { TGroupDALFactory } from "../group/group-dal";
import { TUserGroupMembershipDALFactory } from "../group/user-group-membership-dal";
@@ -16,10 +20,28 @@ import { TLicenseServiceFactory } from "../license/license-service";
import { OrgPermissionActions, OrgPermissionSubjects } from "../permission/org-permission";
import { TPermissionServiceFactory } from "../permission/permission-service-types";
import { TGithubOrgSyncDALFactory } from "./github-org-sync-dal";
import { TCreateGithubOrgSyncDTO, TDeleteGithubOrgSyncDTO, TUpdateGithubOrgSyncDTO } from "./github-org-sync-types";
import {
TCreateGithubOrgSyncDTO,
TDeleteGithubOrgSyncDTO,
TSyncAllTeamsDTO,
TSyncResult,
TUpdateGithubOrgSyncDTO,
TValidateGithubTokenDTO
} from "./github-org-sync-types";
const OctokitWithPlugin = Octokit.plugin(paginateGraphql);
// Type definitions for GitHub API errors
interface GitHubApiError extends Error {
status?: number;
response?: {
status?: number;
headers?: {
"x-ratelimit-reset"?: string;
};
};
}
type TGithubOrgSyncServiceFactoryDep = {
githubOrgSyncDAL: TGithubOrgSyncDALFactory;
permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
@@ -30,6 +52,8 @@ type TGithubOrgSyncServiceFactoryDep = {
>;
groupDAL: Pick<TGroupDALFactory, "insertMany" | "transaction" | "find">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
orgMembershipDAL: Pick<TOrgMembershipDALFactory, "find">;
identityMetadataDAL: TIdentityMetadataDALFactory;
};
export type TGithubOrgSyncServiceFactory = ReturnType<typeof githubOrgSyncServiceFactory>;
@@ -40,7 +64,9 @@ export const githubOrgSyncServiceFactory = ({
kmsService,
userGroupMembershipDAL,
groupDAL,
licenseService
licenseService,
orgMembershipDAL,
identityMetadataDAL
}: TGithubOrgSyncServiceFactoryDep) => {
const createGithubOrgSync = async ({
githubOrgName,
@@ -344,11 +370,469 @@ export const githubOrgSyncServiceFactory = ({
}
};
const validateGithubToken = async ({ orgPermission, githubOrgAccessToken }: TValidateGithubTokenDTO) => {
const { permission } = await permissionService.getOrgPermission(
orgPermission.type,
orgPermission.id,
orgPermission.orgId,
orgPermission.authMethod,
orgPermission.orgId
);
ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.GithubOrgSync);
const plan = await licenseService.getPlan(orgPermission.orgId);
if (!plan.githubOrgSync) {
throw new BadRequestError({
message:
"Failed to validate GitHub token due to plan restriction. Upgrade plan to use GitHub organization sync."
});
}
const config = await githubOrgSyncDAL.findOne({ orgId: orgPermission.orgId });
if (!config) {
throw new BadRequestError({ message: "GitHub organization sync is not configured" });
}
try {
const testOctokit = new OctokitRest({
auth: githubOrgAccessToken,
request: {
signal: AbortSignal.timeout(10000)
}
});
const { data: org } = await testOctokit.rest.orgs.get({
org: config.githubOrgName
});
const octokitGraphQL = new OctokitWithPlugin({
auth: githubOrgAccessToken,
request: {
signal: AbortSignal.timeout(10000)
}
});
await octokitGraphQL.graphql(`query { organization(login: $org) { id name } }`, { org: config.githubOrgName });
return {
valid: true,
organizationInfo: {
id: org.id,
login: org.login,
name: org.name || org.login,
publicRepos: org.public_repos,
privateRepos: org.owned_private_repos || 0
}
};
} catch (error) {
logger.error(error, `GitHub token validation failed for org ${config.githubOrgName}`);
const gitHubError = error as GitHubApiError;
const statusCode = gitHubError.status || gitHubError.response?.status;
if (statusCode) {
if (statusCode === 401) {
throw new BadRequestError({
message: "GitHub access token is invalid or expired."
});
}
if (statusCode === 403) {
throw new BadRequestError({
message: "GitHub access token lacks required permissions. Ensure it has 'read:org' and 'read:user' scopes."
});
}
if (statusCode === 404) {
throw new BadRequestError({
message: `Organization '${config.githubOrgName}' not found or access token does not have access to it.`
});
}
}
throw new BadRequestError({
message: `GitHub token validation failed: ${(error as Error).message}`
});
}
};
const syncAllTeams = async ({ orgPermission, githubOrgAccessToken }: TSyncAllTeamsDTO): Promise<TSyncResult> => {
const { permission } = await permissionService.getOrgPermission(
orgPermission.type,
orgPermission.id,
orgPermission.orgId,
orgPermission.authMethod,
orgPermission.orgId
);
ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.GithubOrgSync);
const plan = await licenseService.getPlan(orgPermission.orgId);
if (!plan.githubOrgSync) {
throw new BadRequestError({
message:
"Failed to sync all GitHub teams due to plan restriction. Upgrade plan to use GitHub organization sync."
});
}
const config = await githubOrgSyncDAL.findOne({ orgId: orgPermission.orgId });
if (!config || !config?.isActive) {
throw new BadRequestError({ message: "GitHub organization sync is not configured or not active" });
}
const { encryptor, decryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.Organization,
orgId: orgPermission.orgId
});
let orgAccessToken: string;
let shouldUpdateStoredToken = false;
// If a new token is provided, use it and update the stored token
if (githubOrgAccessToken) {
orgAccessToken = githubOrgAccessToken;
shouldUpdateStoredToken = true;
} else if (config.encryptedGithubOrgAccessToken) {
// Use the stored token
orgAccessToken = decryptor({ cipherTextBlob: config.encryptedGithubOrgAccessToken }).toString();
} else {
throw new BadRequestError({
message: "GitHub organization access token is required for bulk sync. Please provide a token."
});
}
try {
const testOctokit = new OctokitRest({
auth: orgAccessToken,
request: {
signal: AbortSignal.timeout(10000)
}
});
await testOctokit.rest.orgs.get({
org: config.githubOrgName
});
if (shouldUpdateStoredToken) {
await githubOrgSyncDAL.updateById(config.id, {
encryptedGithubOrgAccessToken: encryptor({ plainText: Buffer.from(orgAccessToken) }).cipherTextBlob
});
}
} catch (error) {
if (!githubOrgAccessToken && config.encryptedGithubOrgAccessToken) {
throw new BadRequestError({
message: "Stored GitHub access token is invalid or expired. Please provide a new token."
});
}
throw new BadRequestError({
message: `Invalid GitHub access token or insufficient permissions: ${(error as Error).message}`
});
}
// Get all organization members
const orgMembers = await orgMembershipDAL.find({ orgId: orgPermission.orgId });
const activeMembers = orgMembers.filter((member) => member.status === "accepted" && member.isActive);
// Get GitHub usernames from metadata for all users
const userMetadata = await identityMetadataDAL.find({
orgId: orgPermission.orgId,
key: "github_username"
});
const githubUsernameMap = new Map<string, string>();
userMetadata.forEach((meta) => {
if (meta.userId) {
githubUsernameMap.set(meta.userId, meta.value);
}
});
const startTime = Date.now();
let syncedUsersCount = 0;
let skippedUsersCount = 0;
const syncErrors: string[] = [];
const createdTeams = new Set<string>();
const updatedTeams = new Set<string>();
let totalRemovedMemberships = 0;
const delay = (ms: number) =>
new Promise<void>((resolve) => {
setTimeout(() => resolve(), ms);
});
const retryWithBackoff = async <T>(fn: () => Promise<T>, maxRetries = 3, baseDelay = 1000): Promise<T> => {
let lastError: Error;
for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
try {
return await fn();
} catch (error) {
lastError = error as Error;
const gitHubError = error as GitHubApiError;
const statusCode = gitHubError.status || gitHubError.response?.status;
if (statusCode === 403) {
const rateLimitReset = gitHubError.response?.headers?.["x-ratelimit-reset"];
if (rateLimitReset) {
const resetTime = parseInt(rateLimitReset, 10) * 1000;
const waitTime = Math.max(resetTime - Date.now(), baseDelay);
logger.warn(`Rate limit hit, waiting ${waitTime}ms until reset`);
await delay(Math.min(waitTime, 60000)); // Cap at 1 minute
} else {
await delay(baseDelay * 2 ** attempt);
}
} else if (attempt < maxRetries) {
await delay(baseDelay * 2 ** attempt);
}
}
}
throw lastError!;
};
const RATE_LIMIT_DELAY = 150;
const octokit = new OctokitWithPlugin({
auth: orgAccessToken,
request: {
signal: AbortSignal.timeout(30000)
}
});
const syncUserGroupsWithStoredUsername = async (
orgId: string,
userId: string,
githubUsername: string,
githubOrgName: string,
octokitInstance: InstanceType<typeof OctokitWithPlugin>
): Promise<{ createdTeams: string[]; updatedTeams: string[]; removedMemberships: number } | null> => {
const infisicalUserGroups = await userGroupMembershipDAL.findGroupMembershipsByUserIdInOrg(userId, orgId);
const infisicalUserGroupSet = new Set(infisicalUserGroups.map((el) => el.groupName));
const data = await octokitInstance.graphql
.paginate<{
organization: { teams: { totalCount: number; edges: { node: { name: string; description: string } }[] } };
}>(
`
query orgTeams($cursor: String,$org: String!, $username: String!){
organization(login: $org) {
teams(first: 100, userLogins: [$username], after: $cursor) {
totalCount
edges {
node {
name
description
}
}
pageInfo {
hasNextPage
endCursor
}
}
}
}
`,
{
org: githubOrgName,
username: githubUsername
}
)
.catch((err) => {
logger.error(err, `GitHub GraphQL error for user ${githubUsername}`);
const gitHubError = err as GitHubApiError;
const statusCode = gitHubError.status || gitHubError.response?.status;
if (statusCode) {
if (statusCode === 401) {
throw new BadRequestError({
message: "GitHub access token is invalid or expired. Please provide a new token."
});
}
if (statusCode === 403) {
throw new BadRequestError({
message:
"GitHub access token lacks required permissions. Please ensure the token has 'read:org' and 'read:user' scopes."
});
}
if (statusCode === 404) {
throw new BadRequestError({
message: `Organization ${config.githubOrgName} not found or user ${githubUsername} is not a member.`
});
}
}
if ((err as Error)?.message?.includes("Although you appear to have the correct authorization credential")) {
throw new BadRequestError({
message:
"Please check your organization have approved Infisical Oauth application. For more info: https://infisical.com/docs/documentation/platform/github-org-sync#troubleshooting"
});
}
throw new BadRequestError({ message: (err as Error)?.message });
});
const {
organization: { teams }
} = data;
const githubUserTeams = teams?.edges?.map((el) => el.node.name.toLowerCase()) || [];
const githubUserTeamSet = new Set(githubUserTeams);
const githubUserTeamOnInfisical = await groupDAL.find({ orgId, $in: { name: githubUserTeams } });
const githubUserTeamOnInfisicalGroupByName = groupBy(githubUserTeamOnInfisical, (i) => i.name);
const newTeams = githubUserTeams.filter(
(el) => !infisicalUserGroupSet.has(el) && !(el in githubUserTeamOnInfisicalGroupByName)
);
const updateTeams = githubUserTeams.filter(
(el) => !infisicalUserGroupSet.has(el) && el in githubUserTeamOnInfisicalGroupByName
);
const removeFromTeams = infisicalUserGroups.filter((el) => !githubUserTeamSet.has(el.groupName));
if (newTeams.length || updateTeams.length || removeFromTeams.length) {
return await groupDAL.transaction(async (tx) => {
const result = {
createdTeams: [] as string[],
updatedTeams: [] as string[],
removedMemberships: 0
};
try {
if (newTeams.length) {
logger.info({ userId, githubUsername, newTeams, orgId }, "Creating new teams for user");
const newGroups = await groupDAL.insertMany(
newTeams.map((newGroupName) => ({
name: newGroupName,
role: OrgMembershipRole.Member,
slug: newGroupName,
orgId
})),
tx
);
await userGroupMembershipDAL.insertMany(
newGroups.map((el) => ({
groupId: el.id,
userId
})),
tx
);
result.createdTeams = newTeams;
}
if (updateTeams.length) {
logger.info({ userId, githubUsername, updateTeams, orgId }, "Adding user to existing teams");
await userGroupMembershipDAL.insertMany(
updateTeams.map((el) => ({
groupId: githubUserTeamOnInfisicalGroupByName[el][0].id,
userId
})),
tx
);
result.updatedTeams = updateTeams;
}
if (removeFromTeams.length) {
logger.info(
{ userId, githubUsername, removeFromTeams: removeFromTeams.map((t) => t.groupName), orgId },
"Removing user from teams"
);
await userGroupMembershipDAL.delete(
{ userId, $in: { groupId: removeFromTeams.map((el) => el.groupId) } },
tx
);
result.removedMemberships = removeFromTeams.length;
}
return result;
} catch (error) {
logger.error(error, `Failed to update team memberships for user ${userId} (${githubUsername})`);
throw error;
}
});
}
return null;
};
for (const member of activeMembers) {
try {
if (!member.userId) {
skippedUsersCount += 1;
syncErrors.push("Member without userId found, skipping");
// eslint-disable-next-line no-continue
continue;
}
const githubUsername = githubUsernameMap.get(member.userId);
if (!githubUsername) {
skippedUsersCount += 1;
syncErrors.push(`User ${member.userId}: No GitHub username found. User needs to log in at least once.`);
// eslint-disable-next-line no-continue
continue;
}
const syncResult = await retryWithBackoff(async () => {
return syncUserGroupsWithStoredUsername(
orgPermission.orgId,
member.userId!,
githubUsername,
config.githubOrgName,
octokit
);
});
if (syncResult) {
syncResult.createdTeams.forEach((team) => createdTeams.add(team));
syncResult.updatedTeams.forEach((team) => updatedTeams.add(team));
totalRemovedMemberships += syncResult.removedMemberships;
}
syncedUsersCount += 1;
await delay(RATE_LIMIT_DELAY);
} catch (error) {
logger.error(error, `Failed to sync teams for user ${member.userId || "unknown"}`);
syncErrors.push(`User ${member.userId || "unknown"}: ${(error as Error).message}`);
}
}
const syncDuration = Date.now() - startTime;
logger.info(
{
orgId: orgPermission.orgId,
syncedUsersCount,
skippedUsersCount,
totalUsers: activeMembers.length,
createdTeams: createdTeams.size,
updatedTeams: updatedTeams.size,
removedMemberships: totalRemovedMemberships,
syncDuration,
errorCount: syncErrors.length
},
"GitHub team sync completed"
);
return {
syncedUsersCount,
skippedUsersCount,
totalUsers: activeMembers.length,
errors: syncErrors,
createdTeams: Array.from(createdTeams),
updatedTeams: Array.from(updatedTeams),
removedMemberships: totalRemovedMemberships,
syncDuration
};
};
return {
createGithubOrgSync,
updateGithubOrgSync,
deleteGithubOrgSync,
getGithubOrgSync,
syncUserGroups
syncUserGroups,
syncAllTeams,
validateGithubToken
};
};
@@ -21,3 +21,24 @@ export interface TDeleteGithubOrgSyncDTO {
export interface TGetGithubOrgSyncDTO {
orgPermission: OrgServiceActor;
}
export interface TSyncAllTeamsDTO {
orgPermission: OrgServiceActor;
githubOrgAccessToken?: string;
}
export interface TSyncResult {
syncedUsersCount: number;
skippedUsersCount: number;
totalUsers: number;
errors: string[];
createdTeams: string[];
updatedTeams: string[];
removedMemberships: number;
syncDuration: number;
}
export interface TValidateGithubTokenDTO {
orgPermission: OrgServiceActor;
githubOrgAccessToken: string;
}
+3 -1
View File
@@ -680,7 +680,9 @@ export const registerRoutes = async (
kmsService,
permissionService,
groupDAL,
userGroupMembershipDAL
userGroupMembershipDAL,
orgMembershipDAL,
identityMetadataDAL
});
const ldapService = ldapConfigServiceFactory({