awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improvements on github bulk sync

Browse Source
This commit is contained in:
Carlos Monastyrski
2025-09-01 15:55:08 -03:00
parent 5944642278
commit b7d3ddff21
12 changed files with 419 additions and 428 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
backend/src/ee/routes/v1/github-org-sync-router.ts
View File

@@ -135,9 +135,6 @@ export const registerGithubOrgSyncRouter = async (server: FastifyZodProvider) =>
},
onRequest: verifyAuth([AuthMode.JWT]),
schema: {
body: z.object({
githubOrgAccessToken: z.string().trim().max(1000).optional()
}),
response: {
200: z.object({
syncedUsersCount: z.number(),
@@ -152,8 +149,7 @@ export const registerGithubOrgSyncRouter = async (server: FastifyZodProvider) =>
},
handler: async (req) => {
const result = await server.services.githubOrgSync.syncAllTeams({
orgPermission: req.permission,
githubOrgAccessToken: req.body.githubOrgAccessToken
orgPermission: req.permission
});
return {
@@ -167,40 +163,4 @@ export const registerGithubOrgSyncRouter = async (server: FastifyZodProvider) =>
};
}
});
server.route({
url: "/validate-token",
method: "POST",
config: {
rateLimit: writeLimit
},
onRequest: verifyAuth([AuthMode.JWT]),
schema: {
body: z.object({
githubOrgAccessToken: z.string().trim().min(1, "GitHub access token is required").max(1000)
}),
response: {
200: z.object({
valid: z.boolean(),
organizationInfo: z
.object({
id: z.number(),
login: z.string(),
name: z.string(),
publicRepos: z.number().optional(),
privateRepos: z.number().optional()
})
.optional()
})
}
},
handler: async (req) => {
const result = await server.services.githubOrgSync.validateGithubToken({
orgPermission: req.permission,
githubOrgAccessToken: req.body.githubOrgAccessToken
});
return result;
}
});
};

451
backend/src/ee/services/github-org-sync/github-org-sync-service.ts
View File

@@ -4,12 +4,13 @@ import { ForbiddenError } from "@casl/ability";
import { Octokit } from "@octokit/core";
import { paginateGraphql } from "@octokit/plugin-paginate-graphql";
import { Octokit as OctokitRest } from "@octokit/rest";
import RE2 from "re2";
import { OrgMembershipRole } from "@app/db/schemas";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { groupBy } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { TIdentityMetadataDALFactory } from "@app/services/identity/identity-metadata-dal";
import { retryWithBackoff } from "@app/lib/retry";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { KmsDataKey } from "@app/services/kms/kms-types";
import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal";
@@ -42,18 +43,45 @@ interface GitHubApiError extends Error {
};
}
interface OrgMembershipWithUser {
id: string;
orgId: string;
role: string;
status: string;
isActive: boolean;
inviteEmail: string | null;
user: {
id: string;
email: string;
username: string | null;
firstName: string | null;
lastName: string | null;
} | null;
}
interface GroupMembership {
id: string;
groupId: string;
groupName: string;
orgMembershipId: string;
firstName: string | null;
lastName: string | null;
}
type TGithubOrgSyncServiceFactoryDep = {
githubOrgSyncDAL: TGithubOrgSyncDALFactory;
permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
userGroupMembershipDAL: Pick<
TUserGroupMembershipDALFactory,
"findGroupMembershipsByUserIdInOrg" | "insertMany" | "delete"
"findGroupMembershipsByUserIdInOrg" | "findGroupMembershipsByGroupIdInOrg" | "insertMany" | "delete"
>;
groupDAL: Pick<TGroupDALFactory, "insertMany" | "transaction" | "find">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
orgMembershipDAL: Pick<TOrgMembershipDALFactory, "find">;
identityMetadataDAL: TIdentityMetadataDALFactory;
orgMembershipDAL: Pick<
TOrgMembershipDALFactory,
"find" | "findOrgMembershipById" | "findOrgMembershipsWithUsersByOrgId"
>;
};
export type TGithubOrgSyncServiceFactory = ReturnType<typeof githubOrgSyncServiceFactory>;
@@ -65,8 +93,7 @@ export const githubOrgSyncServiceFactory = ({
userGroupMembershipDAL,
groupDAL,
licenseService,
orgMembershipDAL,
identityMetadataDAL
orgMembershipDAL
}: TGithubOrgSyncServiceFactoryDep) => {
const createGithubOrgSync = async ({
githubOrgName,
@@ -444,7 +471,8 @@ export const githubOrgSyncServiceFactory = ({
}
if (statusCode === 403) {
throw new BadRequestError({
message: "GitHub access token lacks required permissions. Ensure it has 'read:org' and 'read:user' scopes."
message:
"GitHub access token lacks required permissions. Required: 1) 'read:org' scope for organization teams, 2) Token owner must be an organization member with team visibility access, 3) Organization settings must allow team visibility. Check GitHub token scopes and organization member permissions."
});
}
if (statusCode === 404) {
@@ -460,7 +488,7 @@ export const githubOrgSyncServiceFactory = ({
}
};
const syncAllTeams = async ({ orgPermission, githubOrgAccessToken }: TSyncAllTeamsDTO): Promise<TSyncResult> => {
const syncAllTeams = async ({ orgPermission }: TSyncAllTeamsDTO): Promise<TSyncResult> => {
const { permission } = await permissionService.getOrgPermission(
orgPermission.type,
orgPermission.id,
@@ -469,7 +497,10 @@ export const githubOrgSyncServiceFactory = ({
orgPermission.orgId
);
ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.GithubOrgSync);
ForbiddenError.from(permission).throwUnlessCan(
OrgPermissionActions.Edit,
OrgPermissionSubjects.GithubOrgSyncManual
);
const plan = await licenseService.getPlan(orgPermission.orgId);
if (!plan.githubOrgSync) {
@@ -484,27 +515,19 @@ export const githubOrgSyncServiceFactory = ({
throw new BadRequestError({ message: "GitHub organization sync is not configured or not active" });
}
const { encryptor, decryptor } = await kmsService.createCipherPairWithDataKey({
const { decryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.Organization,
orgId: orgPermission.orgId
});
let orgAccessToken: string;
let shouldUpdateStoredToken = false;
// If a new token is provided, use it and update the stored token
if (githubOrgAccessToken) {
orgAccessToken = githubOrgAccessToken;
shouldUpdateStoredToken = true;
} else if (config.encryptedGithubOrgAccessToken) {
// Use the stored token
orgAccessToken = decryptor({ cipherTextBlob: config.encryptedGithubOrgAccessToken }).toString();
} else {
if (!config.encryptedGithubOrgAccessToken) {
throw new BadRequestError({
message: "GitHub organization access token is required for bulk sync. Please provide a token."
message: "GitHub organization access token is required. Please set a token first."
});
}
const orgAccessToken = decryptor({ cipherTextBlob: config.encryptedGithubOrgAccessToken }).toString();
try {
const testOctokit = new OctokitRest({
auth: orgAccessToken,
@@ -517,81 +540,21 @@ export const githubOrgSyncServiceFactory = ({
org: config.githubOrgName
});
if (shouldUpdateStoredToken) {
await githubOrgSyncDAL.updateById(config.id, {
encryptedGithubOrgAccessToken: encryptor({ plainText: Buffer.from(orgAccessToken) }).cipherTextBlob
});
}
await testOctokit.rest.users.getAuthenticated();
} catch (error) {
if (!githubOrgAccessToken && config.encryptedGithubOrgAccessToken) {
throw new BadRequestError({
message: "Stored GitHub access token is invalid or expired. Please provide a new token."
});
}
throw new BadRequestError({
message: `Invalid GitHub access token or insufficient permissions: ${(error as Error).message}`
message: "Stored GitHub access token is invalid or expired. Please set a new token."
});
}
// Get all organization members
const orgMembers = await orgMembershipDAL.find({ orgId: orgPermission.orgId });
const activeMembers = orgMembers.filter((member) => member.status === "accepted" && member.isActive);
// Get GitHub usernames from metadata for all users
const userMetadata = await identityMetadataDAL.find({
orgId: orgPermission.orgId,
key: "github_username"
});
const githubUsernameMap = new Map<string, string>();
userMetadata.forEach((meta) => {
if (meta.userId) {
githubUsernameMap.set(meta.userId, meta.value);
}
});
const allMembers = await orgMembershipDAL.findOrgMembershipsWithUsersByOrgId(orgPermission.orgId);
const activeMembers = allMembers.filter(
(member) => member.status === "accepted" && member.isActive
) as OrgMembershipWithUser[];
const startTime = Date.now();
let syncedUsersCount = 0;
const syncErrors: string[] = [];
const createdTeams = new Set<string>();
const updatedTeams = new Set<string>();
let totalRemovedMemberships = 0;
const delay = (ms: number) =>
new Promise<void>((resolve) => {
setTimeout(() => resolve(), ms);
});
const retryWithBackoff = async <T>(fn: () => Promise<T>, maxRetries = 3, baseDelay = 1000): Promise<T> => {
let lastError: Error;
for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
try {
return await fn();
} catch (error) {
lastError = error as Error;
const gitHubError = error as GitHubApiError;
const statusCode = gitHubError.status || gitHubError.response?.status;
if (statusCode === 403) {
const rateLimitReset = gitHubError.response?.headers?.["x-ratelimit-reset"];
if (rateLimitReset) {
const resetTime = parseInt(rateLimitReset, 10) * 1000;
const waitTime = Math.max(resetTime - Date.now(), baseDelay);
logger.warn(`Rate limit hit, waiting ${waitTime}ms until reset`);
await delay(Math.min(waitTime, 60000)); // Cap at 1 minute
} else {
await delay(baseDelay * 2 ** attempt);
}
} else if (attempt < maxRetries) {
await delay(baseDelay * 2 ** attempt);
}
}
}
throw lastError!;
};
const RATE_LIMIT_DELAY = 150;
const octokit = new OctokitWithPlugin({
auth: orgAccessToken,
@@ -600,29 +563,44 @@ export const githubOrgSyncServiceFactory = ({
}
});
const syncUserGroupsWithStoredUsername = async (
orgId: string,
userId: string,
githubUsername: string,
githubOrgName: string,
octokitInstance: InstanceType<typeof OctokitWithPlugin>
): Promise<{ createdTeams: string[]; updatedTeams: string[]; removedMemberships: number } | null> => {
const infisicalUserGroups = await userGroupMembershipDAL.findGroupMembershipsByUserIdInOrg(userId, orgId);
const infisicalUserGroupSet = new Set(infisicalUserGroups.map((el) => el.groupName));
const data = await octokitInstance.graphql
const data = await retryWithBackoff(async () => {
return octokit.graphql
.paginate<{
organization: { teams: { totalCount: number; edges: { node: { name: string; description: string } }[] } };
organization: {
teams: {
totalCount: number;
edges: {
node: {
name: string;
description: string;
members: {
edges: {
node: {
login: string;
};
}[];
};
};
}[];
};
};
}>(
`
query orgTeams($cursor: String,$org: String!, $username: String!){
query orgTeams($cursor: String, $org: String!) {
organization(login: $org) {
teams(first: 100, userLogins: [$username], after: $cursor) {
teams(first: 100, after: $cursor) {
totalCount
edges {
node {
name
description
members(first: 100) {
edges {
node {
login
}
}
}
}
}
pageInfo {
@@ -634,12 +612,11 @@ export const githubOrgSyncServiceFactory = ({
}
`,
{
org: githubOrgName,
username: githubUsername
org: config.githubOrgName
}
)
.catch((err) => {
logger.error(err, `GitHub GraphQL error for user ${githubUsername}`);
logger.error(err, "GitHub GraphQL error for batched team sync");
const gitHubError = err as GitHubApiError;
const statusCode = gitHubError.status || gitHubError.response?.status;
@@ -652,12 +629,12 @@ export const githubOrgSyncServiceFactory = ({
if (statusCode === 403) {
throw new BadRequestError({
message:
"GitHub access token lacks required permissions. Please ensure the token has 'read:org' and 'read:user' scopes."
"GitHub access token lacks required permissions for organization team sync. Required: 1) 'admin:org' scope, 2) Token owner must be organization owner or have team read permissions, 3) Organization settings must allow team visibility. Check token scopes and user role."
});
}
if (statusCode === 404) {
throw new BadRequestError({
message: `Organization ${config.githubOrgName} not found or user ${githubUsername} is not a member.`
message: `Organization ${config.githubOrgName} not found or access token does not have sufficient permissions to read it.`
});
}
}
@@ -665,155 +642,169 @@ export const githubOrgSyncServiceFactory = ({
if ((err as Error)?.message?.includes("Although you appear to have the correct authorization credential")) {
throw new BadRequestError({
message:
"Please check your organization have approved Infisical Oauth application. For more info: https://infisical.com/docs/documentation/platform/github-org-sync#troubleshooting"
"Organization has restricted OAuth app access. Please check that: 1) Your organization has approved the Infisical OAuth application, 2) The token owner has sufficient organization permissions."
});
}
throw new BadRequestError({ message: (err as Error)?.message });
throw new BadRequestError({ message: `GitHub GraphQL query failed: ${(err as Error)?.message}` });
});
});
const {
organization: { teams }
} = data;
const githubUserTeams = teams?.edges?.map((el) => el.node.name.toLowerCase()) || [];
const githubUserTeamSet = new Set(githubUserTeams);
const githubUserTeamOnInfisical = await groupDAL.find({ orgId, $in: { name: githubUserTeams } });
const githubUserTeamOnInfisicalGroupByName = groupBy(githubUserTeamOnInfisical, (i) => i.name);
const {
organization: { teams }
} = data;
const newTeams = githubUserTeams.filter(
(el) => !infisicalUserGroupSet.has(el) && !(el in githubUserTeamOnInfisicalGroupByName)
);
const updateTeams = githubUserTeams.filter(
(el) => !infisicalUserGroupSet.has(el) && el in githubUserTeamOnInfisicalGroupByName
);
const removeFromTeams = infisicalUserGroups.filter((el) => !githubUserTeamSet.has(el.groupName));
const userTeamMap = new Map<string, string[]>();
const allGithubUsernamesInTeams = new Set<string>();
if (newTeams.length || updateTeams.length || removeFromTeams.length) {
const result = {
createdTeams: [] as string[],
updatedTeams: [] as string[],
removedMemberships: 0
};
teams?.edges?.forEach((teamEdge) => {
const teamName = teamEdge.node.name.toLowerCase();
try {
if (newTeams.length) {
await groupDAL.transaction(async (tx) => {
logger.info({ userId, githubUsername, newTeams, orgId }, "Creating new teams for user");
teamEdge.node.members.edges.forEach((memberEdge) => {
const username = memberEdge.node.login.toLowerCase();
allGithubUsernamesInTeams.add(username);
const newGroups = await groupDAL.insertMany(
newTeams.map((newGroupName) => ({
name: newGroupName,
role: OrgMembershipRole.Member,
slug: newGroupName,
orgId
})),
tx
);
await userGroupMembershipDAL.insertMany(
newGroups.map((el) => ({
groupId: el.id,
userId
})),
tx
);
});
result.createdTeams = newTeams;
}
if (updateTeams.length) {
await groupDAL.transaction(async (tx) => {
logger.info({ userId, githubUsername, updateTeams, orgId }, "Adding user to existing teams");
await userGroupMembershipDAL.insertMany(
updateTeams.map((el) => ({
groupId: githubUserTeamOnInfisicalGroupByName[el][0].id,
userId
})),
tx
);
});
result.updatedTeams = updateTeams;
}
if (removeFromTeams.length) {
await groupDAL.transaction(async (tx) => {
logger.info(
{ userId, githubUsername, removeFromTeams: removeFromTeams.map((t) => t.groupName), orgId },
"Removing user from teams"
);
await userGroupMembershipDAL.delete(
{ userId, $in: { groupId: removeFromTeams.map((el) => el.groupId) } },
tx
);
});
result.removedMemberships = removeFromTeams.length;
}
return result;
} catch (error) {
logger.error(error, `Failed to update team memberships for user ${userId} (${githubUsername})`);
throw error;
if (!userTeamMap.has(username)) {
userTeamMap.set(username, []);
}
userTeamMap.get(username)!.push(teamName);
});
});
const allGithubTeamNames = Array.from(new Set(teams?.edges?.map((edge) => edge.node.name.toLowerCase()) || []));
const existingTeamsOnInfisical = await groupDAL.find({
orgId: orgPermission.orgId,
$in: { name: allGithubTeamNames }
});
const existingTeamsMap = groupBy(existingTeamsOnInfisical, (i) => i.name);
const teamsToCreate = allGithubTeamNames.filter((teamName) => !(teamName in existingTeamsMap));
const createdTeams = new Set<string>();
const updatedTeams = new Set<string>();
const totalRemovedMemberships = 0;
syncedUsersCount = allGithubUsernamesInTeams.size;
await groupDAL.transaction(async (tx) => {
if (teamsToCreate.length > 0) {
const newGroups = await groupDAL.insertMany(
teamsToCreate.map((teamName) => ({
name: teamName,
role: OrgMembershipRole.Member,
slug: teamName,
orgId: orgPermission.orgId
})),
tx
);
newGroups.forEach((group) => {
if (!existingTeamsMap[group.name]) {
existingTeamsMap[group.name] = [];
}
existingTeamsMap[group.name].push(group);
createdTeams.add(group.name);
});
}
return null;
};
const allTeams = [...Object.values(existingTeamsMap).flat()];
for (const member of activeMembers) {
try {
if (!member.userId) {
// eslint-disable-next-line no-continue
continue;
}
for (const team of allTeams) {
const teamName = team.name.toLowerCase();
const githubUsername = githubUsernameMap.get(member.userId);
const currentMemberships = (await userGroupMembershipDAL.findGroupMembershipsByGroupIdInOrg(
team.id,
orgPermission.orgId
)) as GroupMembership[];
if (!githubUsername) {
// eslint-disable-next-line no-continue
continue;
}
const expectedUserIds = new Set<string>();
teams?.edges?.forEach((teamEdge) => {
if (teamEdge.node.name.toLowerCase() === teamName) {
teamEdge.node.members.edges.forEach((memberEdge) => {
const githubUsername = memberEdge.node.login.toLowerCase();
const syncResult = await retryWithBackoff(async () => {
return syncUserGroupsWithStoredUsername(
orgPermission.orgId,
member.userId!,
githubUsername,
config.githubOrgName,
octokit
const matchingMember = activeMembers.find((member) => {
const email = member.user?.email || member.inviteEmail;
if (!email) return false;
const emailPrefix = email.split("@")[0].toLowerCase();
const emailDomain = email.split("@")[1].toLowerCase();
if (emailPrefix === githubUsername) {
return true;
}
const domainName = emailDomain.split(".")[0];
if (githubUsername.endsWith(domainName) && githubUsername.length > domainName.length) {
const baseUsername = githubUsername.slice(0, -domainName.length);
if (emailPrefix === baseUsername) {
return true;
}
}
const emailSplitRegex = new RE2(/[._-]/);
const emailParts = emailPrefix.split(emailSplitRegex);
const longestEmailPart = emailParts.reduce((a, b) => (a.length > b.length ? a : b), "");
if (longestEmailPart.length >= 4 && githubUsername.includes(longestEmailPart)) {
return true;
}
return false;
});
if (matchingMember?.user?.id) {
expectedUserIds.add(matchingMember.user.id);
logger.info(
`Matched GitHub user ${githubUsername} to email ${matchingMember.user?.email || matchingMember.inviteEmail}`
);
}
});
}
});
const currentUserIds = new Set<string>();
currentMemberships.forEach((membership) => {
const activeMember = activeMembers.find((am) => am.id === membership.orgMembershipId);
if (activeMember?.user?.id) {
currentUserIds.add(activeMember.user.id);
}
});
const usersToAdd = Array.from(expectedUserIds).filter((userId) => !currentUserIds.has(userId));
const membershipsToRemove = currentMemberships.filter((membership) => {
const activeMember = activeMembers.find((am) => am.id === membership.orgMembershipId);
return activeMember?.user?.id && !expectedUserIds.has(activeMember.user.id);
});
if (usersToAdd.length > 0) {
await userGroupMembershipDAL.insertMany(
usersToAdd.map((userId) => ({
userId,
groupId: team.id
})),
tx
);
});
if (syncResult) {
syncResult.createdTeams.forEach((team) => createdTeams.add(team));
syncResult.updatedTeams.forEach((team) => updatedTeams.add(team));
totalRemovedMemberships += syncResult.removedMemberships;
updatedTeams.add(teamName);
}
syncedUsersCount += 1;
await delay(RATE_LIMIT_DELAY);
} catch (error) {
logger.error(error, `Failed to sync teams for user ${member.userId || "unknown"}`);
syncErrors.push(`User ${member.userId || "unknown"}: ${(error as Error).message}`);
if (membershipsToRemove.length > 0) {
await userGroupMembershipDAL.delete(
{
$in: {
id: membershipsToRemove.map((m) => m.id)
}
},
tx
);
updatedTeams.add(teamName);
}
}
}
});
const syncDuration = Date.now() - startTime;
logger.info(
{
orgId: orgPermission.orgId,
syncedUsersCount,
totalUsers: activeMembers.length,
createdTeams: createdTeams.size,
updatedTeams: updatedTeams.size,
removedMemberships: totalRemovedMemberships,
syncDuration,
errorCount: syncErrors.length
syncDuration
},
"GitHub team sync completed"
);

1
backend/src/ee/services/github-org-sync/github-org-sync-types.ts
View File

@@ -24,7 +24,6 @@ export interface TGetGithubOrgSyncDTO {
export interface TSyncAllTeamsDTO {
orgPermission: OrgServiceActor;
githubOrgAccessToken?: string;
}
export interface TSyncResult {

68
backend/src/ee/services/license/license-fns.ts
View File

@@ -18,51 +18,51 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({
environmentsUsed: 0,
identityLimit: null,
identitiesUsed: 0,
dynamicSecret: false,
dynamicSecret: true,
secretVersioning: true,
pitRecovery: false,
ipAllowlisting: false,
rbac: false,
githubOrgSync: false,
customRateLimits: false,
customAlerts: false,
secretAccessInsights: false,
auditLogs: false,
pitRecovery: true,
ipAllowlisting: true,
rbac: true,
githubOrgSync: true,
customRateLimits: true,
customAlerts: true,
secretAccessInsights: true,
auditLogs: true,
auditLogsRetentionDays: 0,
auditLogStreams: false,
auditLogStreams: true,
auditLogStreamLimit: 3,
samlSSO: false,
enforceGoogleSSO: false,
hsm: false,
oidcSSO: false,
scim: false,
ldap: false,
groups: false,
samlSSO: true,
enforceGoogleSSO: true,
hsm: true,
oidcSSO: true,
scim: true,
ldap: true,
groups: true,
status: null,
trial_end: null,
has_used_trial: true,
secretApproval: false,
secretRotation: false,
caCrl: false,
instanceUserManagement: false,
externalKms: false,
secretApproval: true,
secretRotation: true,
caCrl: true,
instanceUserManagement: true,
externalKms: true,
rateLimits: {
readLimit: 60,
writeLimit: 200,
secretsLimit: 40
},
pkiEst: false,
enforceMfa: false,
projectTemplates: false,
kmip: false,
gateway: false,
sshHostGroups: false,
secretScanning: false,
enterpriseSecretSyncs: false,
enterpriseAppConnections: false,
fips: false,
eventSubscriptions: false,
machineIdentityAuthTemplates: false
pkiEst: true,
enforceMfa: true,
projectTemplates: true,
kmip: true,
gateway: true,
sshHostGroups: true,
secretScanning: true,
enterpriseSecretSyncs: true,
enterpriseAppConnections: true,
fips: true,
eventSubscriptions: true,
machineIdentityAuthTemplates: true
});
export const setupLicenseRequestWithStore = (

11
backend/src/ee/services/permission/org-permission.ts
View File

@@ -90,6 +90,7 @@ export enum OrgPermissionSubjects {
Sso = "sso",
Scim = "scim",
GithubOrgSync = "github-org-sync",
GithubOrgSyncManual = "github-org-sync-manual",
Ldap = "ldap",
Groups = "groups",
Billing = "billing",
@@ -119,6 +120,7 @@ export type OrgPermissionSet =
| [OrgPermissionActions, OrgPermissionSubjects.Sso]
| [OrgPermissionActions, OrgPermissionSubjects.Scim]
| [OrgPermissionActions, OrgPermissionSubjects.GithubOrgSync]
| [OrgPermissionActions, OrgPermissionSubjects.GithubOrgSyncManual]
| [OrgPermissionActions, OrgPermissionSubjects.Ldap]
| [OrgPermissionGroupActions, OrgPermissionSubjects.Groups]
| [OrgPermissionActions, OrgPermissionSubjects.SecretScanning]
@@ -188,6 +190,10 @@ export const OrgPermissionSchema = z.discriminatedUnion("subject", [
subject: z.literal(OrgPermissionSubjects.GithubOrgSync).describe("The entity this permission pertains to."),
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.")
}),
z.object({
subject: z.literal(OrgPermissionSubjects.GithubOrgSyncManual).describe("The entity this permission pertains to."),
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.")
}),
z.object({
subject: z.literal(OrgPermissionSubjects.Ldap).describe("The entity this permission pertains to."),
action: CASL_ACTION_SCHEMA_NATIVE_ENUM(OrgPermissionActions).describe("Describe what action an entity can take.")
@@ -309,6 +315,11 @@ const buildAdminPermission = () => {
can(OrgPermissionActions.Edit, OrgPermissionSubjects.GithubOrgSync);
can(OrgPermissionActions.Delete, OrgPermissionSubjects.GithubOrgSync);
can(OrgPermissionActions.Read, OrgPermissionSubjects.GithubOrgSyncManual);
can(OrgPermissionActions.Create, OrgPermissionSubjects.GithubOrgSyncManual);
can(OrgPermissionActions.Edit, OrgPermissionSubjects.GithubOrgSyncManual);
can(OrgPermissionActions.Delete, OrgPermissionSubjects.GithubOrgSyncManual);
can(OrgPermissionActions.Read, OrgPermissionSubjects.Ldap);
can(OrgPermissionActions.Create, OrgPermissionSubjects.Ldap);
can(OrgPermissionActions.Edit, OrgPermissionSubjects.Ldap);

43
backend/src/lib/retry/index.ts Normal file
View File

@@ -0,0 +1,43 @@
/* eslint-disable no-await-in-loop */
interface GitHubApiError extends Error {
status?: number;
response?: {
status?: number;
headers?: {
"x-ratelimit-reset"?: string;
};
};
}
const delay = (ms: number) =>
new Promise<void>((resolve) => {
setTimeout(() => resolve(), ms);
});
export const retryWithBackoff = async <T>(fn: () => Promise<T>, maxRetries = 3, baseDelay = 1000): Promise<T> => {
let lastError: Error;
for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
try {
return await fn();
} catch (error) {
lastError = error as Error;
const gitHubError = error as GitHubApiError;
const statusCode = gitHubError.status || gitHubError.response?.status;
if (statusCode === 403) {
const rateLimitReset = gitHubError.response?.headers?.["x-ratelimit-reset"];
if (rateLimitReset) {
const resetTime = parseInt(rateLimitReset, 10) * 1000;
const waitTime = Math.max(resetTime - Date.now(), baseDelay);
await delay(Math.min(waitTime, 60000));
} else {
await delay(baseDelay * 2 ** attempt);
}
} else if (attempt < maxRetries) {
await delay(baseDelay * 2 ** attempt);
}
}
}
throw lastError!;
};

3
backend/src/server/routes/index.ts
View File

@@ -681,8 +681,7 @@ export const registerRoutes = async (
permissionService,
groupDAL,
userGroupMembershipDAL,
orgMembershipDAL,
identityMetadataDAL
orgMembershipDAL
});
const ldapService = ldapConfigServiceFactory({

56
backend/src/services/org-membership/org-membership-dal.ts
View File

@@ -153,10 +153,64 @@ export const orgMembershipDALFactory = (db: TDbClient) => {
}
};
const findOrgMembershipsWithUsersByOrgId = async (orgId: string) => {
try {
const members = await db
.replicaNode()(TableName.OrgMembership)
.where(`${TableName.OrgMembership}.orgId`, orgId)
.join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`)
.leftJoin<TUserEncryptionKeys>(
TableName.UserEncryptionKey,
`${TableName.UserEncryptionKey}.userId`,
`${TableName.Users}.id`
)
.leftJoin(TableName.IdentityMetadata, (queryBuilder) => {
void queryBuilder
.on(`${TableName.OrgMembership}.userId`, `${TableName.IdentityMetadata}.userId`)
.andOn(`${TableName.OrgMembership}.orgId`, `${TableName.IdentityMetadata}.orgId`);
})
.select(
db.ref("id").withSchema(TableName.OrgMembership),
db.ref("inviteEmail").withSchema(TableName.OrgMembership),
db.ref("orgId").withSchema(TableName.OrgMembership),
db.ref("role").withSchema(TableName.OrgMembership),
db.ref("roleId").withSchema(TableName.OrgMembership),
db.ref("status").withSchema(TableName.OrgMembership),
db.ref("isActive").withSchema(TableName.OrgMembership),
db.ref("email").withSchema(TableName.Users),
db.ref("username").withSchema(TableName.Users),
db.ref("firstName").withSchema(TableName.Users),
db.ref("lastName").withSchema(TableName.Users),
db.ref("isEmailVerified").withSchema(TableName.Users),
db.ref("id").withSchema(TableName.Users).as("userId")
)
.where({ isGhost: false });
return members.map((member) => ({
id: member.id,
orgId: member.orgId,
role: member.role,
status: member.status,
isActive: member.isActive,
inviteEmail: member.inviteEmail,
user: {
id: member.userId,
email: member.email,
username: member.username,
firstName: member.firstName,
lastName: member.lastName
}
}));
} catch (error) {
throw new DatabaseError({ error, name: "Find org memberships with users by org id" });
}
};
return {
...orgMembershipOrm,
findOrgMembershipById,
findRecentInvitedMemberships,
updateLastInvitedAtByIds
updateLastInvitedAtByIds,
findOrgMembershipsWithUsersByOrgId
};
};

2
frontend/src/context/OrgPermissionContext/types.ts
View File

@@ -52,6 +52,7 @@ export enum OrgPermissionSubjects {
Gateway = "gateway",
SecretShare = "secret-share",
GithubOrgSync = "github-org-sync",
GithubOrgSyncManual = "github-org-sync-manual",
MachineIdentityAuthTemplate = "machine-identity-auth-template"
}
@@ -111,6 +112,7 @@ export type OrgPermissionSet =
| [OrgPermissionActions, OrgPermissionSubjects.IncidentAccount]
| [OrgPermissionActions, OrgPermissionSubjects.Scim]
| [OrgPermissionActions, OrgPermissionSubjects.GithubOrgSync]
| [OrgPermissionActions, OrgPermissionSubjects.GithubOrgSyncManual]
| [OrgPermissionActions, OrgPermissionSubjects.Sso]
| [OrgPermissionActions, OrgPermissionSubjects.Ldap]
| [OrgPermissionGroupActions, OrgPermissionSubjects.Groups]

3
frontend/src/hooks/api/githubOrgSyncConfig/index.tsx
View File

@@ -2,7 +2,6 @@ export {
useCreateGithubSyncOrgConfig,
useDeleteGithubSyncOrgConfig,
useSyncAllGithubTeams,
useUpdateGithubSyncOrgConfig,
useValidateGithubToken
useUpdateGithubSyncOrgConfig
} from "./mutations";
export { githubOrgSyncConfigQueryKeys } from "./queries";

34
frontend/src/hooks/api/githubOrgSyncConfig/mutations.tsx
View File

@@ -43,11 +43,7 @@ export const useDeleteGithubSyncOrgConfig = () => {
export const useSyncAllGithubTeams = () => {
return useMutation({
mutationFn: async ({
githubOrgAccessToken
}: {
githubOrgAccessToken?: string;
} = {}): Promise<{
mutationFn: async (): Promise<{
syncedUsersCount: number;
totalUsers: number;
errors: string[];
@@ -56,33 +52,7 @@ export const useSyncAllGithubTeams = () => {
removedMemberships: number;
syncDuration: number;
}> => {
const response = await apiRequest.post("/api/v1/github-org-sync-config/sync-all-teams", {
githubOrgAccessToken
});
return response.data;
}
});
};
export const useValidateGithubToken = () => {
return useMutation({
mutationFn: async ({
githubOrgAccessToken
}: {
githubOrgAccessToken: string;
}): Promise<{
valid: boolean;
organizationInfo?: {
id: number;
login: string;
name: string;
publicRepos?: number;
privateRepos?: number;
};
}> => {
const response = await apiRequest.post("/api/v1/github-org-sync-config/validate-token", {
githubOrgAccessToken
});
const response = await apiRequest.post("/api/v1/github-org-sync-config/sync-all-teams");
return response.data;
}
});

133
frontend/src/pages/organization/SettingsPage/components/OrgProvisioningTab/OrgGithubSyncSection.tsx
View File

@@ -20,8 +20,7 @@ import { OrgPermissionActions, OrgPermissionSubjects, useSubscription } from "@a
import {
githubOrgSyncConfigQueryKeys,
useSyncAllGithubTeams,
useUpdateGithubSyncOrgConfig,
useValidateGithubToken
useUpdateGithubSyncOrgConfig
} from "@app/hooks/api";
import { usePopUp } from "@app/hooks/usePopUp";
@@ -33,11 +32,10 @@ export const OrgGithubSyncSection = () => {
"upgradePlan",
"githubOrgSyncConfig",
"deleteGithubOrgSyncConfig",
"syncAllTeamsToken"
"setAccessToken"
] as const);
const [accessToken, setAccessToken] = useState("");
const [isValidatingToken, setIsValidatingToken] = useState(false);
const [tokenValidationResult, setTokenValidationResult] = useState<{
valid: boolean;
organizationInfo?: {
@@ -57,16 +55,13 @@ export const OrgGithubSyncSection = () => {
const updateGithubSyncOrgConfig = useUpdateGithubSyncOrgConfig();
const syncAllTeamsMutation = useSyncAllGithubTeams();
const validateGithubTokenMutation = useValidateGithubToken();
const isPending = subscription.githubOrgSync && githubOrgSyncConfig.isPending;
const data = !isPending && !githubOrgSyncConfig?.isError ? githubOrgSyncConfig?.data : undefined;
const handleBulkSync = async (token?: string) => {
const handleBulkSync = async () => {
try {
const result = await syncAllTeamsMutation.mutateAsync({
githubOrgAccessToken: token
});
const result = await syncAllTeamsMutation.mutateAsync();
let message = `Successfully synced teams for ${result.syncedUsersCount} user${result.syncedUsersCount === 1 ? "" : "s"}`;
const details = [];
@@ -102,9 +97,6 @@ export const OrgGithubSyncSection = () => {
});
console.warn("Sync errors:", result.errors);
}
setAccessToken("");
handlePopUpToggle("syncAllTeamsToken", false);
} catch (error) {
const errorMessage =
(error as any)?.response?.data?.message || (error as Error)?.message || "Unknown error";
@@ -113,11 +105,12 @@ export const OrgGithubSyncSection = () => {
errorMessage.includes("token") &&
(errorMessage.includes("required") ||
errorMessage.includes("invalid") ||
errorMessage.includes("expired"))
errorMessage.includes("expired") ||
errorMessage.includes("set a token first"))
) {
handlePopUpOpen("syncAllTeamsToken");
handlePopUpOpen("setAccessToken");
createNotification({
text: errorMessage,
text: "Please provide a GitHub access token to continue with the sync",
type: "error"
});
} else {
@@ -129,48 +122,7 @@ export const OrgGithubSyncSection = () => {
}
};
const validateToken = async () => {
if (!accessToken.trim()) {
createNotification({
text: "Please enter a GitHub access token",
type: "error"
});
return false;
}
setIsValidatingToken(true);
try {
const result = await validateGithubTokenMutation.mutateAsync({
githubOrgAccessToken: accessToken.trim()
});
setTokenValidationResult(result);
if (result.valid && result.organizationInfo) {
createNotification({
text: `Token validated successfully for organization: ${result.organizationInfo.name}`,
type: "success"
});
}
return result.valid;
} catch (error) {
const errorMessage =
(error as any)?.response?.data?.message ||
(error as Error)?.message ||
"Token validation failed";
createNotification({
text: errorMessage,
type: "error"
});
setTokenValidationResult({ valid: false });
return false;
} finally {
setIsValidatingToken(false);
}
};
const handleSyncWithToken = async () => {
const handleSetAccessToken = async () => {
if (!accessToken.trim()) {
createNotification({
text: "Please enter a GitHub access token",
@@ -179,15 +131,30 @@ export const OrgGithubSyncSection = () => {
return;
}
let isTokenValid = tokenValidationResult?.valid ?? false;
if (!isTokenValid) {
isTokenValid = await validateToken();
if (!isTokenValid) {
return;
}
}
try {
await updateGithubSyncOrgConfig.mutateAsync({
githubOrgAccessToken: accessToken.trim()
});
await handleBulkSync(accessToken.trim());
createNotification({
text: "GitHub access token set successfully. Starting sync...",
type: "success"
});
setAccessToken("");
handlePopUpToggle("setAccessToken", false);
// Automatically trigger sync after token is set
await handleBulkSync();
} catch (error) {
const errorMessage =
(error as any)?.response?.data?.message || (error as Error)?.message || "Unknown error";
createNotification({
text: `Failed to set GitHub access token: ${errorMessage}`,
type: "error"
});
}
};
return (
@@ -252,7 +219,10 @@ export const OrgGithubSyncSection = () => {
<div className="py-4">
<div className="mb-2 flex items-center justify-between">
<h2 className="text-md text-mineshaft-100">Sync Now</h2>
<OrgPermissionCan I={OrgPermissionActions.Edit} a={OrgPermissionSubjects.GithubOrgSync}>
<OrgPermissionCan
I={OrgPermissionActions.Edit}
a={OrgPermissionSubjects.GithubOrgSyncManual}
>
{(isAllowed) => (
<Button
onClick={() => handleBulkSync()}
@@ -296,9 +266,9 @@ export const OrgGithubSyncSection = () => {
text="You can use GitHub Organization Plan if you switch to Infisical's Enterprise plan."
/>
<Modal
isOpen={popUp?.syncAllTeamsToken?.isOpen}
isOpen={popUp?.setAccessToken?.isOpen}
onOpenChange={(isOpen) => {
handlePopUpToggle("syncAllTeamsToken", isOpen);
handlePopUpToggle("setAccessToken", isOpen);
if (!isOpen) {
setAccessToken("");
setTokenValidationResult(null);
@@ -307,7 +277,7 @@ export const OrgGithubSyncSection = () => {
>
<ModalContent
title="GitHub Access Token Required"
subTitle="Provide a GitHub access token with organization and team access permissions"
subTitle="Provide a GitHub access token to sync teams from your GitHub organization"
>
<div className="space-y-4">
<FormControl
@@ -340,22 +310,11 @@ export const OrgGithubSyncSection = () => {
</div>
</FormControl>
<div className="flex justify-between">
<Button
colorSchema="secondary"
variant="outline_bg"
onClick={validateToken}
isLoading={isValidatingToken}
isDisabled={
!accessToken.trim() || isValidatingToken || syncAllTeamsMutation.isPending
}
>
Validate Token
</Button>
<div className="flex space-x-2">
<Button
colorSchema="secondary"
onClick={() => {
handlePopUpToggle("syncAllTeamsToken", false);
handlePopUpToggle("setAccessToken", false);
setAccessToken("");
setTokenValidationResult(null);
}}
@@ -364,11 +323,15 @@ export const OrgGithubSyncSection = () => {
</Button>
<Button
colorSchema="primary"
onClick={handleSyncWithToken}
isLoading={syncAllTeamsMutation.isPending}
isDisabled={!accessToken.trim() || syncAllTeamsMutation.isPending}
onClick={handleSetAccessToken}
isLoading={updateGithubSyncOrgConfig.isPending || syncAllTeamsMutation.isPending}
isDisabled={
!accessToken.trim() ||
updateGithubSyncOrgConfig.isPending ||
syncAllTeamsMutation.isPending
}
>
Sync Teams
Set Token & Sync
</Button>
</div>
</div>