merge main

This commit is contained in:
Scott Wilson
2025-07-30 16:38:35 -07:00
369 changed files with 2581 additions and 27780 deletions
@@ -583,7 +583,7 @@ export const appConnectionServiceFactory = ({
deleteAppConnection,
connectAppConnectionById,
listAvailableAppConnectionsForUser,
github: githubConnectionService(connectAppConnectionById),
github: githubConnectionService(connectAppConnectionById, gatewayService),
githubRadar: githubRadarConnectionService(connectAppConnectionById),
gcp: gcpConnectionService(connectAppConnectionById),
databricks: databricksConnectionService(connectAppConnectionById, appConnectionDAL, kmsService),
@@ -1,3 +1,4 @@
export enum AzureAppConfigurationConnectionMethod {
OAuth = "oauth"
OAuth = "oauth",
ClientSecret = "client-secret"
}
@@ -1,3 +1,4 @@
/* eslint-disable no-case-declarations */
import { AxiosError, AxiosResponse } from "axios";
import { getConfig } from "@app/lib/config/env";
@@ -19,7 +20,10 @@ export const getAzureAppConfigurationConnectionListItem = () => {
return {
name: "Azure App Configuration" as const,
app: AppConnection.AzureAppConfiguration as const,
methods: Object.values(AzureAppConfigurationConnectionMethod) as [AzureAppConfigurationConnectionMethod.OAuth],
methods: Object.values(AzureAppConfigurationConnectionMethod) as [
AzureAppConfigurationConnectionMethod.OAuth,
AzureAppConfigurationConnectionMethod.ClientSecret
],
oauthClientId: INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_ID
};
};
@@ -35,71 +39,111 @@ export const validateAzureAppConfigurationConnectionCredentials = async (
SITE_URL
} = getConfig();
if (
!INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_ID ||
!INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_SECRET
) {
throw new InternalServerError({
message: `Azure ${getAppConnectionMethodName(method)} environment variables have not been configured`
});
}
let tokenResp: AxiosResponse<ExchangeCodeAzureResponse> | null = null;
let tokenError: AxiosError | null = null;
try {
tokenResp = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", inputCredentials.tenantId || "common"),
new URLSearchParams({
grant_type: "authorization_code",
code: inputCredentials.code,
scope: `openid offline_access https://azconfig.io/.default`,
client_id: INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_ID,
client_secret: INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_SECRET,
redirect_uri: `${SITE_URL}/organization/app-connections/azure/oauth/callback`
})
);
} catch (e: unknown) {
if (e instanceof AxiosError) {
tokenError = e;
} else {
throw new BadRequestError({
message: `Unable to validate connection: verify credentials`
});
}
}
if (tokenError) {
if (tokenError instanceof AxiosError) {
throw new BadRequestError({
message: `Failed to get access token: ${
(tokenError?.response?.data as { error_description?: string })?.error_description || "Unknown error"
}`
});
} else {
throw new InternalServerError({
message: "Failed to get access token"
});
}
}
if (!tokenResp) {
throw new InternalServerError({
message: `Failed to get access token: Token was empty with no error`
});
}
switch (method) {
case AzureAppConfigurationConnectionMethod.OAuth:
if (
!INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_ID ||
!INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_SECRET
) {
throw new InternalServerError({
message: `Azure ${getAppConnectionMethodName(method)} environment variables have not been configured`
});
}
let tokenResp: AxiosResponse<ExchangeCodeAzureResponse> | null = null;
let tokenError: AxiosError | null = null;
const oauthCredentials = inputCredentials as { code: string; tenantId?: string };
try {
tokenResp = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", oauthCredentials.tenantId || "common"),
new URLSearchParams({
grant_type: "authorization_code",
code: oauthCredentials.code,
scope: `openid offline_access https://azconfig.io/.default`,
client_id: INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_ID,
client_secret: INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_SECRET,
redirect_uri: `${SITE_URL}/organization/app-connections/azure/oauth/callback`
})
);
} catch (e: unknown) {
if (e instanceof AxiosError) {
tokenError = e;
} else {
throw new BadRequestError({
message: `Unable to validate connection: verify credentials`
});
}
}
if (tokenError) {
if (tokenError instanceof AxiosError) {
throw new BadRequestError({
message: `Failed to get access token: ${
(tokenError?.response?.data as { error_description?: string })?.error_description || "Unknown error"
}`
});
} else {
throw new InternalServerError({
message: "Failed to get access token"
});
}
}
if (!tokenResp) {
throw new InternalServerError({
message: `Failed to get access token: Token was empty with no error`
});
}
return {
tenantId: inputCredentials.tenantId,
tenantId: oauthCredentials.tenantId,
accessToken: tokenResp.data.access_token,
refreshToken: tokenResp.data.refresh_token,
expiresAt: Date.now() + tokenResp.data.expires_in * 1000
};
case AzureAppConfigurationConnectionMethod.ClientSecret:
const { tenantId, clientId, clientSecret } = inputCredentials as {
tenantId: string;
clientId: string;
clientSecret: string;
};
try {
const { data: clientData } = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", tenantId || "common"),
new URLSearchParams({
grant_type: "client_credentials",
scope: `https://azconfig.io/.default`,
client_id: clientId,
client_secret: clientSecret
})
);
return {
tenantId,
accessToken: clientData.access_token,
expiresAt: Date.now() + clientData.expires_in * 1000,
clientId,
clientSecret
};
} catch (e: unknown) {
if (e instanceof AxiosError) {
throw new BadRequestError({
message: `Failed to get access token: ${
(e?.response?.data as { error_description?: string })?.error_description || "Unknown error"
}`
});
} else {
throw new InternalServerError({
message: "Failed to get access token"
});
}
}
default:
throw new InternalServerError({
message: `Unhandled Azure connection method: ${method as AzureAppConfigurationConnectionMethod}`
message: `Unhandled Azure App Configuration connection method: ${method as AzureAppConfigurationConnectionMethod}`
});
}
};
@@ -22,6 +22,29 @@ export const AzureAppConfigurationConnectionOAuthOutputCredentialsSchema = z.obj
expiresAt: z.number()
});
export const AzureAppConfigurationConnectionClientSecretInputCredentialsSchema = z.object({
clientId: z
.string()
.uuid()
.trim()
.min(1, "Client ID required")
.max(50, "Client ID must be at most 50 characters long"),
clientSecret: z
.string()
.trim()
.min(1, "Client Secret required")
.max(50, "Client Secret must be at most 50 characters long"),
tenantId: z.string().uuid().trim().min(1, "Tenant ID required")
});
export const AzureAppConfigurationConnectionClientSecretOutputCredentialsSchema = z.object({
clientId: z.string(),
clientSecret: z.string(),
tenantId: z.string(),
accessToken: z.string(),
expiresAt: z.number()
});
export const ValidateAzureAppConfigurationConnectionCredentialsSchema = z.discriminatedUnion("method", [
z.object({
method: z
@@ -30,6 +53,14 @@ export const ValidateAzureAppConfigurationConnectionCredentialsSchema = z.discri
credentials: AzureAppConfigurationConnectionOAuthInputCredentialsSchema.describe(
AppConnections.CREATE(AppConnection.AzureAppConfiguration).credentials
)
}),
z.object({
method: z
.literal(AzureAppConfigurationConnectionMethod.ClientSecret)
.describe(AppConnections.CREATE(AppConnection.AzureAppConfiguration).method),
credentials: AzureAppConfigurationConnectionClientSecretInputCredentialsSchema.describe(
AppConnections.CREATE(AppConnection.AzureAppConfiguration).credentials
)
})
]);
@@ -39,9 +70,13 @@ export const CreateAzureAppConfigurationConnectionSchema = ValidateAzureAppConfi
export const UpdateAzureAppConfigurationConnectionSchema = z
.object({
credentials: AzureAppConfigurationConnectionOAuthInputCredentialsSchema.optional().describe(
AppConnections.UPDATE(AppConnection.AzureAppConfiguration).credentials
)
credentials: z
.union([
AzureAppConfigurationConnectionOAuthInputCredentialsSchema,
AzureAppConfigurationConnectionClientSecretInputCredentialsSchema
])
.optional()
.describe(AppConnections.UPDATE(AppConnection.AzureAppConfiguration).credentials)
})
.and(GenericUpdateAppConnectionFieldsSchema(AppConnection.AzureAppConfiguration));
@@ -55,6 +90,10 @@ export const AzureAppConfigurationConnectionSchema = z.intersection(
z.object({
method: z.literal(AzureAppConfigurationConnectionMethod.OAuth),
credentials: AzureAppConfigurationConnectionOAuthOutputCredentialsSchema
}),
z.object({
method: z.literal(AzureAppConfigurationConnectionMethod.ClientSecret),
credentials: AzureAppConfigurationConnectionClientSecretOutputCredentialsSchema
})
])
);
@@ -65,6 +104,13 @@ export const SanitizedAzureAppConfigurationConnectionSchema = z.discriminatedUni
credentials: AzureAppConfigurationConnectionOAuthOutputCredentialsSchema.pick({
tenantId: true
})
}),
BaseAzureAppConfigurationConnectionSchema.extend({
method: z.literal(AzureAppConfigurationConnectionMethod.ClientSecret),
credentials: AzureAppConfigurationConnectionClientSecretOutputCredentialsSchema.pick({
clientId: true,
tenantId: true
})
})
]);
@@ -4,6 +4,7 @@ import { DiscriminativePick } from "@app/lib/types";
import { AppConnection } from "../app-connection-enums";
import {
AzureAppConfigurationConnectionClientSecretOutputCredentialsSchema,
AzureAppConfigurationConnectionOAuthOutputCredentialsSchema,
AzureAppConfigurationConnectionSchema,
CreateAzureAppConfigurationConnectionSchema,
@@ -39,3 +40,7 @@ export type ExchangeCodeAzureResponse = {
export type TAzureAppConfigurationConnectionCredentials = z.infer<
typeof AzureAppConfigurationConnectionOAuthOutputCredentialsSchema
>;
export type TAzureAppConfigurationConnectionClientSecretCredentials = z.infer<
typeof AzureAppConfigurationConnectionClientSecretOutputCredentialsSchema
>;
@@ -1,4 +1,5 @@
export enum AzureDevOpsConnectionMethod {
OAuth = "oauth",
AccessToken = "access-token"
AccessToken = "access-token",
ClientSecret = "client-secret"
}
@@ -18,6 +18,7 @@ import { AppConnection } from "../app-connection-enums";
import { AzureDevOpsConnectionMethod } from "./azure-devops-enums";
import {
ExchangeCodeAzureResponse,
TAzureDevOpsConnectionClientSecretCredentials,
TAzureDevOpsConnectionConfig,
TAzureDevOpsConnectionCredentials
} from "./azure-devops-types";
@@ -30,7 +31,8 @@ export const getAzureDevopsConnectionListItem = () => {
app: AppConnection.AzureDevOps as const,
methods: Object.values(AzureDevOpsConnectionMethod) as [
AzureDevOpsConnectionMethod.OAuth,
AzureDevOpsConnectionMethod.AccessToken
AzureDevOpsConnectionMethod.AccessToken,
AzureDevOpsConnectionMethod.ClientSecret
],
oauthClientId: INF_APP_CONNECTION_AZURE_DEVOPS_CLIENT_ID
};
@@ -53,11 +55,7 @@ export const getAzureDevopsConnection = async (
});
}
const credentials = (await decryptAppConnectionCredentials({
orgId: appConnection.orgId,
kmsService,
encryptedCredentials: appConnection.encryptedCredentials
})) as TAzureDevOpsConnectionCredentials;
const currentTime = Date.now();
// Handle different connection methods
switch (appConnection.method) {
@@ -69,12 +67,17 @@ export const getAzureDevopsConnection = async (
});
}
if (!("refreshToken" in credentials)) {
const oauthCredentials = (await decryptAppConnectionCredentials({
orgId: appConnection.orgId,
kmsService,
encryptedCredentials: appConnection.encryptedCredentials
})) as TAzureDevOpsConnectionCredentials;
if (!("refreshToken" in oauthCredentials)) {
throw new BadRequestError({ message: "Invalid OAuth credentials" });
}
const { refreshToken, tenantId } = credentials;
const currentTime = Date.now();
const { refreshToken, tenantId } = oauthCredentials;
const { data } = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", tenantId || "common"),
@@ -87,29 +90,75 @@ export const getAzureDevopsConnection = async (
})
);
const updatedCredentials = {
...credentials,
const updatedOAuthCredentials = {
...oauthCredentials,
accessToken: data.access_token,
expiresAt: currentTime + data.expires_in * 1000,
refreshToken: data.refresh_token
};
const encryptedCredentials = await encryptAppConnectionCredentials({
credentials: updatedCredentials,
const encryptedOAuthCredentials = await encryptAppConnectionCredentials({
credentials: updatedOAuthCredentials,
orgId: appConnection.orgId,
kmsService
});
await appConnectionDAL.updateById(appConnection.id, { encryptedCredentials });
await appConnectionDAL.updateById(appConnection.id, { encryptedCredentials: encryptedOAuthCredentials });
return data.access_token;
case AzureDevOpsConnectionMethod.AccessToken:
if (!("accessToken" in credentials)) {
const accessTokenCredentials = (await decryptAppConnectionCredentials({
orgId: appConnection.orgId,
kmsService,
encryptedCredentials: appConnection.encryptedCredentials
})) as { accessToken: string };
if (!("accessToken" in accessTokenCredentials)) {
throw new BadRequestError({ message: "Invalid API token credentials" });
}
// For access token, return the basic auth token directly
return credentials.accessToken;
return accessTokenCredentials.accessToken;
case AzureDevOpsConnectionMethod.ClientSecret:
const clientSecretCredentials = (await decryptAppConnectionCredentials({
orgId: appConnection.orgId,
kmsService,
encryptedCredentials: appConnection.encryptedCredentials
})) as TAzureDevOpsConnectionClientSecretCredentials;
const { accessToken, expiresAt, clientId, clientSecret, tenantId: clientTenantId } = clientSecretCredentials;
// Check if token is still valid (with 5 minute buffer)
if (accessToken && expiresAt && expiresAt > currentTime + 300000) {
return accessToken;
}
const { data: clientData } = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", clientTenantId || "common"),
new URLSearchParams({
grant_type: "client_credentials",
scope: `https://app.vssps.visualstudio.com/.default`,
client_id: clientId,
client_secret: clientSecret
})
);
const updatedClientCredentials = {
...clientSecretCredentials,
accessToken: clientData.access_token,
expiresAt: currentTime + clientData.expires_in * 1000
};
const encryptedClientCredentials = await encryptAppConnectionCredentials({
credentials: updatedClientCredentials,
orgId: appConnection.orgId,
kmsService
});
await appConnectionDAL.updateById(appConnection.id, { encryptedCredentials: encryptedClientCredentials });
return clientData.access_token;
default:
throw new BadRequestError({ message: `Unsupported connection method` });
@@ -138,7 +187,7 @@ export const validateAzureDevOpsConnectionCredentials = async (config: TAzureDev
let tokenError: AxiosError | null = null;
try {
const oauthCredentials = inputCredentials as { code: string; tenantId: string };
const oauthCredentials = inputCredentials as { code: string; tenantId: string; orgName: string };
tokenResp = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", oauthCredentials.tenantId || "common"),
new URLSearchParams({
@@ -262,9 +311,67 @@ export const validateAzureDevOpsConnectionCredentials = async (config: TAzureDev
});
}
case AzureDevOpsConnectionMethod.ClientSecret:
const { tenantId, clientId, clientSecret, orgName } = inputCredentials as {
tenantId: string;
clientId: string;
clientSecret: string;
orgName: string;
};
try {
// First, get the access token using client credentials flow
const { data: clientData } = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", tenantId || "common"),
new URLSearchParams({
grant_type: "client_credentials",
scope: `https://app.vssps.visualstudio.com/.default`,
client_id: clientId,
client_secret: clientSecret
})
);
// Validate access to the specific organization
const response = await request.get(
`${IntegrationUrls.AZURE_DEVOPS_API_URL}/${encodeURIComponent(orgName)}/_apis/projects?api-version=7.2-preview.2&$top=1`,
{
headers: {
Authorization: `Bearer ${clientData.access_token}`
}
}
);
if (response.status !== 200) {
throw new BadRequestError({
message: `Failed to validate connection to organization '${orgName}': ${response.status}`
});
}
return {
tenantId,
clientId,
clientSecret,
orgName,
accessToken: clientData.access_token,
expiresAt: Date.now() + clientData.expires_in * 1000
};
} catch (e: unknown) {
if (e instanceof AxiosError) {
throw new BadRequestError({
message: `Failed to authenticate with Azure DevOps using client credentials: ${
(e?.response?.data as { error_description?: string })?.error_description || e.message
}`
});
} else {
throw new InternalServerError({
message: "Failed to validate Azure DevOps client credentials"
});
}
}
default:
throw new InternalServerError({
message: `Unhandled Azure connection method: ${method as AzureDevOpsConnectionMethod}`
message: `Unhandled Azure DevOps connection method: ${method as AzureDevOpsConnectionMethod}`
});
}
};
@@ -38,6 +38,42 @@ export const AzureDevOpsConnectionAccessTokenOutputCredentialsSchema = z.object(
orgName: z.string()
});
export const AzureDevOpsConnectionClientSecretInputCredentialsSchema = z.object({
clientId: z
.string()
.uuid()
.trim()
.min(1, "Client ID required")
.max(50, "Client ID must be at most 50 characters long")
.describe(AppConnections.CREDENTIALS.AZURE_DEVOPS.clientId),
clientSecret: z
.string()
.trim()
.min(1, "Client Secret required")
.max(50, "Client Secret must be at most 50 characters long")
.describe(AppConnections.CREDENTIALS.AZURE_DEVOPS.clientSecret),
tenantId: z
.string()
.uuid()
.trim()
.min(1, "Tenant ID required")
.describe(AppConnections.CREDENTIALS.AZURE_DEVOPS.tenantId),
orgName: z
.string()
.trim()
.min(1, "Organization name required")
.describe(AppConnections.CREDENTIALS.AZURE_DEVOPS.orgName)
});
export const AzureDevOpsConnectionClientSecretOutputCredentialsSchema = z.object({
clientId: z.string(),
clientSecret: z.string(),
tenantId: z.string(),
orgName: z.string(),
accessToken: z.string(),
expiresAt: z.number()
});
export const ValidateAzureDevOpsConnectionCredentialsSchema = z.discriminatedUnion("method", [
z.object({
method: z
@@ -54,6 +90,14 @@ export const ValidateAzureDevOpsConnectionCredentialsSchema = z.discriminatedUni
credentials: AzureDevOpsConnectionAccessTokenInputCredentialsSchema.describe(
AppConnections.CREATE(AppConnection.AzureDevOps).credentials
)
}),
z.object({
method: z
.literal(AzureDevOpsConnectionMethod.ClientSecret)
.describe(AppConnections.CREATE(AppConnection.AzureDevOps).method),
credentials: AzureDevOpsConnectionClientSecretInputCredentialsSchema.describe(
AppConnections.CREATE(AppConnection.AzureDevOps).credentials
)
})
]);
@@ -64,7 +108,11 @@ export const CreateAzureDevOpsConnectionSchema = ValidateAzureDevOpsConnectionCr
export const UpdateAzureDevOpsConnectionSchema = z
.object({
credentials: z
.union([AzureDevOpsConnectionOAuthInputCredentialsSchema, AzureDevOpsConnectionAccessTokenInputCredentialsSchema])
.union([
AzureDevOpsConnectionOAuthInputCredentialsSchema,
AzureDevOpsConnectionAccessTokenInputCredentialsSchema,
AzureDevOpsConnectionClientSecretInputCredentialsSchema
])
.optional()
.describe(AppConnections.UPDATE(AppConnection.AzureDevOps).credentials)
})
@@ -84,6 +132,10 @@ export const AzureDevOpsConnectionSchema = z.intersection(
z.object({
method: z.literal(AzureDevOpsConnectionMethod.AccessToken),
credentials: AzureDevOpsConnectionAccessTokenOutputCredentialsSchema
}),
z.object({
method: z.literal(AzureDevOpsConnectionMethod.ClientSecret),
credentials: AzureDevOpsConnectionClientSecretOutputCredentialsSchema
})
])
);
@@ -101,6 +153,14 @@ export const SanitizedAzureDevOpsConnectionSchema = z.discriminatedUnion("method
credentials: AzureDevOpsConnectionAccessTokenOutputCredentialsSchema.pick({
orgName: true
})
}),
BaseAzureDevOpsConnectionSchema.extend({
method: z.literal(AzureDevOpsConnectionMethod.ClientSecret),
credentials: AzureDevOpsConnectionClientSecretOutputCredentialsSchema.pick({
clientId: true,
tenantId: true,
orgName: true
})
})
]);
@@ -52,6 +52,11 @@ const getAuthHeaders = (appConnection: TAzureDevOpsConnection, accessToken: stri
Authorization: `Basic ${basicAuthToken}`,
Accept: "application/json"
};
case AzureDevOpsConnectionMethod.ClientSecret:
return {
Authorization: `Bearer ${accessToken}`,
Accept: "application/json"
};
default:
throw new BadRequestError({ message: "Unsupported connection method" });
}
@@ -4,6 +4,7 @@ import { DiscriminativePick } from "@app/lib/types";
import { AppConnection } from "../app-connection-enums";
import {
AzureDevOpsConnectionClientSecretOutputCredentialsSchema,
AzureDevOpsConnectionOAuthOutputCredentialsSchema,
AzureDevOpsConnectionSchema,
CreateAzureDevOpsConnectionSchema,
@@ -27,6 +28,10 @@ export type TAzureDevOpsConnectionConfig = DiscriminativePick<
export type TAzureDevOpsConnectionCredentials = z.infer<typeof AzureDevOpsConnectionOAuthOutputCredentialsSchema>;
export type TAzureDevOpsConnectionClientSecretCredentials = z.infer<
typeof AzureDevOpsConnectionClientSecretOutputCredentialsSchema
>;
export interface ExchangeCodeAzureResponse {
token_type: string;
scope: string;
@@ -1,3 +1,4 @@
export enum AzureKeyVaultConnectionMethod {
OAuth = "oauth"
OAuth = "oauth",
ClientSecret = "client-secret"
}
@@ -1,3 +1,4 @@
/* eslint-disable no-case-declarations */
import { AxiosError, AxiosResponse } from "axios";
import { getConfig } from "@app/lib/config/env";
@@ -16,25 +17,16 @@ import { AppConnection } from "../app-connection-enums";
import { AzureKeyVaultConnectionMethod } from "./azure-key-vault-connection-enums";
import {
ExchangeCodeAzureResponse,
TAzureKeyVaultConnectionClientSecretCredentials,
TAzureKeyVaultConnectionConfig,
TAzureKeyVaultConnectionCredentials
} from "./azure-key-vault-connection-types";
export const getAzureConnectionAccessToken = async (
connectionId: string,
appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "update">,
appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "updateById">,
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">
) => {
const appCfg = getConfig();
if (
!appCfg.INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID ||
!appCfg.INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET
) {
throw new BadRequestError({
message: `Azure environment variables have not been configured`
});
}
const appConnection = await appConnectionDAL.findById(connectionId);
if (!appConnection) {
@@ -49,49 +41,101 @@ export const getAzureConnectionAccessToken = async (
throw new BadRequestError({ message: `Connection with ID '${connectionId}' is not a valid Azure connection` });
}
const credentials = (await decryptAppConnectionCredentials({
orgId: appConnection.orgId,
kmsService,
encryptedCredentials: appConnection.encryptedCredentials
})) as TAzureKeyVaultConnectionCredentials;
const currentTime = Date.now();
const { data } = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", credentials.tenantId || "common"),
new URLSearchParams({
grant_type: "refresh_token",
scope: `openid offline_access`,
client_id: appCfg.INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID,
client_secret: appCfg.INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET,
refresh_token: credentials.refreshToken
})
);
switch (appConnection.method) {
case AzureKeyVaultConnectionMethod.OAuth:
const appCfg = getConfig();
if (
!appCfg.INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID ||
!appCfg.INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET
) {
throw new BadRequestError({
message: `Azure environment variables have not been configured`
});
}
const accessExpiresAt = new Date();
accessExpiresAt.setSeconds(accessExpiresAt.getSeconds() + data.expires_in);
const oauthCredentials = (await decryptAppConnectionCredentials({
orgId: appConnection.orgId,
kmsService,
encryptedCredentials: appConnection.encryptedCredentials
})) as TAzureKeyVaultConnectionCredentials;
const updatedCredentials = {
...credentials,
accessToken: data.access_token,
expiresAt: accessExpiresAt.getTime(),
refreshToken: data.refresh_token
};
const { data } = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", oauthCredentials.tenantId || "common"),
new URLSearchParams({
grant_type: "refresh_token",
scope: `openid offline_access https://vault.azure.net/.default`,
client_id: appCfg.INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID,
client_secret: appCfg.INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET,
refresh_token: oauthCredentials.refreshToken
})
);
const encryptedCredentials = await encryptAppConnectionCredentials({
credentials: updatedCredentials,
orgId: appConnection.orgId,
kmsService
});
const updatedOAuthCredentials = {
...oauthCredentials,
accessToken: data.access_token,
expiresAt: currentTime + data.expires_in * 1000,
refreshToken: data.refresh_token
};
await appConnectionDAL.update(
{ id: connectionId },
{
encryptedCredentials
}
);
const encryptedOAuthCredentials = await encryptAppConnectionCredentials({
credentials: updatedOAuthCredentials,
orgId: appConnection.orgId,
kmsService
});
return {
accessToken: data.access_token
};
await appConnectionDAL.updateById(appConnection.id, { encryptedCredentials: encryptedOAuthCredentials });
return {
accessToken: data.access_token
};
case AzureKeyVaultConnectionMethod.ClientSecret:
const clientSecretCredentials = (await decryptAppConnectionCredentials({
orgId: appConnection.orgId,
kmsService,
encryptedCredentials: appConnection.encryptedCredentials
})) as TAzureKeyVaultConnectionClientSecretCredentials;
const { accessToken, expiresAt, clientId, clientSecret, tenantId } = clientSecretCredentials;
// Check if token is still valid (with 5 minute buffer)
if (accessToken && expiresAt && expiresAt > currentTime + 300000) {
return { accessToken };
}
const { data: clientData } = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", tenantId || "common"),
new URLSearchParams({
grant_type: "client_credentials",
scope: `https://vault.azure.net/.default`,
client_id: clientId,
client_secret: clientSecret
})
);
const updatedClientCredentials = {
...clientSecretCredentials,
accessToken: clientData.access_token,
expiresAt: currentTime + clientData.expires_in * 1000
};
const encryptedClientCredentials = await encryptAppConnectionCredentials({
credentials: updatedClientCredentials,
orgId: appConnection.orgId,
kmsService
});
await appConnectionDAL.updateById(appConnection.id, { encryptedCredentials: encryptedClientCredentials });
return { accessToken: clientData.access_token };
default:
throw new InternalServerError({
message: `Unhandled Azure Key Vault connection method: ${appConnection.method as AzureKeyVaultConnectionMethod}`
});
}
};
export const getAzureKeyVaultConnectionListItem = () => {
@@ -100,7 +144,10 @@ export const getAzureKeyVaultConnectionListItem = () => {
return {
name: "Azure Key Vault" as const,
app: AppConnection.AzureKeyVault as const,
methods: Object.values(AzureKeyVaultConnectionMethod) as [AzureKeyVaultConnectionMethod.OAuth],
methods: Object.values(AzureKeyVaultConnectionMethod) as [
AzureKeyVaultConnectionMethod.OAuth,
AzureKeyVaultConnectionMethod.ClientSecret
],
oauthClientId: INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID
};
};
@@ -111,68 +158,108 @@ export const validateAzureKeyVaultConnectionCredentials = async (config: TAzureK
const { INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID, INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET, SITE_URL } =
getConfig();
if (!INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID || !INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET) {
throw new InternalServerError({
message: `Azure ${getAppConnectionMethodName(method)} environment variables have not been configured`
});
}
let tokenResp: AxiosResponse<ExchangeCodeAzureResponse> | null = null;
let tokenError: AxiosError | null = null;
try {
tokenResp = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", inputCredentials.tenantId || "common"),
new URLSearchParams({
grant_type: "authorization_code",
code: inputCredentials.code,
scope: `openid offline_access https://vault.azure.net/.default`,
client_id: INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID,
client_secret: INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET,
redirect_uri: `${SITE_URL}/organization/app-connections/azure/oauth/callback`
})
);
} catch (e: unknown) {
if (e instanceof AxiosError) {
tokenError = e;
} else {
throw new BadRequestError({
message: `Unable to validate connection: verify credentials`
});
}
}
if (tokenError) {
if (tokenError instanceof AxiosError) {
throw new BadRequestError({
message: `Failed to get access token: ${
(tokenError?.response?.data as { error_description?: string })?.error_description || "Unknown error"
}`
});
} else {
throw new InternalServerError({
message: "Failed to get access token"
});
}
}
if (!tokenResp) {
throw new InternalServerError({
message: `Failed to get access token: Token was empty with no error`
});
}
switch (method) {
case AzureKeyVaultConnectionMethod.OAuth:
if (!INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID || !INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET) {
throw new InternalServerError({
message: `Azure ${getAppConnectionMethodName(method)} environment variables have not been configured`
});
}
let tokenResp: AxiosResponse<ExchangeCodeAzureResponse> | null = null;
let tokenError: AxiosError | null = null;
const oauthCredentials = inputCredentials as { code: string; tenantId?: string };
try {
tokenResp = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", oauthCredentials.tenantId || "common"),
new URLSearchParams({
grant_type: "authorization_code",
code: oauthCredentials.code,
scope: `openid offline_access https://vault.azure.net/.default`,
client_id: INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID,
client_secret: INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET,
redirect_uri: `${SITE_URL}/organization/app-connections/azure/oauth/callback`
})
);
} catch (e: unknown) {
if (e instanceof AxiosError) {
tokenError = e;
} else {
throw new BadRequestError({
message: `Unable to validate connection: verify credentials`
});
}
}
if (tokenError) {
if (tokenError instanceof AxiosError) {
throw new BadRequestError({
message: `Failed to get access token: ${
(tokenError?.response?.data as { error_description?: string })?.error_description || "Unknown error"
}`
});
} else {
throw new InternalServerError({
message: "Failed to get access token"
});
}
}
if (!tokenResp) {
throw new InternalServerError({
message: `Failed to get access token: Token was empty with no error`
});
}
return {
tenantId: inputCredentials.tenantId,
tenantId: oauthCredentials.tenantId,
accessToken: tokenResp.data.access_token,
refreshToken: tokenResp.data.refresh_token,
expiresAt: Date.now() + tokenResp.data.expires_in * 1000
};
case AzureKeyVaultConnectionMethod.ClientSecret:
const { tenantId, clientId, clientSecret } = inputCredentials as {
tenantId: string;
clientId: string;
clientSecret: string;
};
try {
const { data: clientData } = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", tenantId || "common"),
new URLSearchParams({
grant_type: "client_credentials",
scope: `https://vault.azure.net/.default`,
client_id: clientId,
client_secret: clientSecret
})
);
return {
tenantId,
accessToken: clientData.access_token,
expiresAt: Date.now() + clientData.expires_in * 1000,
clientId,
clientSecret
};
} catch (e: unknown) {
if (e instanceof AxiosError) {
throw new BadRequestError({
message: `Failed to get access token: ${
(e?.response?.data as { error_description?: string })?.error_description || "Unknown error"
}`
});
} else {
throw new InternalServerError({
message: "Failed to get access token"
});
}
}
default:
throw new InternalServerError({
message: `Unhandled Azure connection method: ${method as AzureKeyVaultConnectionMethod}`
message: `Unhandled Azure Key Vault connection method: ${method as AzureKeyVaultConnectionMethod}`
});
}
};
@@ -22,6 +22,29 @@ export const AzureKeyVaultConnectionOAuthOutputCredentialsSchema = z.object({
expiresAt: z.number()
});
export const AzureKeyVaultConnectionClientSecretInputCredentialsSchema = z.object({
clientId: z
.string()
.uuid()
.trim()
.min(1, "Client ID required")
.max(50, "Client ID must be at most 50 characters long"),
clientSecret: z
.string()
.trim()
.min(1, "Client Secret required")
.max(50, "Client Secret must be at most 50 characters long"),
tenantId: z.string().uuid().trim().min(1, "Tenant ID required")
});
export const AzureKeyVaultConnectionClientSecretOutputCredentialsSchema = z.object({
clientId: z.string(),
clientSecret: z.string(),
tenantId: z.string(),
accessToken: z.string(),
expiresAt: z.number()
});
export const ValidateAzureKeyVaultConnectionCredentialsSchema = z.discriminatedUnion("method", [
z.object({
method: z
@@ -30,6 +53,14 @@ export const ValidateAzureKeyVaultConnectionCredentialsSchema = z.discriminatedU
credentials: AzureKeyVaultConnectionOAuthInputCredentialsSchema.describe(
AppConnections.CREATE(AppConnection.AzureKeyVault).credentials
)
}),
z.object({
method: z
.literal(AzureKeyVaultConnectionMethod.ClientSecret)
.describe(AppConnections.CREATE(AppConnection.AzureKeyVault).method),
credentials: AzureKeyVaultConnectionClientSecretInputCredentialsSchema.describe(
AppConnections.CREATE(AppConnection.AzureKeyVault).credentials
)
})
]);
@@ -39,9 +70,13 @@ export const CreateAzureKeyVaultConnectionSchema = ValidateAzureKeyVaultConnecti
export const UpdateAzureKeyVaultConnectionSchema = z
.object({
credentials: AzureKeyVaultConnectionOAuthInputCredentialsSchema.optional().describe(
AppConnections.UPDATE(AppConnection.AzureKeyVault).credentials
)
credentials: z
.union([
AzureKeyVaultConnectionOAuthInputCredentialsSchema,
AzureKeyVaultConnectionClientSecretInputCredentialsSchema
])
.optional()
.describe(AppConnections.UPDATE(AppConnection.AzureKeyVault).credentials)
})
.and(GenericUpdateAppConnectionFieldsSchema(AppConnection.AzureKeyVault));
@@ -55,6 +90,10 @@ export const AzureKeyVaultConnectionSchema = z.intersection(
z.object({
method: z.literal(AzureKeyVaultConnectionMethod.OAuth),
credentials: AzureKeyVaultConnectionOAuthOutputCredentialsSchema
}),
z.object({
method: z.literal(AzureKeyVaultConnectionMethod.ClientSecret),
credentials: AzureKeyVaultConnectionClientSecretOutputCredentialsSchema
})
])
);
@@ -65,6 +104,13 @@ export const SanitizedAzureKeyVaultConnectionSchema = z.discriminatedUnion("meth
credentials: AzureKeyVaultConnectionOAuthOutputCredentialsSchema.pick({
tenantId: true
})
}),
BaseAzureKeyVaultConnectionSchema.extend({
method: z.literal(AzureKeyVaultConnectionMethod.ClientSecret),
credentials: AzureKeyVaultConnectionClientSecretOutputCredentialsSchema.pick({
clientId: true,
tenantId: true
})
})
]);
@@ -4,6 +4,7 @@ import { DiscriminativePick } from "@app/lib/types";
import { AppConnection } from "../app-connection-enums";
import {
AzureKeyVaultConnectionClientSecretOutputCredentialsSchema,
AzureKeyVaultConnectionOAuthOutputCredentialsSchema,
AzureKeyVaultConnectionSchema,
CreateAzureKeyVaultConnectionSchema,
@@ -36,3 +37,7 @@ export type ExchangeCodeAzureResponse = {
};
export type TAzureKeyVaultConnectionCredentials = z.infer<typeof AzureKeyVaultConnectionOAuthOutputCredentialsSchema>;
export type TAzureKeyVaultConnectionClientSecretCredentials = z.infer<
typeof AzureKeyVaultConnectionClientSecretOutputCredentialsSchema
>;
@@ -1,10 +1,16 @@
import { createAppAuth } from "@octokit/auth-app";
import { Octokit } from "@octokit/rest";
import { AxiosResponse } from "axios";
import { AxiosError, AxiosRequestConfig, AxiosResponse } from "axios";
import https from "https";
import RE2 from "re2";
import { verifyHostInputValidity } from "@app/ee/services/dynamic-secret/dynamic-secret-fns";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { getConfig } from "@app/lib/config/env";
import { request } from "@app/lib/config/request";
import { request as httpRequest } from "@app/lib/config/request";
import { BadRequestError, ForbiddenRequestError, InternalServerError } from "@app/lib/errors";
import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway";
import { logger } from "@app/lib/logger";
import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
import { getAppConnectionMethodName } from "@app/services/app-connection/app-connection-fns";
import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
@@ -24,123 +30,224 @@ export const getGitHubConnectionListItem = () => {
};
};
export const getGitHubClient = (appConnection: TGitHubConnection) => {
export const requestWithGitHubGateway = async <T>(
appConnection: { gatewayId?: string | null },
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
requestConfig: AxiosRequestConfig
): Promise<AxiosResponse<T>> => {
const { gatewayId } = appConnection;
// If gateway isn't set up, don't proxy request
if (!gatewayId) {
return httpRequest.request(requestConfig);
}
const url = new URL(requestConfig.url as string);
await blockLocalAndPrivateIpAddresses(url.toString());
const [targetHost] = await verifyHostInputValidity(url.host, true);
const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(gatewayId);
const [relayHost, relayPort] = relayDetails.relayAddress.split(":");
return withGatewayProxy(
async (proxyPort) => {
const httpsAgent = new https.Agent({
servername: targetHost
});
url.protocol = "https:";
url.host = `localhost:${proxyPort}`;
const finalRequestConfig: AxiosRequestConfig = {
...requestConfig,
url: url.toString(),
httpsAgent,
headers: {
...requestConfig.headers,
Host: targetHost
}
};
try {
return await httpRequest.request(finalRequestConfig);
} catch (error) {
const axiosError = error as AxiosError;
logger.error("Error during GitHub gateway request:", axiosError.message, axiosError.response?.data);
throw error;
}
},
{
protocol: GatewayProxyProtocol.Tcp,
targetHost,
targetPort: 443,
relayHost,
relayPort: Number(relayPort),
identityId: relayDetails.identityId,
orgId: relayDetails.orgId,
tlsOptions: {
ca: relayDetails.certChain,
cert: relayDetails.certificate,
key: relayDetails.privateKey.toString()
}
}
);
};
export const getGitHubAppAuthToken = async (appConnection: TGitHubConnection) => {
const appCfg = getConfig();
const { method, credentials } = appConnection;
let client: Octokit;
const appId = appCfg.INF_APP_CONNECTION_GITHUB_APP_ID;
const appPrivateKey = appCfg.INF_APP_CONNECTION_GITHUB_APP_PRIVATE_KEY;
switch (method) {
case GitHubConnectionMethod.App:
if (!appId || !appPrivateKey) {
throw new InternalServerError({
message: `GitHub ${getAppConnectionMethodName(method).replace("GitHub", "")} has not been configured`
});
}
client = new Octokit({
authStrategy: createAppAuth,
auth: {
appId,
privateKey: appPrivateKey,
installationId: credentials.installationId
}
});
break;
case GitHubConnectionMethod.OAuth:
client = new Octokit({
auth: credentials.accessToken
});
break;
default:
throw new InternalServerError({
message: `Unhandled GitHub connection method: ${method as GitHubConnectionMethod}`
});
if (!appId || !appPrivateKey) {
throw new InternalServerError({
message: `GitHub App keys are not configured.`
});
}
return client;
if (appConnection.method !== GitHubConnectionMethod.App) {
throw new InternalServerError({ message: "Cannot generate GitHub App token for non-app connection" });
}
const appAuth = createAppAuth({
appId,
privateKey: appPrivateKey,
installationId: appConnection.credentials.installationId
});
const { token } = await appAuth({ type: "installation" });
return token;
};
function extractNextPageUrl(linkHeader: string | undefined): string | null {
if (!linkHeader) return null;
const links = linkHeader.split(",");
const nextLink = links.find((link) => link.includes('rel="next"'));
if (!nextLink) return null;
const match = new RE2(/<([^>]+)>/).exec(nextLink);
return match ? match[1] : null;
}
export const makePaginatedGitHubRequest = async <T, R = T[]>(
appConnection: TGitHubConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
path: string,
dataMapper?: (data: R) => T[]
): Promise<T[]> => {
const { credentials, method } = appConnection;
const token =
method === GitHubConnectionMethod.OAuth ? credentials.accessToken : await getGitHubAppAuthToken(appConnection);
let url: string | null = `https://api.${credentials.host || "github.com"}${path}`;
let results: T[] = [];
let i = 0;
while (url && i < 1000) {
// eslint-disable-next-line no-await-in-loop
const response: AxiosResponse<R> = await requestWithGitHubGateway<R>(appConnection, gatewayService, {
url,
method: "GET",
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28"
}
});
const items = dataMapper ? dataMapper(response.data) : (response.data as unknown as T[]);
results = results.concat(items);
url = extractNextPageUrl(response.headers.link as string | undefined);
i += 1;
}
return results;
};
type GitHubOrganization = {
login: string;
id: number;
type: string;
};
type GitHubRepository = {
id: number;
name: string;
owner: GitHubOrganization;
permissions?: {
admin: boolean;
maintain: boolean;
push: boolean;
triage: boolean;
pull: boolean;
};
};
export const getGitHubRepositories = async (appConnection: TGitHubConnection) => {
const client = getGitHubClient(appConnection);
type GitHubEnvironment = {
id: number;
name: string;
};
let repositories: GitHubRepository[];
switch (appConnection.method) {
case GitHubConnectionMethod.App:
repositories = await client.paginate("GET /installation/repositories");
break;
case GitHubConnectionMethod.OAuth:
default:
repositories = (await client.paginate("GET /user/repos")).filter((repo) => repo.permissions?.admin);
break;
export const getGitHubRepositories = async (
appConnection: TGitHubConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
if (appConnection.method === GitHubConnectionMethod.App) {
return makePaginatedGitHubRequest<GitHubRepository, { repositories: GitHubRepository[] }>(
appConnection,
gatewayService,
"/installation/repositories",
(data) => data.repositories
);
}
return repositories;
const repos = await makePaginatedGitHubRequest<GitHubRepository>(appConnection, gatewayService, "/user/repos");
return repos.filter((repo) => repo.permissions?.admin);
};
export const getGitHubOrganizations = async (appConnection: TGitHubConnection) => {
const client = getGitHubClient(appConnection);
export const getGitHubOrganizations = async (
appConnection: TGitHubConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
if (appConnection.method === GitHubConnectionMethod.App) {
const installationRepositories = await makePaginatedGitHubRequest<
GitHubRepository,
{ repositories: GitHubRepository[] }
>(appConnection, gatewayService, "/installation/repositories", (data) => data.repositories);
let organizations: GitHubOrganization[];
switch (appConnection.method) {
case GitHubConnectionMethod.App: {
const installationRepositories = await client.paginate("GET /installation/repositories");
const organizationMap: Record<string, GitHubOrganization> = {};
installationRepositories.forEach((repo) => {
if (repo.owner.type === "Organization") {
organizationMap[repo.owner.id] = repo.owner;
}
});
organizations = Object.values(organizationMap);
break;
}
case GitHubConnectionMethod.OAuth:
default:
organizations = await client.paginate("GET /user/orgs");
break;
}
return organizations;
};
export const getGitHubEnvironments = async (appConnection: TGitHubConnection, owner: string, repo: string) => {
const client = getGitHubClient(appConnection);
try {
const environments = await client.paginate("GET /repos/{owner}/{repo}/environments", {
owner,
repo
const organizationMap: Record<string, GitHubOrganization> = {};
installationRepositories.forEach((repo) => {
if (repo.owner.type === "Organization") {
organizationMap[repo.owner.id] = repo.owner;
}
});
return environments;
} catch (e) {
// repo doesn't have envs
if ((e as { status: number }).status === 404) {
return [];
}
return Object.values(organizationMap);
}
throw e;
return makePaginatedGitHubRequest<GitHubOrganization>(appConnection, gatewayService, "/user/orgs");
};
export const getGitHubEnvironments = async (
appConnection: TGitHubConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
owner: string,
repo: string
) => {
try {
return await makePaginatedGitHubRequest<GitHubEnvironment, { environments: GitHubEnvironment[] }>(
appConnection,
gatewayService,
`/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/environments`,
(data) => data.environments
);
} catch (error) {
const axiosError = error as AxiosError;
if (axiosError.response?.status === 404) return [];
throw error;
}
};
@@ -159,9 +266,11 @@ export function isGithubErrorResponse(data: GithubTokenRespData): data is Github
return "error" in data;
}
export const validateGitHubConnectionCredentials = async (config: TGitHubConnectionConfig) => {
export const validateGitHubConnectionCredentials = async (
config: TGitHubConnectionConfig,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
const { credentials, method } = config;
const {
INF_APP_CONNECTION_GITHUB_OAUTH_CLIENT_ID,
INF_APP_CONNECTION_GITHUB_OAUTH_CLIENT_SECRET,
@@ -192,10 +301,13 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
}
let tokenResp: AxiosResponse<GithubTokenRespData>;
const host = credentials.host || "github.com";
try {
tokenResp = await request.get<GithubTokenRespData>("https://github.com/login/oauth/access_token", {
params: {
tokenResp = await requestWithGitHubGateway<GithubTokenRespData>(config, gatewayService, {
url: `https://${host}/login/oauth/access_token`,
method: "POST",
data: {
client_id: clientId,
client_secret: clientSecret,
code: credentials.code,
@@ -203,7 +315,7 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
},
headers: {
Accept: "application/json",
"Accept-Encoding": "application/json"
"Content-Type": "application/json"
}
});
@@ -233,7 +345,7 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
throw new InternalServerError({ message: `Missing access token: ${tokenResp.data.error}` });
}
const installationsResp = await request.get<{
const installationsResp = await requestWithGitHubGateway<{
installations: {
id: number;
account: {
@@ -242,7 +354,8 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
id: number;
};
}[];
}>(IntegrationUrls.GITHUB_USER_INSTALLATIONS, {
}>(config, gatewayService, {
url: IntegrationUrls.GITHUB_USER_INSTALLATIONS.replace("api.github.com", `api.${host}`),
headers: {
Accept: "application/json",
Authorization: `Bearer ${tokenResp.data.access_token}`,
@@ -11,20 +11,24 @@ import {
import { GitHubConnectionMethod } from "./github-connection-enums";
export const GitHubConnectionOAuthInputCredentialsSchema = z.object({
code: z.string().trim().min(1, "OAuth code required")
code: z.string().trim().min(1, "OAuth code required"),
host: z.string().trim().optional()
});
export const GitHubConnectionAppInputCredentialsSchema = z.object({
code: z.string().trim().min(1, "GitHub App code required"),
installationId: z.string().min(1, "GitHub App Installation ID required")
installationId: z.string().min(1, "GitHub App Installation ID required"),
host: z.string().trim().optional()
});
export const GitHubConnectionOAuthOutputCredentialsSchema = z.object({
accessToken: z.string()
accessToken: z.string(),
host: z.string().trim().optional()
});
export const GitHubConnectionAppOutputCredentialsSchema = z.object({
installationId: z.string()
installationId: z.string(),
host: z.string().trim().optional()
});
export const ValidateGitHubConnectionCredentialsSchema = z.discriminatedUnion("method", [
@@ -43,7 +47,9 @@ export const ValidateGitHubConnectionCredentialsSchema = z.discriminatedUnion("m
]);
export const CreateGitHubConnectionSchema = ValidateGitHubConnectionCredentialsSchema.and(
GenericCreateAppConnectionFieldsSchema(AppConnection.GitHub)
GenericCreateAppConnectionFieldsSchema(AppConnection.GitHub, {
supportsGateways: true
})
);
export const UpdateGitHubConnectionSchema = z
@@ -53,7 +59,11 @@ export const UpdateGitHubConnectionSchema = z
.optional()
.describe(AppConnections.UPDATE(AppConnection.GitHub).credentials)
})
.and(GenericUpdateAppConnectionFieldsSchema(AppConnection.GitHub));
.and(
GenericUpdateAppConnectionFieldsSchema(AppConnection.GitHub, {
supportsGateways: true
})
);
const BaseGitHubConnectionSchema = BaseAppConnectionSchema.extend({ app: z.literal(AppConnection.GitHub) });
@@ -1,3 +1,4 @@
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { OrgServiceActor } from "@app/lib/types";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import {
@@ -19,11 +20,14 @@ type TListGitHubEnvironmentsDTO = {
owner: string;
};
export const githubConnectionService = (getAppConnection: TGetAppConnectionFunc) => {
export const githubConnectionService = (
getAppConnection: TGetAppConnectionFunc,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
const listRepositories = async (connectionId: string, actor: OrgServiceActor) => {
const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);
const repositories = await getGitHubRepositories(appConnection);
const repositories = await getGitHubRepositories(appConnection, gatewayService);
return repositories;
};
@@ -31,7 +35,7 @@ export const githubConnectionService = (getAppConnection: TGetAppConnectionFunc)
const listOrganizations = async (connectionId: string, actor: OrgServiceActor) => {
const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);
const organizations = await getGitHubOrganizations(appConnection);
const organizations = await getGitHubOrganizations(appConnection, gatewayService);
return organizations;
};
@@ -42,7 +46,7 @@ export const githubConnectionService = (getAppConnection: TGetAppConnectionFunc)
) => {
const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);
const environments = await getGitHubEnvironments(appConnection, owner, repo);
const environments = await getGitHubEnvironments(appConnection, gatewayService, owner, repo);
return environments;
};
@@ -17,4 +17,7 @@ export type TGitHubConnectionInput = z.infer<typeof CreateGitHubConnectionSchema
export type TValidateGitHubConnectionCredentialsSchema = typeof ValidateGitHubConnectionCredentialsSchema;
export type TGitHubConnectionConfig = DiscriminativePick<TGitHubConnectionInput, "method" | "app" | "credentials">;
export type TGitHubConnectionConfig = DiscriminativePick<
TGitHubConnectionInput,
"method" | "app" | "credentials" | "gatewayId"
>;
@@ -645,7 +645,7 @@ export const projectServiceFactory = ({
const updateProject = async ({ actor, actorId, actorOrgId, actorAuthMethod, update, filter }: TUpdateProjectDTO) => {
const project = await projectDAL.findProjectByFilter(filter);
const { permission } = await permissionService.getProjectPermission({
const { permission, hasRole } = await permissionService.getProjectPermission({
actor,
actorId,
projectId: project.id,
@@ -667,6 +667,12 @@ export const projectServiceFactory = ({
}
}
if (update.secretDetectionIgnoreValues && !hasRole(ProjectMembershipRole.Admin)) {
throw new ForbiddenRequestError({
message: "Only admins can update secret detection ignore values"
});
}
const updatedProject = await projectDAL.updateById(project.id, {
name: update.name,
description: update.description,
@@ -676,7 +682,8 @@ export const projectServiceFactory = ({
slug: update.slug,
secretSharing: update.secretSharing,
defaultProduct: update.defaultProduct,
showSnapshotsLegacy: update.showSnapshotsLegacy
showSnapshotsLegacy: update.showSnapshotsLegacy,
secretDetectionIgnoreValues: update.secretDetectionIgnoreValues
});
return updatedProject;
@@ -96,6 +96,7 @@ export type TUpdateProjectDTO = {
slug?: string;
secretSharing?: boolean;
showSnapshotsLegacy?: boolean;
secretDetectionIgnoreValues?: string[];
};
} & Omit<TProjectPermission, "projectId">;
@@ -13,7 +13,7 @@ import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
import { TAzureAppConfigurationSyncWithCredentials } from "./azure-app-configuration-sync-types";
type TAzureAppConfigurationSyncFactoryDeps = {
appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "update">;
appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "updateById">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
};
@@ -12,7 +12,7 @@ import { SecretSyncError } from "../secret-sync-errors";
import { GetAzureKeyVaultSecret, TAzureKeyVaultSyncWithCredentials } from "./azure-key-vault-sync-types";
type TAzureKeyVaultSyncFactoryDeps = {
appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "update">;
appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "updateById">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
};
@@ -1,7 +1,12 @@
import { Octokit } from "@octokit/rest";
import sodium from "libsodium-wrappers";
import { getGitHubClient } from "@app/services/app-connection/github";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import {
getGitHubAppAuthToken,
GitHubConnectionMethod,
makePaginatedGitHubRequest,
requestWithGitHubGateway
} from "@app/services/app-connection/github";
import { GitHubSyncScope, GitHubSyncVisibility } from "@app/services/secret-sync/github/github-sync-enums";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import { matchesSchema } from "@app/services/secret-sync/secret-sync-fns";
@@ -12,155 +17,165 @@ import { TGitHubPublicKey, TGitHubSecret, TGitHubSecretPayload, TGitHubSyncWithC
// TODO: rate limit handling
const getEncryptedSecrets = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => {
let encryptedSecrets: TGitHubSecret[];
const { destinationConfig } = secretSync;
const getEncryptedSecrets = async (
secretSync: TGitHubSyncWithCredentials,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
const { destinationConfig, connection } = secretSync;
let path: string;
switch (destinationConfig.scope) {
case GitHubSyncScope.Organization: {
encryptedSecrets = await client.paginate("GET /orgs/{org}/actions/secrets", {
org: destinationConfig.org
});
path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets`;
break;
}
case GitHubSyncScope.Repository: {
encryptedSecrets = await client.paginate("GET /repos/{owner}/{repo}/actions/secrets", {
owner: destinationConfig.owner,
repo: destinationConfig.repo
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets`;
break;
}
case GitHubSyncScope.RepositoryEnvironment:
default: {
encryptedSecrets = await client.paginate("GET /repos/{owner}/{repo}/environments/{environment_name}/secrets", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
environment_name: destinationConfig.env
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets`;
break;
}
}
return encryptedSecrets;
return makePaginatedGitHubRequest<TGitHubSecret, { secrets: TGitHubSecret[] }>(
connection,
gatewayService,
path,
(data) => data.secrets
);
};
const getPublicKey = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => {
let publicKey: TGitHubPublicKey;
const { destinationConfig } = secretSync;
const getPublicKey = async (
secretSync: TGitHubSyncWithCredentials,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
token: string
) => {
const { destinationConfig, connection } = secretSync;
let path: string;
switch (destinationConfig.scope) {
case GitHubSyncScope.Organization: {
publicKey = (
await client.request("GET /orgs/{org}/actions/secrets/public-key", {
org: destinationConfig.org
})
).data;
path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets/public-key`;
break;
}
case GitHubSyncScope.Repository: {
publicKey = (
await client.request("GET /repos/{owner}/{repo}/actions/secrets/public-key", {
owner: destinationConfig.owner,
repo: destinationConfig.repo
})
).data;
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets/public-key`;
break;
}
case GitHubSyncScope.RepositoryEnvironment:
default: {
publicKey = (
await client.request("GET /repos/{owner}/{repo}/environments/{environment_name}/secrets/public-key", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
environment_name: destinationConfig.env
})
).data;
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets/public-key`;
break;
}
}
return publicKey;
const response = await requestWithGitHubGateway<TGitHubPublicKey>(connection, gatewayService, {
url: `https://api.${connection.credentials.host || "github.com"}${path}`,
method: "GET",
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28"
}
});
return response.data;
};
const deleteSecret = async (
client: Octokit,
secretSync: TGitHubSyncWithCredentials,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
token: string,
encryptedSecret: TGitHubSecret
) => {
const { destinationConfig } = secretSync;
const { destinationConfig, connection } = secretSync;
let path: string;
switch (destinationConfig.scope) {
case GitHubSyncScope.Organization: {
await client.request(`DELETE /orgs/{org}/actions/secrets/{secret_name}`, {
org: destinationConfig.org,
secret_name: encryptedSecret.name
});
path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets/${encodeURIComponent(encryptedSecret.name)}`;
break;
}
case GitHubSyncScope.Repository: {
await client.request("DELETE /repos/{owner}/{repo}/actions/secrets/{secret_name}", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
secret_name: encryptedSecret.name
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets/${encodeURIComponent(encryptedSecret.name)}`;
break;
}
case GitHubSyncScope.RepositoryEnvironment:
default: {
await client.request("DELETE /repos/{owner}/{repo}/environments/{environment_name}/secrets/{secret_name}", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
environment_name: destinationConfig.env,
secret_name: encryptedSecret.name
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets/${encodeURIComponent(encryptedSecret.name)}`;
break;
}
}
await requestWithGitHubGateway(connection, gatewayService, {
url: `https://api.${connection.credentials.host || "github.com"}${path}`,
method: "DELETE",
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28"
}
});
};
const putSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredentials, payload: TGitHubSecretPayload) => {
const { destinationConfig } = secretSync;
const putSecret = async (
secretSync: TGitHubSyncWithCredentials,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
token: string,
payload: TGitHubSecretPayload
) => {
const { destinationConfig, connection } = secretSync;
let path: string;
let body: Record<string, string | number[]> = payload;
switch (destinationConfig.scope) {
case GitHubSyncScope.Organization: {
const { visibility, selectedRepositoryIds } = destinationConfig;
await client.request(`PUT /orgs/{org}/actions/secrets/{secret_name}`, {
org: destinationConfig.org,
path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets/${encodeURIComponent(payload.secret_name)}`;
body = {
...payload,
visibility,
...(visibility === GitHubSyncVisibility.Selected && {
selected_repository_ids: selectedRepositoryIds
})
});
};
break;
}
case GitHubSyncScope.Repository: {
await client.request("PUT /repos/{owner}/{repo}/actions/secrets/{secret_name}", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
...payload
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets/${encodeURIComponent(payload.secret_name)}`;
break;
}
case GitHubSyncScope.RepositoryEnvironment:
default: {
await client.request("PUT /repos/{owner}/{repo}/environments/{environment_name}/secrets/{secret_name}", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
environment_name: destinationConfig.env,
...payload
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets/${encodeURIComponent(payload.secret_name)}`;
break;
}
}
await requestWithGitHubGateway(connection, gatewayService, {
url: `https://api.${connection.credentials.host || "github.com"}${path}`,
method: "PUT",
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28"
},
data: body
});
};
export const GithubSyncFns = {
syncSecrets: async (secretSync: TGitHubSyncWithCredentials, secretMap: TSecretMap) => {
syncSecrets: async (
secretSync: TGitHubSyncWithCredentials,
ogSecretMap: TSecretMap,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
const secretMap = Object.fromEntries(Object.entries(ogSecretMap).map(([i, v]) => [i.toUpperCase(), v]));
switch (secretSync.destinationConfig.scope) {
case GitHubSyncScope.Organization:
if (Object.values(secretMap).length > 1000) {
@@ -187,38 +202,40 @@ export const GithubSyncFns = {
);
}
const client = getGitHubClient(secretSync.connection);
const { connection } = secretSync;
const token =
connection.method === GitHubConnectionMethod.OAuth
? connection.credentials.accessToken
: await getGitHubAppAuthToken(connection);
const encryptedSecrets = await getEncryptedSecrets(client, secretSync);
const encryptedSecrets = await getEncryptedSecrets(secretSync, gatewayService);
const publicKey = await getPublicKey(secretSync, gatewayService, token);
const publicKey = await getPublicKey(client, secretSync);
await sodium.ready;
for await (const key of Object.keys(secretMap)) {
// convert secret & base64 key to Uint8Array.
const binaryKey = sodium.from_base64(publicKey.key, sodium.base64_variants.ORIGINAL);
const binarySecretValue = sodium.from_string(secretMap[key].value);
await sodium.ready.then(async () => {
for await (const key of Object.keys(secretMap)) {
// convert secret & base64 key to Uint8Array.
const binaryKey = sodium.from_base64(publicKey.key, sodium.base64_variants.ORIGINAL);
const binarySecretValue = sodium.from_string(secretMap[key].value);
// encrypt secret using libsodium
const encryptedBytes = sodium.crypto_box_seal(binarySecretValue, binaryKey);
// encrypt secret using libsodium
const encryptedBytes = sodium.crypto_box_seal(binarySecretValue, binaryKey);
// convert encrypted Uint8Array to base64
const encryptedSecretValue = sodium.to_base64(encryptedBytes, sodium.base64_variants.ORIGINAL);
// convert encrypted Uint8Array to base64
const encryptedSecretValue = sodium.to_base64(encryptedBytes, sodium.base64_variants.ORIGINAL);
try {
await putSecret(client, secretSync, {
secret_name: key,
encrypted_value: encryptedSecretValue,
key_id: publicKey.key_id
});
} catch (error) {
throw new SecretSyncError({
error,
secretKey: key
});
}
try {
await putSecret(secretSync, gatewayService, token, {
secret_name: key,
encrypted_value: encryptedSecretValue,
key_id: publicKey.key_id
});
} catch (error) {
throw new SecretSyncError({
error,
secretKey: key
});
}
});
}
if (secretSync.syncOptions.disableSecretDeletion) return;
@@ -228,21 +245,31 @@ export const GithubSyncFns = {
continue;
if (!(encryptedSecret.name in secretMap)) {
await deleteSecret(client, secretSync, encryptedSecret);
await deleteSecret(secretSync, gatewayService, token, encryptedSecret);
}
}
},
getSecrets: async (secretSync: TGitHubSyncWithCredentials) => {
throw new Error(`${SECRET_SYNC_NAME_MAP[secretSync.destination]} does not support importing secrets.`);
},
removeSecrets: async (secretSync: TGitHubSyncWithCredentials, secretMap: TSecretMap) => {
const client = getGitHubClient(secretSync.connection);
removeSecrets: async (
secretSync: TGitHubSyncWithCredentials,
ogSecretMap: TSecretMap,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
const secretMap = Object.fromEntries(Object.entries(ogSecretMap).map(([i, v]) => [i.toUpperCase(), v]));
const encryptedSecrets = await getEncryptedSecrets(client, secretSync);
const { connection } = secretSync;
const token =
connection.method === GitHubConnectionMethod.OAuth
? connection.credentials.accessToken
: await getGitHubAppAuthToken(connection);
const encryptedSecrets = await getEncryptedSecrets(secretSync, gatewayService);
for await (const encryptedSecret of encryptedSecrets) {
if (encryptedSecret.name in secretMap) {
await deleteSecret(client, secretSync, encryptedSecret);
await deleteSecret(secretSync, gatewayService, token, encryptedSecret);
}
}
}
@@ -1,6 +1,7 @@
import { AxiosError } from "axios";
import handlebars from "handlebars";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import { OCI_VAULT_SYNC_LIST_OPTION, OCIVaultSyncFns } from "@app/ee/services/secret-sync/oci-vault";
import { BadRequestError } from "@app/lib/errors";
@@ -97,6 +98,7 @@ export const listSecretSyncOptions = () => {
type TSyncSecretDeps = {
appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "update" | "updateById">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
};
// Add schema to secret keys
@@ -191,7 +193,7 @@ export const SecretSyncFns = {
syncSecrets: (
secretSync: TSecretSyncWithCredentials,
secretMap: TSecretMap,
{ kmsService, appConnectionDAL }: TSyncSecretDeps
{ kmsService, appConnectionDAL, gatewayService }: TSyncSecretDeps
): Promise<void> => {
const schemaSecretMap = addSchema(secretMap, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema);
@@ -201,7 +203,7 @@ export const SecretSyncFns = {
case SecretSync.AWSSecretsManager:
return AwsSecretsManagerSyncFns.syncSecrets(secretSync, schemaSecretMap);
case SecretSync.GitHub:
return GithubSyncFns.syncSecrets(secretSync, schemaSecretMap);
return GithubSyncFns.syncSecrets(secretSync, schemaSecretMap, gatewayService);
case SecretSync.GCPSecretManager:
return GcpSyncFns.syncSecrets(secretSync, schemaSecretMap);
case SecretSync.AzureKeyVault:
@@ -395,7 +397,7 @@ export const SecretSyncFns = {
removeSecrets: (
secretSync: TSecretSyncWithCredentials,
secretMap: TSecretMap,
{ kmsService, appConnectionDAL }: TSyncSecretDeps
{ kmsService, appConnectionDAL, gatewayService }: TSyncSecretDeps
): Promise<void> => {
const schemaSecretMap = addSchema(secretMap, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema);
@@ -405,7 +407,7 @@ export const SecretSyncFns = {
case SecretSync.AWSSecretsManager:
return AwsSecretsManagerSyncFns.removeSecrets(secretSync, schemaSecretMap);
case SecretSync.GitHub:
return GithubSyncFns.removeSecrets(secretSync, schemaSecretMap);
return GithubSyncFns.removeSecrets(secretSync, schemaSecretMap, gatewayService);
case SecretSync.GCPSecretManager:
return GcpSyncFns.removeSecrets(secretSync, schemaSecretMap);
case SecretSync.AzureKeyVault:
@@ -4,6 +4,7 @@ import { Job } from "bullmq";
import { ProjectMembershipRole, SecretType } from "@app/db/schemas";
import { EventType, TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-types";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
@@ -96,6 +97,7 @@ type TSecretSyncQueueFactoryDep = {
resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">;
folderCommitService: Pick<TFolderCommitServiceFactory, "createCommit">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
};
type SecretSyncActionJob = Job<
@@ -138,7 +140,8 @@ export const secretSyncQueueFactory = ({
secretVersionTagV2BridgeDAL,
resourceMetadataDAL,
folderCommitService,
licenseService
licenseService,
gatewayService
}: TSecretSyncQueueFactoryDep) => {
const appCfg = getConfig();
@@ -353,7 +356,8 @@ export const secretSyncQueueFactory = ({
const importedSecrets = await SecretSyncFns.getSecrets(secretSync, {
appConnectionDAL,
kmsService
kmsService,
gatewayService
});
if (!Object.keys(importedSecrets).length) return {};
@@ -481,7 +485,8 @@ export const secretSyncQueueFactory = ({
await SecretSyncFns.syncSecrets(secretSyncWithCredentials, secretMap, {
appConnectionDAL,
kmsService
kmsService,
gatewayService
});
isSynced = true;
@@ -730,7 +735,8 @@ export const secretSyncQueueFactory = ({
secretMap,
{
appConnectionDAL,
kmsService
kmsService,
gatewayService
}
);
@@ -25,6 +25,7 @@ import {
import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal";
import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal";
import { scanSecretPolicyViolations } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
import { TKeyStoreFactory } from "@app/keystore/keystore";
import { DatabaseErrorCode } from "@app/lib/error-codes";
@@ -38,6 +39,7 @@ import { ActorType } from "../auth/auth-type";
import { TCommitResourceChangeDTO, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service";
import { TKmsServiceFactory } from "../kms/kms-service";
import { KmsDataKey } from "../kms/kms-types";
import { TProjectDALFactory } from "../project/project-dal";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { TReminderServiceFactory } from "../reminder/reminder-types";
import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal";
@@ -88,6 +90,7 @@ import { TSecretVersionV2TagDALFactory } from "./secret-version-tag-dal";
type TSecretV2BridgeServiceFactoryDep = {
secretDAL: TSecretV2BridgeDALFactory;
projectDAL: Pick<TProjectDALFactory, "findById">;
secretVersionDAL: TSecretVersionV2DALFactory;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
secretVersionTagDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
@@ -126,6 +129,7 @@ export type TSecretV2BridgeServiceFactory = ReturnType<typeof secretV2BridgeServ
*/
export const secretV2BridgeServiceFactory = ({
secretDAL,
projectDAL,
projectEnvDAL,
secretTagDAL,
secretVersionDAL,
@@ -295,6 +299,19 @@ export const secretV2BridgeServiceFactory = ({
})
);
const project = await projectDAL.findById(projectId);
await scanSecretPolicyViolations(
projectId,
secretPath,
[
{
secretKey: inputSecret.secretName,
secretValue: inputSecret.secretValue
}
],
project.secretDetectionIgnoreValues || []
);
const { nestedReferences, localReferences } = getAllSecretReferences(inputSecret.secretValue);
const allSecretReferences = nestedReferences.concat(
localReferences.map((el) => ({ secretKey: el, secretPath, environment }))
@@ -506,6 +523,21 @@ export const secretV2BridgeServiceFactory = ({
const { secretName, secretValue } = inputSecret;
if (secretValue) {
const project = await projectDAL.findById(projectId);
await scanSecretPolicyViolations(
projectId,
secretPath,
[
{
secretKey: inputSecret.newSecretName || secretName,
secretValue
}
],
project.secretDetectionIgnoreValues || []
);
}
const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId
@@ -1585,6 +1617,9 @@ export const secretV2BridgeServiceFactory = ({
if (secrets.length)
throw new BadRequestError({ message: `Secret already exist: ${secrets.map((el) => el.key).join(",")}` });
const project = await projectDAL.findById(projectId);
await scanSecretPolicyViolations(projectId, secretPath, inputSecrets, project.secretDetectionIgnoreValues || []);
// get all tags
const sanitizedTagIds = inputSecrets.flatMap(({ tagIds = [] }) => tagIds);
const tags = sanitizedTagIds.length ? await secretTagDAL.findManyTagsById(projectId, sanitizedTagIds) : [];
@@ -1925,6 +1960,19 @@ export const secretV2BridgeServiceFactory = ({
});
await $validateSecretReferences(projectId, permission, secretReferences, tx);
const project = await projectDAL.findById(projectId);
await scanSecretPolicyViolations(
projectId,
secretPath,
secretsToUpdate
.filter((el) => el.secretValue)
.map((el) => ({
secretKey: el.newSecretName || el.secretKey,
secretValue: el.secretValue as string
})),
project.secretDetectionIgnoreValues || []
);
const bulkUpdatedSecrets = await fnSecretBulkUpdate({
folderId,
orgId: actorOrgId,