merge main

This commit is contained in:
Scott Wilson
2025-07-30 16:38:35 -07:00
369 changed files with 2581 additions and 27780 deletions
+28 -27
View File
@@ -7,7 +7,6 @@
"": {
"name": "backend",
"version": "1.0.0",
"hasInstallScript": true,
"license": "ISC",
"dependencies": {
"@aws-sdk/client-elasticache": "^3.637.0",
@@ -34,7 +33,7 @@
"@gitbeaker/rest": "^42.5.0",
"@google-cloud/kms": "^4.5.0",
"@infisical/quic": "^1.0.8",
"@node-saml/passport-saml": "^5.0.1",
"@node-saml/passport-saml": "^5.1.0",
"@octokit/auth-app": "^7.1.1",
"@octokit/core": "^5.2.1",
"@octokit/plugin-paginate-graphql": "^4.0.1",
@@ -9574,20 +9573,20 @@
}
},
"node_modules/@node-saml/node-saml": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/@node-saml/node-saml/-/node-saml-5.0.1.tgz",
"integrity": "sha512-YQzFPEC+CnsfO9AFYnwfYZKIzOLx3kITaC1HrjHVLTo6hxcQhc+LgHODOMvW4VCV95Gwrz1MshRUWCPzkDqmnA==",
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/@node-saml/node-saml/-/node-saml-5.1.0.tgz",
"integrity": "sha512-t3cJnZ4aC7HhPZ6MGylGZULvUtBOZ6FzuUndaHGXjmIZHXnLfC/7L8a57O9Q9V7AxJGKAiRM5zu2wNm9EsvQpw==",
"license": "MIT",
"dependencies": {
"@types/debug": "^4.1.12",
"@types/qs": "^6.9.11",
"@types/qs": "^6.9.18",
"@types/xml-encryption": "^1.2.4",
"@types/xml2js": "^0.4.14",
"@xmldom/is-dom-node": "^1.0.1",
"@xmldom/xmldom": "^0.8.10",
"debug": "^4.3.4",
"xml-crypto": "^6.0.1",
"xml-encryption": "^3.0.2",
"debug": "^4.4.0",
"xml-crypto": "^6.1.2",
"xml-encryption": "^3.1.0",
"xml2js": "^0.6.2",
"xmlbuilder": "^15.1.1",
"xpath": "^0.0.34"
@@ -9597,9 +9596,9 @@
}
},
"node_modules/@node-saml/node-saml/node_modules/debug": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
"integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
"version": "4.4.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz",
"integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==",
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
@@ -9636,14 +9635,14 @@
}
},
"node_modules/@node-saml/passport-saml": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/@node-saml/passport-saml/-/passport-saml-5.0.1.tgz",
"integrity": "sha512-fMztg3zfSnjLEgxvpl6HaDMNeh0xeQX4QHiF9e2Lsie2dc4qFE37XYbQZhVmn8XJ2awPpSWLQ736UskYgGU8lQ==",
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/@node-saml/passport-saml/-/passport-saml-5.1.0.tgz",
"integrity": "sha512-pBm+iFjv9eihcgeJuSUs4c0AuX1QEFdHwP8w1iaWCfDzXdeWZxUBU5HT2bY2S4dvNutcy+A9hYsH7ZLBGtgwDg==",
"license": "MIT",
"dependencies": {
"@node-saml/node-saml": "^5.0.1",
"@types/express": "^4.17.21",
"@types/passport": "^1.0.16",
"@node-saml/node-saml": "^5.1.0",
"@types/express": "^4.17.23",
"@types/passport": "^1.0.17",
"@types/passport-strategy": "^0.2.38",
"passport": "^0.7.0",
"passport-strategy": "^1.0.0"
@@ -13351,9 +13350,10 @@
"license": "MIT"
},
"node_modules/@types/express": {
"version": "4.17.21",
"resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz",
"integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==",
"version": "4.17.23",
"resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.23.tgz",
"integrity": "sha512-Crp6WY9aTYP3qPi2wGDo9iUe/rceX01UMhnF1jmwDcKCFM6cx7YhGP/Mpr3y9AASpfHixIG0E6azCcL5OcDHsQ==",
"license": "MIT",
"dependencies": {
"@types/body-parser": "*",
"@types/express-serve-static-core": "^4.17.33",
@@ -13523,9 +13523,10 @@
}
},
"node_modules/@types/passport": {
"version": "1.0.16",
"resolved": "https://registry.npmjs.org/@types/passport/-/passport-1.0.16.tgz",
"integrity": "sha512-FD0qD5hbPWQzaM0wHUnJ/T0BBCJBxCeemtnCwc/ThhTg3x9jfrAcRUmj5Dopza+MfFS9acTe3wk7rcVnRIp/0A==",
"version": "1.0.17",
"resolved": "https://registry.npmjs.org/@types/passport/-/passport-1.0.17.tgz",
"integrity": "sha512-aciLyx+wDwT2t2/kJGJR2AEeBz0nJU4WuRX04Wu9Dqc5lSUtwu0WERPHYsLhF9PtseiAMPBGNUOtFjxZ56prsg==",
"license": "MIT",
"dependencies": {
"@types/express": "*"
}
@@ -31953,9 +31954,9 @@
"license": "MIT"
},
"node_modules/xml-crypto": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/xml-crypto/-/xml-crypto-6.0.1.tgz",
"integrity": "sha512-v05aU7NS03z4jlZ0iZGRFeZsuKO1UfEbbYiaeRMiATBFs6Jq9+wqKquEMTn4UTrYZ9iGD8yz3KT4L9o2iF682w==",
"version": "6.1.2",
"resolved": "https://registry.npmjs.org/xml-crypto/-/xml-crypto-6.1.2.tgz",
"integrity": "sha512-leBOVQdVi8FvPJrMYoum7Ici9qyxfE4kVi+AkpUoYCSXaQF4IlBm1cneTK9oAxR61LpYxTx7lNcsnBIeRpGW2w==",
"license": "MIT",
"dependencies": {
"@xmldom/is-dom-node": "^1.0.1",
+1 -1
View File
@@ -153,7 +153,7 @@
"@gitbeaker/rest": "^42.5.0",
"@google-cloud/kms": "^4.5.0",
"@infisical/quic": "^1.0.8",
"@node-saml/passport-saml": "^5.0.1",
"@node-saml/passport-saml": "^5.1.0",
"@octokit/auth-app": "^7.1.1",
"@octokit/core": "^5.2.1",
"@octokit/plugin-paginate-graphql": "^4.0.1",
@@ -0,0 +1,19 @@
import { Knex } from "knex";
import { TableName } from "../schemas";
export async function up(knex: Knex): Promise<void> {
if (!(await knex.schema.hasColumn(TableName.Project, "secretDetectionIgnoreValues"))) {
await knex.schema.alterTable(TableName.Project, (t) => {
t.specificType("secretDetectionIgnoreValues", "text[]");
});
}
}
export async function down(knex: Knex): Promise<void> {
if (await knex.schema.hasColumn(TableName.Project, "secretDetectionIgnoreValues")) {
await knex.schema.alterTable(TableName.Project, (t) => {
t.dropColumn("secretDetectionIgnoreValues");
});
}
}
+2 -1
View File
@@ -30,7 +30,8 @@ export const ProjectsSchema = z.object({
hasDeleteProtection: z.boolean().default(false).nullable().optional(),
secretSharing: z.boolean().default(true),
showSnapshotsLegacy: z.boolean().default(false),
defaultProduct: z.string().nullable().optional()
defaultProduct: z.string().nullable().optional(),
secretDetectionIgnoreValues: z.string().array().nullable().optional()
});
export type TProjects = z.infer<typeof ProjectsSchema>;
@@ -5,13 +5,14 @@
// TODO(akhilmhdh): With tony find out the api structure and fill it here
import { ForbiddenError } from "@casl/ability";
import { AxiosError } from "axios";
import { CronJob } from "cron";
import { Knex } from "knex";
import { TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { verifyOfflineLicense } from "@app/lib/crypto";
import { NotFoundError } from "@app/lib/errors";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
import { TIdentityOrgDALFactory } from "@app/services/identity/identity-org-dal";
import { TOrgDALFactory } from "@app/services/org/org-dal";
@@ -603,10 +604,22 @@ export const licenseServiceFactory = ({
});
}
const { data } = await licenseServerCloudApi.request.delete(
`/api/license-server/v1/customers/${organization.customerId}/billing-details/payment-methods/${pmtMethodId}`
);
return data;
try {
const { data } = await licenseServerCloudApi.request.delete(
`/api/license-server/v1/customers/${organization.customerId}/billing-details/payment-methods/${pmtMethodId}`
);
return data;
} catch (error) {
if (error instanceof AxiosError) {
throw new BadRequestError({
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
message: `Failed to remove payment method: ${error.response?.data?.message}`
});
}
throw new BadRequestError({
message: "Unable to remove payment method"
});
}
};
const getOrgTaxIds = async ({ orgId, actor, actorId, actorAuthMethod, actorOrgId }: TGetOrgTaxIdDTO) => {
+4 -2
View File
@@ -579,6 +579,9 @@ export const scimServiceFactory = ({
});
const serverCfg = await getServerCfg();
const hasEmailChanged = email?.toLowerCase() !== membership.email;
const defaultEmailVerified =
org.orgAuthMethod === OrgAuthMethod.OIDC ? serverCfg.trustOidcEmails : serverCfg.trustSamlEmails;
await userDAL.transaction(async (tx) => {
await userAliasDAL.update(
{
@@ -605,8 +608,7 @@ export const scimServiceFactory = ({
firstName,
email: email?.toLowerCase(),
lastName,
isEmailVerified:
org.orgAuthMethod === OrgAuthMethod.OIDC ? serverCfg.trustOidcEmails : serverCfg.trustSamlEmails
isEmailVerified: hasEmailChanged ? defaultEmailVerified : undefined
},
tx
);
@@ -65,10 +65,14 @@ import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
import { TUserDALFactory } from "@app/services/user/user-dal";
import { TLicenseServiceFactory } from "../license/license-service";
import { throwIfMissingSecretReadValueOrDescribePermission } from "../permission/permission-fns";
import {
hasSecretReadValueOrDescribePermission,
throwIfMissingSecretReadValueOrDescribePermission
} from "../permission/permission-fns";
import { TPermissionServiceFactory } from "../permission/permission-service-types";
import { ProjectPermissionSecretActions, ProjectPermissionSub } from "../permission/project-permission";
import { TSecretApprovalPolicyDALFactory } from "../secret-approval-policy/secret-approval-policy-dal";
import { scanSecretPolicyViolations } from "../secret-scanning-v2/secret-scanning-v2-fns";
import { TSecretSnapshotServiceFactory } from "../secret-snapshot/secret-snapshot-service";
import { TSecretApprovalRequestDALFactory } from "./secret-approval-request-dal";
import { sendApprovalEmailsFn } from "./secret-approval-request-fns";
@@ -276,13 +280,19 @@ export const secretApprovalRequestServiceFactory = ({
) {
throw new ForbiddenRequestError({ message: "User has insufficient privileges" });
}
const hasSecretReadAccess = permission.can(
ProjectPermissionSecretActions.DescribeAndReadValue,
ProjectPermissionSub.Secrets
);
const getHasSecretReadAccess = (environment: string, tags: { slug: string }[], secretPath?: string) => {
const canRead = hasSecretReadValueOrDescribePermission(permission, ProjectPermissionSecretActions.ReadValue, {
environment,
secretPath: secretPath || "/",
secretTags: tags.map((i) => i.slug)
});
return canRead;
};
let secrets;
const secretPath = await folderDAL.findSecretPathByFolderIds(secretApprovalRequest.projectId, [
secretApprovalRequest.folderId
]);
if (shouldUseSecretV2Bridge) {
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
@@ -298,8 +308,8 @@ export const secretApprovalRequestServiceFactory = ({
version: el.version,
secretMetadata: el.secretMetadata as ResourceMetadataDTO,
isRotatedSecret: el.secret?.isRotatedSecret ?? false,
secretValueHidden: !hasSecretReadAccess,
secretValue: !hasSecretReadAccess
secretValueHidden: !getHasSecretReadAccess(secretApprovalRequest.environment, el.tags, secretPath?.[0]?.path),
secretValue: !getHasSecretReadAccess(secretApprovalRequest.environment, el.tags, secretPath?.[0]?.path)
? INFISICAL_SECRET_VALUE_HIDDEN_MASK
: el.secret && el.secret.isRotatedSecret
? undefined
@@ -314,8 +324,12 @@ export const secretApprovalRequestServiceFactory = ({
secretKey: el.secret.key,
id: el.secret.id,
version: el.secret.version,
secretValueHidden: !hasSecretReadAccess,
secretValue: !hasSecretReadAccess
secretValueHidden: !getHasSecretReadAccess(
secretApprovalRequest.environment,
el.tags,
secretPath?.[0]?.path
),
secretValue: !getHasSecretReadAccess(secretApprovalRequest.environment, el.tags, secretPath?.[0]?.path)
? INFISICAL_SECRET_VALUE_HIDDEN_MASK
: el.secret.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: el.secret.encryptedValue }).toString()
@@ -330,8 +344,12 @@ export const secretApprovalRequestServiceFactory = ({
secretKey: el.secretVersion.key,
id: el.secretVersion.id,
version: el.secretVersion.version,
secretValueHidden: !hasSecretReadAccess,
secretValue: !hasSecretReadAccess
secretValueHidden: !getHasSecretReadAccess(
secretApprovalRequest.environment,
el.tags,
secretPath?.[0]?.path
),
secretValue: !getHasSecretReadAccess(secretApprovalRequest.environment, el.tags, secretPath?.[0]?.path)
? INFISICAL_SECRET_VALUE_HIDDEN_MASK
: el.secretVersion.encryptedValue
? secretManagerDecryptor({ cipherTextBlob: el.secretVersion.encryptedValue }).toString()
@@ -349,7 +367,7 @@ export const secretApprovalRequestServiceFactory = ({
const encryptedSecrets = await secretApprovalRequestSecretDAL.findByRequestId(secretApprovalRequest.id);
secrets = encryptedSecrets.map((el) => ({
...el,
secretValueHidden: !hasSecretReadAccess,
secretValueHidden: !getHasSecretReadAccess(secretApprovalRequest.environment, el.tags, secretPath?.[0]?.path),
...decryptSecretWithBot(el, botKey),
secret: el.secret
? {
@@ -369,9 +387,6 @@ export const secretApprovalRequestServiceFactory = ({
: undefined
}));
}
const secretPath = await folderDAL.findSecretPathByFolderIds(secretApprovalRequest.projectId, [
secretApprovalRequest.folderId
]);
return { ...secretApprovalRequest, secretPath: secretPath?.[0]?.path || "/", commits: secrets };
};
@@ -1412,6 +1427,20 @@ export const secretApprovalRequestServiceFactory = ({
projectId
});
const project = await projectDAL.findById(projectId);
await scanSecretPolicyViolations(
projectId,
secretPath,
[
...(data[SecretOperations.Create] || []),
...(data[SecretOperations.Update] || []).filter((el) => el.secretValue)
].map((el) => ({
secretKey: el.secretKey,
secretValue: el.secretValue as string
})),
project.secretDetectionIgnoreValues || []
);
// for created secret approval change
const createdSecrets = data[SecretOperations.Create];
if (createdSecrets && createdSecrets?.length) {
@@ -21,6 +21,8 @@ const GRAPH_API_BASE = "https://graph.microsoft.com/v1.0";
type AzureErrorResponse = { error: { message: string } };
const EXPIRY_PADDING_IN_DAYS = 3;
const sleep = async () =>
new Promise((resolve) => {
setTimeout(resolve, 1000);
@@ -33,7 +35,8 @@ export const azureClientSecretRotationFactory: TRotationFactory<
const {
connection,
parameters: { objectId, clientId: clientIdParam },
secretsMapping
secretsMapping,
rotationInterval
} = secretRotation;
/**
@@ -50,7 +53,7 @@ export const azureClientSecretRotationFactory: TRotationFactory<
)}-${now.getFullYear()}`;
const endDateTime = new Date();
endDateTime.setFullYear(now.getFullYear() + 5);
endDateTime.setDate(now.getDate() + rotationInterval * 2 + EXPIRY_PADDING_IN_DAYS); // give 72 hour buffer
try {
const { data } = await request.post<AzureAddPasswordResponse>(
@@ -195,6 +198,12 @@ export const azureClientSecretRotationFactory: TRotationFactory<
callback
) => {
const credentials = await $rotateClientSecret();
// 2.5 years as expiry is set to x2 interval for the inactive period of credential
if (rotationInterval > Math.floor(365 * 2.5) - EXPIRY_PADDING_IN_DAYS) {
throw new BadRequestError({ message: "Azure does not support token duration over 5 years" });
}
return callback(credentials);
};
@@ -51,6 +51,7 @@ const baseSecretRotationV2Query = ({
db.ref("encryptedCredentials").withSchema(TableName.AppConnection).as("connectionEncryptedCredentials"),
db.ref("description").withSchema(TableName.AppConnection).as("connectionDescription"),
db.ref("version").withSchema(TableName.AppConnection).as("connectionVersion"),
db.ref("gatewayId").withSchema(TableName.AppConnection).as("connectionGatewayId"),
db.ref("createdAt").withSchema(TableName.AppConnection).as("connectionCreatedAt"),
db.ref("updatedAt").withSchema(TableName.AppConnection).as("connectionUpdatedAt"),
db
@@ -104,6 +105,7 @@ const expandSecretRotation = <T extends Awaited<ReturnType<typeof baseSecretRota
connectionCreatedAt,
connectionUpdatedAt,
connectionVersion,
connectionGatewayId,
connectionIsPlatformManagedCredentials,
...el
} = secretRotation;
@@ -123,6 +125,7 @@ const expandSecretRotation = <T extends Awaited<ReturnType<typeof baseSecretRota
createdAt: connectionCreatedAt,
updatedAt: connectionUpdatedAt,
version: connectionVersion,
gatewayId: connectionGatewayId,
isPlatformManagedCredentials: connectionIsPlatformManagedCredentials
},
folder: {
@@ -1,12 +1,22 @@
import { AxiosError } from "axios";
import { exec } from "child_process";
import { join } from "path";
import picomatch from "picomatch";
import RE2 from "re2";
import { readFindingsFile } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import {
createTempFolder,
deleteTempFolder,
readFindingsFile,
writeTextToFile
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
import { BITBUCKET_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION } from "@app/ee/services/secret-scanning-v2/bitbucket";
import { GITHUB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION } from "@app/ee/services/secret-scanning-v2/github";
import { GITLAB_SECRET_SCANNING_DATA_SOURCE_LIST_OPTION } from "@app/ee/services/secret-scanning-v2/gitlab";
import { getConfig } from "@app/lib/config/env";
import { crypto } from "@app/lib/crypto";
import { BadRequestError } from "@app/lib/errors";
import { titleCaseToCamelCase } from "@app/lib/fn";
import { SecretScanningDataSource, SecretScanningFindingSeverity } from "./secret-scanning-v2-enums";
@@ -48,6 +58,19 @@ export function scanDirectory(inputPath: string, outputPath: string, configPath?
});
}
export function scanFile(inputPath: string): Promise<void> {
return new Promise((resolve, reject) => {
const command = `infisical scan --exit-code=77 --source "${inputPath}" --no-git`;
exec(command, (error) => {
if (error && error.code === 77) {
reject(error);
} else {
resolve();
}
});
});
}
export const scanGitRepositoryAndGetFindings = async (
scanPath: string,
findingsPath: string,
@@ -142,3 +165,47 @@ export const parseScanErrorMessage = (err: unknown): string => {
? errorMessage
: `${errorMessage.substring(0, MAX_MESSAGE_LENGTH - 3)}...`;
};
export const scanSecretPolicyViolations = async (
projectId: string,
secretPath: string,
secrets: { secretKey: string; secretValue: string }[],
ignoreValues: string[]
) => {
const appCfg = getConfig();
if (!appCfg.PARAMS_FOLDER_SECRET_DETECTION_ENABLED) {
return;
}
const match = appCfg.PARAMS_FOLDER_SECRET_DETECTION_PATHS?.find(
(el) => el.projectId === projectId && picomatch.isMatch(secretPath, el.secretPath, { strictSlashes: false })
);
if (!match) {
return;
}
const tempFolder = await createTempFolder();
try {
const scanPromises = secrets
.filter((secret) => !ignoreValues.includes(secret.secretValue))
.map(async (secret) => {
const secretFilePath = join(tempFolder, `${crypto.nativeCrypto.randomUUID()}.txt`);
await writeTextToFile(secretFilePath, `${secret.secretKey}=${secret.secretValue}`);
try {
await scanFile(secretFilePath);
} catch (error) {
throw new BadRequestError({
message: `Secret value detected in ${secret.secretKey}. Please add this instead to the designated secrets path in the project.`,
name: "SecretPolicyViolation"
});
}
});
await Promise.all(scanPromises);
} finally {
await deleteTempFolder(tempFolder);
}
};
+13 -4
View File
@@ -704,7 +704,8 @@ export const PROJECTS = {
hasDeleteProtection: "Enable or disable delete protection for the project.",
secretSharing: "Enable or disable secret sharing for the project.",
showSnapshotsLegacy: "Enable or disable legacy snapshots for the project.",
defaultProduct: "The default product in which the project will open"
defaultProduct: "The default product in which the project will open",
secretDetectionIgnoreValues: "The list of secret values to ignore for secret detection."
},
GET_KEY: {
workspaceId: "The ID of the project to get the key from."
@@ -2252,7 +2253,9 @@ export const AppConnections = {
AZURE_DEVOPS: {
code: "The OAuth code to use to connect with Azure DevOps.",
tenantId: "The Tenant ID to use to connect with Azure DevOps.",
orgName: "The Organization name to use to connect with Azure DevOps."
orgName: "The Organization name to use to connect with Azure DevOps.",
clientId: "The Client ID to use to connect with Azure Client Secrets.",
clientSecret: "The Client Secret to use to connect with Azure Client Secrets."
},
OCI: {
userOcid: "The OCID (Oracle Cloud Identifier) of the user making the request.",
@@ -2399,12 +2402,18 @@ export const SecretSyncs = {
env: "The name of the GitHub environment."
},
AZURE_KEY_VAULT: {
vaultBaseUrl: "The base URL of the Azure Key Vault to sync secrets to. Example: https://example.vault.azure.net/"
vaultBaseUrl: "The base URL of the Azure Key Vault to sync secrets to. Example: https://example.vault.azure.net/",
tenantId: "The Tenant ID to use to connect with Azure Client Secrets.",
clientId: "The Client ID to use to connect with Azure Client Secrets.",
clientSecret: "The Client Secret to use to connect with Azure Client Secrets."
},
AZURE_APP_CONFIGURATION: {
configurationUrl:
"The URL of the Azure App Configuration to sync secrets to. Example: https://example.azconfig.io/",
label: "An optional label to assign to secrets created in Azure App Configuration."
label: "An optional label to assign to secrets created in Azure App Configuration.",
tenantId: "The Tenant ID to use to connect with Azure Client Secrets.",
clientId: "The Client ID to use to connect with Azure Client Secrets.",
clientSecret: "The Client Secret to use to connect with Azure Client Secrets."
},
AZURE_DEVOPS: {
devopsProjectId: "The ID of the Azure DevOps project to sync secrets to.",
+16 -4
View File
@@ -204,6 +204,17 @@ const envSchema = z
WORKFLOW_SLACK_CLIENT_SECRET: zpStr(z.string().optional()),
ENABLE_MSSQL_SECRET_ROTATION_ENCRYPT: zodStrBool.default("true"),
// Special Detection Feature
PARAMS_FOLDER_SECRET_DETECTION_PATHS: zpStr(
z
.string()
.optional()
.transform((val) => {
if (!val) return undefined;
return JSON.parse(val) as { secretPath: string; projectId: string }[];
})
),
// HSM
HSM_LIB_PATH: zpStr(z.string().optional()),
HSM_PIN: zpStr(z.string().optional()),
@@ -358,6 +369,7 @@ const envSchema = z
Boolean(data.HSM_LIB_PATH) && Boolean(data.HSM_PIN) && Boolean(data.HSM_KEY_LABEL) && data.HSM_SLOT !== undefined,
samlDefaultOrgSlug: data.DEFAULT_SAML_ORG_SLUG,
SECRET_SCANNING_ORG_WHITELIST: data.SECRET_SCANNING_ORG_WHITELIST?.split(","),
PARAMS_FOLDER_SECRET_DETECTION_ENABLED: (data.PARAMS_FOLDER_SECRET_DETECTION_PATHS?.length ?? 0) > 0,
INF_APP_CONNECTION_AZURE_DEVOPS_CLIENT_ID:
data.INF_APP_CONNECTION_AZURE_DEVOPS_CLIENT_ID || data.INF_APP_CONNECTION_AZURE_CLIENT_ID,
INF_APP_CONNECTION_AZURE_DEVOPS_CLIENT_SECRET:
@@ -484,7 +496,7 @@ export const overwriteSchema: {
]
},
azureAppConfiguration: {
name: "Azure App Configuration",
name: "Azure App Connection: App Configuration",
fields: [
{
key: "INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_ID",
@@ -497,7 +509,7 @@ export const overwriteSchema: {
]
},
azureKeyVault: {
name: "Azure Key Vault",
name: "Azure App Connection: Key Vault",
fields: [
{
key: "INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID",
@@ -510,7 +522,7 @@ export const overwriteSchema: {
]
},
azureClientSecrets: {
name: "Azure Client Secrets",
name: "Azure App Connection: Client Secrets",
fields: [
{
key: "INF_APP_CONNECTION_AZURE_CLIENT_SECRETS_CLIENT_ID",
@@ -523,7 +535,7 @@ export const overwriteSchema: {
]
},
azureDevOps: {
name: "Azure DevOps",
name: "Azure App Connection: DevOps",
fields: [
{
key: "INF_APP_CONNECTION_AZURE_DEVOPS_CLIENT_ID",
@@ -14,6 +14,11 @@ export const blockLocalAndPrivateIpAddresses = async (url: string) => {
if (appCfg.isDevelopmentMode) return;
const validUrl = new URL(url);
if (validUrl.username || validUrl.password) {
throw new BadRequestError({ message: "URLs with user credentials (e.g., user:pass@) are not allowed" });
}
const inputHostIps: string[] = [];
if (isIPv4(validUrl.hostname)) {
inputHostIps.push(validUrl.hostname);
+12 -10
View File
@@ -1044,6 +1044,15 @@ export const registerRoutes = async (
kmsService
});
const gatewayService = gatewayServiceFactory({
permissionService,
gatewayDAL,
kmsService,
licenseService,
orgGatewayConfigDAL,
keyStore
});
const secretSyncQueue = secretSyncQueueFactory({
queueService,
secretSyncDAL,
@@ -1067,7 +1076,8 @@ export const registerRoutes = async (
secretVersionTagV2BridgeDAL,
resourceMetadataDAL,
appConnectionDAL,
licenseService
licenseService,
gatewayService
});
const secretQueueService = secretQueueFactory({
@@ -1238,6 +1248,7 @@ export const registerRoutes = async (
const secretV2BridgeService = secretV2BridgeServiceFactory({
folderDAL,
projectDAL,
secretVersionDAL: secretVersionV2BridgeDAL,
folderCommitService,
secretQueueService,
@@ -1489,15 +1500,6 @@ export const registerRoutes = async (
licenseService
});
const gatewayService = gatewayServiceFactory({
permissionService,
gatewayDAL,
kmsService,
licenseService,
orgGatewayConfigDAL,
keyStore
});
const identityKubernetesAuthService = identityKubernetesAuthServiceFactory({
identityKubernetesAuthDAL,
identityOrgMembershipDAL,
@@ -271,7 +271,8 @@ export const SanitizedProjectSchema = ProjectsSchema.pick({
auditLogsRetentionDays: true,
hasDeleteProtection: true,
secretSharing: true,
showSnapshotsLegacy: true
showSnapshotsLegacy: true,
secretDetectionIgnoreValues: true
});
export const SanitizedTagSchema = SecretTagsSchema.pick({
+5 -2
View File
@@ -52,7 +52,8 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => {
defaultAuthOrgAuthEnforced: z.boolean().nullish(),
defaultAuthOrgAuthMethod: z.string().nullish(),
isSecretScanningDisabled: z.boolean(),
kubernetesAutoFetchServiceAccountToken: z.boolean()
kubernetesAutoFetchServiceAccountToken: z.boolean(),
paramsFolderSecretDetectionEnabled: z.boolean()
})
})
}
@@ -67,7 +68,8 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => {
fipsEnabled: crypto.isFipsModeEnabled(),
isMigrationModeOn: serverEnvs.MAINTENANCE_MODE,
isSecretScanningDisabled: serverEnvs.DISABLE_SECRET_SCANNING,
kubernetesAutoFetchServiceAccountToken: serverEnvs.KUBERNETES_AUTO_FETCH_SERVICE_ACCOUNT_TOKEN
kubernetesAutoFetchServiceAccountToken: serverEnvs.KUBERNETES_AUTO_FETCH_SERVICE_ACCOUNT_TOKEN,
paramsFolderSecretDetectionEnabled: serverEnvs.PARAMS_FOLDER_SECRET_DETECTION_ENABLED
}
};
}
@@ -685,6 +687,7 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => {
rateLimit: writeLimit
},
schema: {
hide: false,
body: z.object({
email: z.string().email().trim().min(1),
password: z.string().trim().min(1),
@@ -369,7 +369,11 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
.describe(PROJECTS.UPDATE.slug),
secretSharing: z.boolean().optional().describe(PROJECTS.UPDATE.secretSharing),
showSnapshotsLegacy: z.boolean().optional().describe(PROJECTS.UPDATE.showSnapshotsLegacy),
defaultProduct: z.nativeEnum(ProjectType).optional().describe(PROJECTS.UPDATE.defaultProduct)
defaultProduct: z.nativeEnum(ProjectType).optional().describe(PROJECTS.UPDATE.defaultProduct),
secretDetectionIgnoreValues: z
.array(z.string())
.optional()
.describe(PROJECTS.UPDATE.secretDetectionIgnoreValues)
}),
response: {
200: z.object({
@@ -392,7 +396,8 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
hasDeleteProtection: req.body.hasDeleteProtection,
slug: req.body.slug,
secretSharing: req.body.secretSharing,
showSnapshotsLegacy: req.body.showSnapshotsLegacy
showSnapshotsLegacy: req.body.showSnapshotsLegacy,
secretDetectionIgnoreValues: req.body.secretDetectionIgnoreValues
},
actorAuthMethod: req.permission.authMethod,
actorId: req.permission.id,
@@ -583,7 +583,7 @@ export const appConnectionServiceFactory = ({
deleteAppConnection,
connectAppConnectionById,
listAvailableAppConnectionsForUser,
github: githubConnectionService(connectAppConnectionById),
github: githubConnectionService(connectAppConnectionById, gatewayService),
githubRadar: githubRadarConnectionService(connectAppConnectionById),
gcp: gcpConnectionService(connectAppConnectionById),
databricks: databricksConnectionService(connectAppConnectionById, appConnectionDAL, kmsService),
@@ -1,3 +1,4 @@
export enum AzureAppConfigurationConnectionMethod {
OAuth = "oauth"
OAuth = "oauth",
ClientSecret = "client-secret"
}
@@ -1,3 +1,4 @@
/* eslint-disable no-case-declarations */
import { AxiosError, AxiosResponse } from "axios";
import { getConfig } from "@app/lib/config/env";
@@ -19,7 +20,10 @@ export const getAzureAppConfigurationConnectionListItem = () => {
return {
name: "Azure App Configuration" as const,
app: AppConnection.AzureAppConfiguration as const,
methods: Object.values(AzureAppConfigurationConnectionMethod) as [AzureAppConfigurationConnectionMethod.OAuth],
methods: Object.values(AzureAppConfigurationConnectionMethod) as [
AzureAppConfigurationConnectionMethod.OAuth,
AzureAppConfigurationConnectionMethod.ClientSecret
],
oauthClientId: INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_ID
};
};
@@ -35,71 +39,111 @@ export const validateAzureAppConfigurationConnectionCredentials = async (
SITE_URL
} = getConfig();
if (
!INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_ID ||
!INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_SECRET
) {
throw new InternalServerError({
message: `Azure ${getAppConnectionMethodName(method)} environment variables have not been configured`
});
}
let tokenResp: AxiosResponse<ExchangeCodeAzureResponse> | null = null;
let tokenError: AxiosError | null = null;
try {
tokenResp = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", inputCredentials.tenantId || "common"),
new URLSearchParams({
grant_type: "authorization_code",
code: inputCredentials.code,
scope: `openid offline_access https://azconfig.io/.default`,
client_id: INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_ID,
client_secret: INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_SECRET,
redirect_uri: `${SITE_URL}/organization/app-connections/azure/oauth/callback`
})
);
} catch (e: unknown) {
if (e instanceof AxiosError) {
tokenError = e;
} else {
throw new BadRequestError({
message: `Unable to validate connection: verify credentials`
});
}
}
if (tokenError) {
if (tokenError instanceof AxiosError) {
throw new BadRequestError({
message: `Failed to get access token: ${
(tokenError?.response?.data as { error_description?: string })?.error_description || "Unknown error"
}`
});
} else {
throw new InternalServerError({
message: "Failed to get access token"
});
}
}
if (!tokenResp) {
throw new InternalServerError({
message: `Failed to get access token: Token was empty with no error`
});
}
switch (method) {
case AzureAppConfigurationConnectionMethod.OAuth:
if (
!INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_ID ||
!INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_SECRET
) {
throw new InternalServerError({
message: `Azure ${getAppConnectionMethodName(method)} environment variables have not been configured`
});
}
let tokenResp: AxiosResponse<ExchangeCodeAzureResponse> | null = null;
let tokenError: AxiosError | null = null;
const oauthCredentials = inputCredentials as { code: string; tenantId?: string };
try {
tokenResp = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", oauthCredentials.tenantId || "common"),
new URLSearchParams({
grant_type: "authorization_code",
code: oauthCredentials.code,
scope: `openid offline_access https://azconfig.io/.default`,
client_id: INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_ID,
client_secret: INF_APP_CONNECTION_AZURE_APP_CONFIGURATION_CLIENT_SECRET,
redirect_uri: `${SITE_URL}/organization/app-connections/azure/oauth/callback`
})
);
} catch (e: unknown) {
if (e instanceof AxiosError) {
tokenError = e;
} else {
throw new BadRequestError({
message: `Unable to validate connection: verify credentials`
});
}
}
if (tokenError) {
if (tokenError instanceof AxiosError) {
throw new BadRequestError({
message: `Failed to get access token: ${
(tokenError?.response?.data as { error_description?: string })?.error_description || "Unknown error"
}`
});
} else {
throw new InternalServerError({
message: "Failed to get access token"
});
}
}
if (!tokenResp) {
throw new InternalServerError({
message: `Failed to get access token: Token was empty with no error`
});
}
return {
tenantId: inputCredentials.tenantId,
tenantId: oauthCredentials.tenantId,
accessToken: tokenResp.data.access_token,
refreshToken: tokenResp.data.refresh_token,
expiresAt: Date.now() + tokenResp.data.expires_in * 1000
};
case AzureAppConfigurationConnectionMethod.ClientSecret:
const { tenantId, clientId, clientSecret } = inputCredentials as {
tenantId: string;
clientId: string;
clientSecret: string;
};
try {
const { data: clientData } = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", tenantId || "common"),
new URLSearchParams({
grant_type: "client_credentials",
scope: `https://azconfig.io/.default`,
client_id: clientId,
client_secret: clientSecret
})
);
return {
tenantId,
accessToken: clientData.access_token,
expiresAt: Date.now() + clientData.expires_in * 1000,
clientId,
clientSecret
};
} catch (e: unknown) {
if (e instanceof AxiosError) {
throw new BadRequestError({
message: `Failed to get access token: ${
(e?.response?.data as { error_description?: string })?.error_description || "Unknown error"
}`
});
} else {
throw new InternalServerError({
message: "Failed to get access token"
});
}
}
default:
throw new InternalServerError({
message: `Unhandled Azure connection method: ${method as AzureAppConfigurationConnectionMethod}`
message: `Unhandled Azure App Configuration connection method: ${method as AzureAppConfigurationConnectionMethod}`
});
}
};
@@ -22,6 +22,29 @@ export const AzureAppConfigurationConnectionOAuthOutputCredentialsSchema = z.obj
expiresAt: z.number()
});
export const AzureAppConfigurationConnectionClientSecretInputCredentialsSchema = z.object({
clientId: z
.string()
.uuid()
.trim()
.min(1, "Client ID required")
.max(50, "Client ID must be at most 50 characters long"),
clientSecret: z
.string()
.trim()
.min(1, "Client Secret required")
.max(50, "Client Secret must be at most 50 characters long"),
tenantId: z.string().uuid().trim().min(1, "Tenant ID required")
});
export const AzureAppConfigurationConnectionClientSecretOutputCredentialsSchema = z.object({
clientId: z.string(),
clientSecret: z.string(),
tenantId: z.string(),
accessToken: z.string(),
expiresAt: z.number()
});
export const ValidateAzureAppConfigurationConnectionCredentialsSchema = z.discriminatedUnion("method", [
z.object({
method: z
@@ -30,6 +53,14 @@ export const ValidateAzureAppConfigurationConnectionCredentialsSchema = z.discri
credentials: AzureAppConfigurationConnectionOAuthInputCredentialsSchema.describe(
AppConnections.CREATE(AppConnection.AzureAppConfiguration).credentials
)
}),
z.object({
method: z
.literal(AzureAppConfigurationConnectionMethod.ClientSecret)
.describe(AppConnections.CREATE(AppConnection.AzureAppConfiguration).method),
credentials: AzureAppConfigurationConnectionClientSecretInputCredentialsSchema.describe(
AppConnections.CREATE(AppConnection.AzureAppConfiguration).credentials
)
})
]);
@@ -39,9 +70,13 @@ export const CreateAzureAppConfigurationConnectionSchema = ValidateAzureAppConfi
export const UpdateAzureAppConfigurationConnectionSchema = z
.object({
credentials: AzureAppConfigurationConnectionOAuthInputCredentialsSchema.optional().describe(
AppConnections.UPDATE(AppConnection.AzureAppConfiguration).credentials
)
credentials: z
.union([
AzureAppConfigurationConnectionOAuthInputCredentialsSchema,
AzureAppConfigurationConnectionClientSecretInputCredentialsSchema
])
.optional()
.describe(AppConnections.UPDATE(AppConnection.AzureAppConfiguration).credentials)
})
.and(GenericUpdateAppConnectionFieldsSchema(AppConnection.AzureAppConfiguration));
@@ -55,6 +90,10 @@ export const AzureAppConfigurationConnectionSchema = z.intersection(
z.object({
method: z.literal(AzureAppConfigurationConnectionMethod.OAuth),
credentials: AzureAppConfigurationConnectionOAuthOutputCredentialsSchema
}),
z.object({
method: z.literal(AzureAppConfigurationConnectionMethod.ClientSecret),
credentials: AzureAppConfigurationConnectionClientSecretOutputCredentialsSchema
})
])
);
@@ -65,6 +104,13 @@ export const SanitizedAzureAppConfigurationConnectionSchema = z.discriminatedUni
credentials: AzureAppConfigurationConnectionOAuthOutputCredentialsSchema.pick({
tenantId: true
})
}),
BaseAzureAppConfigurationConnectionSchema.extend({
method: z.literal(AzureAppConfigurationConnectionMethod.ClientSecret),
credentials: AzureAppConfigurationConnectionClientSecretOutputCredentialsSchema.pick({
clientId: true,
tenantId: true
})
})
]);
@@ -4,6 +4,7 @@ import { DiscriminativePick } from "@app/lib/types";
import { AppConnection } from "../app-connection-enums";
import {
AzureAppConfigurationConnectionClientSecretOutputCredentialsSchema,
AzureAppConfigurationConnectionOAuthOutputCredentialsSchema,
AzureAppConfigurationConnectionSchema,
CreateAzureAppConfigurationConnectionSchema,
@@ -39,3 +40,7 @@ export type ExchangeCodeAzureResponse = {
export type TAzureAppConfigurationConnectionCredentials = z.infer<
typeof AzureAppConfigurationConnectionOAuthOutputCredentialsSchema
>;
export type TAzureAppConfigurationConnectionClientSecretCredentials = z.infer<
typeof AzureAppConfigurationConnectionClientSecretOutputCredentialsSchema
>;
@@ -1,4 +1,5 @@
export enum AzureDevOpsConnectionMethod {
OAuth = "oauth",
AccessToken = "access-token"
AccessToken = "access-token",
ClientSecret = "client-secret"
}
@@ -18,6 +18,7 @@ import { AppConnection } from "../app-connection-enums";
import { AzureDevOpsConnectionMethod } from "./azure-devops-enums";
import {
ExchangeCodeAzureResponse,
TAzureDevOpsConnectionClientSecretCredentials,
TAzureDevOpsConnectionConfig,
TAzureDevOpsConnectionCredentials
} from "./azure-devops-types";
@@ -30,7 +31,8 @@ export const getAzureDevopsConnectionListItem = () => {
app: AppConnection.AzureDevOps as const,
methods: Object.values(AzureDevOpsConnectionMethod) as [
AzureDevOpsConnectionMethod.OAuth,
AzureDevOpsConnectionMethod.AccessToken
AzureDevOpsConnectionMethod.AccessToken,
AzureDevOpsConnectionMethod.ClientSecret
],
oauthClientId: INF_APP_CONNECTION_AZURE_DEVOPS_CLIENT_ID
};
@@ -53,11 +55,7 @@ export const getAzureDevopsConnection = async (
});
}
const credentials = (await decryptAppConnectionCredentials({
orgId: appConnection.orgId,
kmsService,
encryptedCredentials: appConnection.encryptedCredentials
})) as TAzureDevOpsConnectionCredentials;
const currentTime = Date.now();
// Handle different connection methods
switch (appConnection.method) {
@@ -69,12 +67,17 @@ export const getAzureDevopsConnection = async (
});
}
if (!("refreshToken" in credentials)) {
const oauthCredentials = (await decryptAppConnectionCredentials({
orgId: appConnection.orgId,
kmsService,
encryptedCredentials: appConnection.encryptedCredentials
})) as TAzureDevOpsConnectionCredentials;
if (!("refreshToken" in oauthCredentials)) {
throw new BadRequestError({ message: "Invalid OAuth credentials" });
}
const { refreshToken, tenantId } = credentials;
const currentTime = Date.now();
const { refreshToken, tenantId } = oauthCredentials;
const { data } = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", tenantId || "common"),
@@ -87,29 +90,75 @@ export const getAzureDevopsConnection = async (
})
);
const updatedCredentials = {
...credentials,
const updatedOAuthCredentials = {
...oauthCredentials,
accessToken: data.access_token,
expiresAt: currentTime + data.expires_in * 1000,
refreshToken: data.refresh_token
};
const encryptedCredentials = await encryptAppConnectionCredentials({
credentials: updatedCredentials,
const encryptedOAuthCredentials = await encryptAppConnectionCredentials({
credentials: updatedOAuthCredentials,
orgId: appConnection.orgId,
kmsService
});
await appConnectionDAL.updateById(appConnection.id, { encryptedCredentials });
await appConnectionDAL.updateById(appConnection.id, { encryptedCredentials: encryptedOAuthCredentials });
return data.access_token;
case AzureDevOpsConnectionMethod.AccessToken:
if (!("accessToken" in credentials)) {
const accessTokenCredentials = (await decryptAppConnectionCredentials({
orgId: appConnection.orgId,
kmsService,
encryptedCredentials: appConnection.encryptedCredentials
})) as { accessToken: string };
if (!("accessToken" in accessTokenCredentials)) {
throw new BadRequestError({ message: "Invalid API token credentials" });
}
// For access token, return the basic auth token directly
return credentials.accessToken;
return accessTokenCredentials.accessToken;
case AzureDevOpsConnectionMethod.ClientSecret:
const clientSecretCredentials = (await decryptAppConnectionCredentials({
orgId: appConnection.orgId,
kmsService,
encryptedCredentials: appConnection.encryptedCredentials
})) as TAzureDevOpsConnectionClientSecretCredentials;
const { accessToken, expiresAt, clientId, clientSecret, tenantId: clientTenantId } = clientSecretCredentials;
// Check if token is still valid (with 5 minute buffer)
if (accessToken && expiresAt && expiresAt > currentTime + 300000) {
return accessToken;
}
const { data: clientData } = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", clientTenantId || "common"),
new URLSearchParams({
grant_type: "client_credentials",
scope: `https://app.vssps.visualstudio.com/.default`,
client_id: clientId,
client_secret: clientSecret
})
);
const updatedClientCredentials = {
...clientSecretCredentials,
accessToken: clientData.access_token,
expiresAt: currentTime + clientData.expires_in * 1000
};
const encryptedClientCredentials = await encryptAppConnectionCredentials({
credentials: updatedClientCredentials,
orgId: appConnection.orgId,
kmsService
});
await appConnectionDAL.updateById(appConnection.id, { encryptedCredentials: encryptedClientCredentials });
return clientData.access_token;
default:
throw new BadRequestError({ message: `Unsupported connection method` });
@@ -138,7 +187,7 @@ export const validateAzureDevOpsConnectionCredentials = async (config: TAzureDev
let tokenError: AxiosError | null = null;
try {
const oauthCredentials = inputCredentials as { code: string; tenantId: string };
const oauthCredentials = inputCredentials as { code: string; tenantId: string; orgName: string };
tokenResp = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", oauthCredentials.tenantId || "common"),
new URLSearchParams({
@@ -262,9 +311,67 @@ export const validateAzureDevOpsConnectionCredentials = async (config: TAzureDev
});
}
case AzureDevOpsConnectionMethod.ClientSecret:
const { tenantId, clientId, clientSecret, orgName } = inputCredentials as {
tenantId: string;
clientId: string;
clientSecret: string;
orgName: string;
};
try {
// First, get the access token using client credentials flow
const { data: clientData } = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", tenantId || "common"),
new URLSearchParams({
grant_type: "client_credentials",
scope: `https://app.vssps.visualstudio.com/.default`,
client_id: clientId,
client_secret: clientSecret
})
);
// Validate access to the specific organization
const response = await request.get(
`${IntegrationUrls.AZURE_DEVOPS_API_URL}/${encodeURIComponent(orgName)}/_apis/projects?api-version=7.2-preview.2&$top=1`,
{
headers: {
Authorization: `Bearer ${clientData.access_token}`
}
}
);
if (response.status !== 200) {
throw new BadRequestError({
message: `Failed to validate connection to organization '${orgName}': ${response.status}`
});
}
return {
tenantId,
clientId,
clientSecret,
orgName,
accessToken: clientData.access_token,
expiresAt: Date.now() + clientData.expires_in * 1000
};
} catch (e: unknown) {
if (e instanceof AxiosError) {
throw new BadRequestError({
message: `Failed to authenticate with Azure DevOps using client credentials: ${
(e?.response?.data as { error_description?: string })?.error_description || e.message
}`
});
} else {
throw new InternalServerError({
message: "Failed to validate Azure DevOps client credentials"
});
}
}
default:
throw new InternalServerError({
message: `Unhandled Azure connection method: ${method as AzureDevOpsConnectionMethod}`
message: `Unhandled Azure DevOps connection method: ${method as AzureDevOpsConnectionMethod}`
});
}
};
@@ -38,6 +38,42 @@ export const AzureDevOpsConnectionAccessTokenOutputCredentialsSchema = z.object(
orgName: z.string()
});
export const AzureDevOpsConnectionClientSecretInputCredentialsSchema = z.object({
clientId: z
.string()
.uuid()
.trim()
.min(1, "Client ID required")
.max(50, "Client ID must be at most 50 characters long")
.describe(AppConnections.CREDENTIALS.AZURE_DEVOPS.clientId),
clientSecret: z
.string()
.trim()
.min(1, "Client Secret required")
.max(50, "Client Secret must be at most 50 characters long")
.describe(AppConnections.CREDENTIALS.AZURE_DEVOPS.clientSecret),
tenantId: z
.string()
.uuid()
.trim()
.min(1, "Tenant ID required")
.describe(AppConnections.CREDENTIALS.AZURE_DEVOPS.tenantId),
orgName: z
.string()
.trim()
.min(1, "Organization name required")
.describe(AppConnections.CREDENTIALS.AZURE_DEVOPS.orgName)
});
export const AzureDevOpsConnectionClientSecretOutputCredentialsSchema = z.object({
clientId: z.string(),
clientSecret: z.string(),
tenantId: z.string(),
orgName: z.string(),
accessToken: z.string(),
expiresAt: z.number()
});
export const ValidateAzureDevOpsConnectionCredentialsSchema = z.discriminatedUnion("method", [
z.object({
method: z
@@ -54,6 +90,14 @@ export const ValidateAzureDevOpsConnectionCredentialsSchema = z.discriminatedUni
credentials: AzureDevOpsConnectionAccessTokenInputCredentialsSchema.describe(
AppConnections.CREATE(AppConnection.AzureDevOps).credentials
)
}),
z.object({
method: z
.literal(AzureDevOpsConnectionMethod.ClientSecret)
.describe(AppConnections.CREATE(AppConnection.AzureDevOps).method),
credentials: AzureDevOpsConnectionClientSecretInputCredentialsSchema.describe(
AppConnections.CREATE(AppConnection.AzureDevOps).credentials
)
})
]);
@@ -64,7 +108,11 @@ export const CreateAzureDevOpsConnectionSchema = ValidateAzureDevOpsConnectionCr
export const UpdateAzureDevOpsConnectionSchema = z
.object({
credentials: z
.union([AzureDevOpsConnectionOAuthInputCredentialsSchema, AzureDevOpsConnectionAccessTokenInputCredentialsSchema])
.union([
AzureDevOpsConnectionOAuthInputCredentialsSchema,
AzureDevOpsConnectionAccessTokenInputCredentialsSchema,
AzureDevOpsConnectionClientSecretInputCredentialsSchema
])
.optional()
.describe(AppConnections.UPDATE(AppConnection.AzureDevOps).credentials)
})
@@ -84,6 +132,10 @@ export const AzureDevOpsConnectionSchema = z.intersection(
z.object({
method: z.literal(AzureDevOpsConnectionMethod.AccessToken),
credentials: AzureDevOpsConnectionAccessTokenOutputCredentialsSchema
}),
z.object({
method: z.literal(AzureDevOpsConnectionMethod.ClientSecret),
credentials: AzureDevOpsConnectionClientSecretOutputCredentialsSchema
})
])
);
@@ -101,6 +153,14 @@ export const SanitizedAzureDevOpsConnectionSchema = z.discriminatedUnion("method
credentials: AzureDevOpsConnectionAccessTokenOutputCredentialsSchema.pick({
orgName: true
})
}),
BaseAzureDevOpsConnectionSchema.extend({
method: z.literal(AzureDevOpsConnectionMethod.ClientSecret),
credentials: AzureDevOpsConnectionClientSecretOutputCredentialsSchema.pick({
clientId: true,
tenantId: true,
orgName: true
})
})
]);
@@ -52,6 +52,11 @@ const getAuthHeaders = (appConnection: TAzureDevOpsConnection, accessToken: stri
Authorization: `Basic ${basicAuthToken}`,
Accept: "application/json"
};
case AzureDevOpsConnectionMethod.ClientSecret:
return {
Authorization: `Bearer ${accessToken}`,
Accept: "application/json"
};
default:
throw new BadRequestError({ message: "Unsupported connection method" });
}
@@ -4,6 +4,7 @@ import { DiscriminativePick } from "@app/lib/types";
import { AppConnection } from "../app-connection-enums";
import {
AzureDevOpsConnectionClientSecretOutputCredentialsSchema,
AzureDevOpsConnectionOAuthOutputCredentialsSchema,
AzureDevOpsConnectionSchema,
CreateAzureDevOpsConnectionSchema,
@@ -27,6 +28,10 @@ export type TAzureDevOpsConnectionConfig = DiscriminativePick<
export type TAzureDevOpsConnectionCredentials = z.infer<typeof AzureDevOpsConnectionOAuthOutputCredentialsSchema>;
export type TAzureDevOpsConnectionClientSecretCredentials = z.infer<
typeof AzureDevOpsConnectionClientSecretOutputCredentialsSchema
>;
export interface ExchangeCodeAzureResponse {
token_type: string;
scope: string;
@@ -1,3 +1,4 @@
export enum AzureKeyVaultConnectionMethod {
OAuth = "oauth"
OAuth = "oauth",
ClientSecret = "client-secret"
}
@@ -1,3 +1,4 @@
/* eslint-disable no-case-declarations */
import { AxiosError, AxiosResponse } from "axios";
import { getConfig } from "@app/lib/config/env";
@@ -16,25 +17,16 @@ import { AppConnection } from "../app-connection-enums";
import { AzureKeyVaultConnectionMethod } from "./azure-key-vault-connection-enums";
import {
ExchangeCodeAzureResponse,
TAzureKeyVaultConnectionClientSecretCredentials,
TAzureKeyVaultConnectionConfig,
TAzureKeyVaultConnectionCredentials
} from "./azure-key-vault-connection-types";
export const getAzureConnectionAccessToken = async (
connectionId: string,
appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "update">,
appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "updateById">,
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">
) => {
const appCfg = getConfig();
if (
!appCfg.INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID ||
!appCfg.INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET
) {
throw new BadRequestError({
message: `Azure environment variables have not been configured`
});
}
const appConnection = await appConnectionDAL.findById(connectionId);
if (!appConnection) {
@@ -49,49 +41,101 @@ export const getAzureConnectionAccessToken = async (
throw new BadRequestError({ message: `Connection with ID '${connectionId}' is not a valid Azure connection` });
}
const credentials = (await decryptAppConnectionCredentials({
orgId: appConnection.orgId,
kmsService,
encryptedCredentials: appConnection.encryptedCredentials
})) as TAzureKeyVaultConnectionCredentials;
const currentTime = Date.now();
const { data } = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", credentials.tenantId || "common"),
new URLSearchParams({
grant_type: "refresh_token",
scope: `openid offline_access`,
client_id: appCfg.INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID,
client_secret: appCfg.INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET,
refresh_token: credentials.refreshToken
})
);
switch (appConnection.method) {
case AzureKeyVaultConnectionMethod.OAuth:
const appCfg = getConfig();
if (
!appCfg.INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID ||
!appCfg.INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET
) {
throw new BadRequestError({
message: `Azure environment variables have not been configured`
});
}
const accessExpiresAt = new Date();
accessExpiresAt.setSeconds(accessExpiresAt.getSeconds() + data.expires_in);
const oauthCredentials = (await decryptAppConnectionCredentials({
orgId: appConnection.orgId,
kmsService,
encryptedCredentials: appConnection.encryptedCredentials
})) as TAzureKeyVaultConnectionCredentials;
const updatedCredentials = {
...credentials,
accessToken: data.access_token,
expiresAt: accessExpiresAt.getTime(),
refreshToken: data.refresh_token
};
const { data } = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", oauthCredentials.tenantId || "common"),
new URLSearchParams({
grant_type: "refresh_token",
scope: `openid offline_access https://vault.azure.net/.default`,
client_id: appCfg.INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID,
client_secret: appCfg.INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET,
refresh_token: oauthCredentials.refreshToken
})
);
const encryptedCredentials = await encryptAppConnectionCredentials({
credentials: updatedCredentials,
orgId: appConnection.orgId,
kmsService
});
const updatedOAuthCredentials = {
...oauthCredentials,
accessToken: data.access_token,
expiresAt: currentTime + data.expires_in * 1000,
refreshToken: data.refresh_token
};
await appConnectionDAL.update(
{ id: connectionId },
{
encryptedCredentials
}
);
const encryptedOAuthCredentials = await encryptAppConnectionCredentials({
credentials: updatedOAuthCredentials,
orgId: appConnection.orgId,
kmsService
});
return {
accessToken: data.access_token
};
await appConnectionDAL.updateById(appConnection.id, { encryptedCredentials: encryptedOAuthCredentials });
return {
accessToken: data.access_token
};
case AzureKeyVaultConnectionMethod.ClientSecret:
const clientSecretCredentials = (await decryptAppConnectionCredentials({
orgId: appConnection.orgId,
kmsService,
encryptedCredentials: appConnection.encryptedCredentials
})) as TAzureKeyVaultConnectionClientSecretCredentials;
const { accessToken, expiresAt, clientId, clientSecret, tenantId } = clientSecretCredentials;
// Check if token is still valid (with 5 minute buffer)
if (accessToken && expiresAt && expiresAt > currentTime + 300000) {
return { accessToken };
}
const { data: clientData } = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", tenantId || "common"),
new URLSearchParams({
grant_type: "client_credentials",
scope: `https://vault.azure.net/.default`,
client_id: clientId,
client_secret: clientSecret
})
);
const updatedClientCredentials = {
...clientSecretCredentials,
accessToken: clientData.access_token,
expiresAt: currentTime + clientData.expires_in * 1000
};
const encryptedClientCredentials = await encryptAppConnectionCredentials({
credentials: updatedClientCredentials,
orgId: appConnection.orgId,
kmsService
});
await appConnectionDAL.updateById(appConnection.id, { encryptedCredentials: encryptedClientCredentials });
return { accessToken: clientData.access_token };
default:
throw new InternalServerError({
message: `Unhandled Azure Key Vault connection method: ${appConnection.method as AzureKeyVaultConnectionMethod}`
});
}
};
export const getAzureKeyVaultConnectionListItem = () => {
@@ -100,7 +144,10 @@ export const getAzureKeyVaultConnectionListItem = () => {
return {
name: "Azure Key Vault" as const,
app: AppConnection.AzureKeyVault as const,
methods: Object.values(AzureKeyVaultConnectionMethod) as [AzureKeyVaultConnectionMethod.OAuth],
methods: Object.values(AzureKeyVaultConnectionMethod) as [
AzureKeyVaultConnectionMethod.OAuth,
AzureKeyVaultConnectionMethod.ClientSecret
],
oauthClientId: INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID
};
};
@@ -111,68 +158,108 @@ export const validateAzureKeyVaultConnectionCredentials = async (config: TAzureK
const { INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID, INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET, SITE_URL } =
getConfig();
if (!INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID || !INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET) {
throw new InternalServerError({
message: `Azure ${getAppConnectionMethodName(method)} environment variables have not been configured`
});
}
let tokenResp: AxiosResponse<ExchangeCodeAzureResponse> | null = null;
let tokenError: AxiosError | null = null;
try {
tokenResp = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", inputCredentials.tenantId || "common"),
new URLSearchParams({
grant_type: "authorization_code",
code: inputCredentials.code,
scope: `openid offline_access https://vault.azure.net/.default`,
client_id: INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID,
client_secret: INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET,
redirect_uri: `${SITE_URL}/organization/app-connections/azure/oauth/callback`
})
);
} catch (e: unknown) {
if (e instanceof AxiosError) {
tokenError = e;
} else {
throw new BadRequestError({
message: `Unable to validate connection: verify credentials`
});
}
}
if (tokenError) {
if (tokenError instanceof AxiosError) {
throw new BadRequestError({
message: `Failed to get access token: ${
(tokenError?.response?.data as { error_description?: string })?.error_description || "Unknown error"
}`
});
} else {
throw new InternalServerError({
message: "Failed to get access token"
});
}
}
if (!tokenResp) {
throw new InternalServerError({
message: `Failed to get access token: Token was empty with no error`
});
}
switch (method) {
case AzureKeyVaultConnectionMethod.OAuth:
if (!INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID || !INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET) {
throw new InternalServerError({
message: `Azure ${getAppConnectionMethodName(method)} environment variables have not been configured`
});
}
let tokenResp: AxiosResponse<ExchangeCodeAzureResponse> | null = null;
let tokenError: AxiosError | null = null;
const oauthCredentials = inputCredentials as { code: string; tenantId?: string };
try {
tokenResp = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", oauthCredentials.tenantId || "common"),
new URLSearchParams({
grant_type: "authorization_code",
code: oauthCredentials.code,
scope: `openid offline_access https://vault.azure.net/.default`,
client_id: INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_ID,
client_secret: INF_APP_CONNECTION_AZURE_KEY_VAULT_CLIENT_SECRET,
redirect_uri: `${SITE_URL}/organization/app-connections/azure/oauth/callback`
})
);
} catch (e: unknown) {
if (e instanceof AxiosError) {
tokenError = e;
} else {
throw new BadRequestError({
message: `Unable to validate connection: verify credentials`
});
}
}
if (tokenError) {
if (tokenError instanceof AxiosError) {
throw new BadRequestError({
message: `Failed to get access token: ${
(tokenError?.response?.data as { error_description?: string })?.error_description || "Unknown error"
}`
});
} else {
throw new InternalServerError({
message: "Failed to get access token"
});
}
}
if (!tokenResp) {
throw new InternalServerError({
message: `Failed to get access token: Token was empty with no error`
});
}
return {
tenantId: inputCredentials.tenantId,
tenantId: oauthCredentials.tenantId,
accessToken: tokenResp.data.access_token,
refreshToken: tokenResp.data.refresh_token,
expiresAt: Date.now() + tokenResp.data.expires_in * 1000
};
case AzureKeyVaultConnectionMethod.ClientSecret:
const { tenantId, clientId, clientSecret } = inputCredentials as {
tenantId: string;
clientId: string;
clientSecret: string;
};
try {
const { data: clientData } = await request.post<ExchangeCodeAzureResponse>(
IntegrationUrls.AZURE_TOKEN_URL.replace("common", tenantId || "common"),
new URLSearchParams({
grant_type: "client_credentials",
scope: `https://vault.azure.net/.default`,
client_id: clientId,
client_secret: clientSecret
})
);
return {
tenantId,
accessToken: clientData.access_token,
expiresAt: Date.now() + clientData.expires_in * 1000,
clientId,
clientSecret
};
} catch (e: unknown) {
if (e instanceof AxiosError) {
throw new BadRequestError({
message: `Failed to get access token: ${
(e?.response?.data as { error_description?: string })?.error_description || "Unknown error"
}`
});
} else {
throw new InternalServerError({
message: "Failed to get access token"
});
}
}
default:
throw new InternalServerError({
message: `Unhandled Azure connection method: ${method as AzureKeyVaultConnectionMethod}`
message: `Unhandled Azure Key Vault connection method: ${method as AzureKeyVaultConnectionMethod}`
});
}
};
@@ -22,6 +22,29 @@ export const AzureKeyVaultConnectionOAuthOutputCredentialsSchema = z.object({
expiresAt: z.number()
});
export const AzureKeyVaultConnectionClientSecretInputCredentialsSchema = z.object({
clientId: z
.string()
.uuid()
.trim()
.min(1, "Client ID required")
.max(50, "Client ID must be at most 50 characters long"),
clientSecret: z
.string()
.trim()
.min(1, "Client Secret required")
.max(50, "Client Secret must be at most 50 characters long"),
tenantId: z.string().uuid().trim().min(1, "Tenant ID required")
});
export const AzureKeyVaultConnectionClientSecretOutputCredentialsSchema = z.object({
clientId: z.string(),
clientSecret: z.string(),
tenantId: z.string(),
accessToken: z.string(),
expiresAt: z.number()
});
export const ValidateAzureKeyVaultConnectionCredentialsSchema = z.discriminatedUnion("method", [
z.object({
method: z
@@ -30,6 +53,14 @@ export const ValidateAzureKeyVaultConnectionCredentialsSchema = z.discriminatedU
credentials: AzureKeyVaultConnectionOAuthInputCredentialsSchema.describe(
AppConnections.CREATE(AppConnection.AzureKeyVault).credentials
)
}),
z.object({
method: z
.literal(AzureKeyVaultConnectionMethod.ClientSecret)
.describe(AppConnections.CREATE(AppConnection.AzureKeyVault).method),
credentials: AzureKeyVaultConnectionClientSecretInputCredentialsSchema.describe(
AppConnections.CREATE(AppConnection.AzureKeyVault).credentials
)
})
]);
@@ -39,9 +70,13 @@ export const CreateAzureKeyVaultConnectionSchema = ValidateAzureKeyVaultConnecti
export const UpdateAzureKeyVaultConnectionSchema = z
.object({
credentials: AzureKeyVaultConnectionOAuthInputCredentialsSchema.optional().describe(
AppConnections.UPDATE(AppConnection.AzureKeyVault).credentials
)
credentials: z
.union([
AzureKeyVaultConnectionOAuthInputCredentialsSchema,
AzureKeyVaultConnectionClientSecretInputCredentialsSchema
])
.optional()
.describe(AppConnections.UPDATE(AppConnection.AzureKeyVault).credentials)
})
.and(GenericUpdateAppConnectionFieldsSchema(AppConnection.AzureKeyVault));
@@ -55,6 +90,10 @@ export const AzureKeyVaultConnectionSchema = z.intersection(
z.object({
method: z.literal(AzureKeyVaultConnectionMethod.OAuth),
credentials: AzureKeyVaultConnectionOAuthOutputCredentialsSchema
}),
z.object({
method: z.literal(AzureKeyVaultConnectionMethod.ClientSecret),
credentials: AzureKeyVaultConnectionClientSecretOutputCredentialsSchema
})
])
);
@@ -65,6 +104,13 @@ export const SanitizedAzureKeyVaultConnectionSchema = z.discriminatedUnion("meth
credentials: AzureKeyVaultConnectionOAuthOutputCredentialsSchema.pick({
tenantId: true
})
}),
BaseAzureKeyVaultConnectionSchema.extend({
method: z.literal(AzureKeyVaultConnectionMethod.ClientSecret),
credentials: AzureKeyVaultConnectionClientSecretOutputCredentialsSchema.pick({
clientId: true,
tenantId: true
})
})
]);
@@ -4,6 +4,7 @@ import { DiscriminativePick } from "@app/lib/types";
import { AppConnection } from "../app-connection-enums";
import {
AzureKeyVaultConnectionClientSecretOutputCredentialsSchema,
AzureKeyVaultConnectionOAuthOutputCredentialsSchema,
AzureKeyVaultConnectionSchema,
CreateAzureKeyVaultConnectionSchema,
@@ -36,3 +37,7 @@ export type ExchangeCodeAzureResponse = {
};
export type TAzureKeyVaultConnectionCredentials = z.infer<typeof AzureKeyVaultConnectionOAuthOutputCredentialsSchema>;
export type TAzureKeyVaultConnectionClientSecretCredentials = z.infer<
typeof AzureKeyVaultConnectionClientSecretOutputCredentialsSchema
>;
@@ -1,10 +1,16 @@
import { createAppAuth } from "@octokit/auth-app";
import { Octokit } from "@octokit/rest";
import { AxiosResponse } from "axios";
import { AxiosError, AxiosRequestConfig, AxiosResponse } from "axios";
import https from "https";
import RE2 from "re2";
import { verifyHostInputValidity } from "@app/ee/services/dynamic-secret/dynamic-secret-fns";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { getConfig } from "@app/lib/config/env";
import { request } from "@app/lib/config/request";
import { request as httpRequest } from "@app/lib/config/request";
import { BadRequestError, ForbiddenRequestError, InternalServerError } from "@app/lib/errors";
import { GatewayProxyProtocol, withGatewayProxy } from "@app/lib/gateway";
import { logger } from "@app/lib/logger";
import { blockLocalAndPrivateIpAddresses } from "@app/lib/validator";
import { getAppConnectionMethodName } from "@app/services/app-connection/app-connection-fns";
import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
@@ -24,123 +30,224 @@ export const getGitHubConnectionListItem = () => {
};
};
export const getGitHubClient = (appConnection: TGitHubConnection) => {
export const requestWithGitHubGateway = async <T>(
appConnection: { gatewayId?: string | null },
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
requestConfig: AxiosRequestConfig
): Promise<AxiosResponse<T>> => {
const { gatewayId } = appConnection;
// If gateway isn't set up, don't proxy request
if (!gatewayId) {
return httpRequest.request(requestConfig);
}
const url = new URL(requestConfig.url as string);
await blockLocalAndPrivateIpAddresses(url.toString());
const [targetHost] = await verifyHostInputValidity(url.host, true);
const relayDetails = await gatewayService.fnGetGatewayClientTlsByGatewayId(gatewayId);
const [relayHost, relayPort] = relayDetails.relayAddress.split(":");
return withGatewayProxy(
async (proxyPort) => {
const httpsAgent = new https.Agent({
servername: targetHost
});
url.protocol = "https:";
url.host = `localhost:${proxyPort}`;
const finalRequestConfig: AxiosRequestConfig = {
...requestConfig,
url: url.toString(),
httpsAgent,
headers: {
...requestConfig.headers,
Host: targetHost
}
};
try {
return await httpRequest.request(finalRequestConfig);
} catch (error) {
const axiosError = error as AxiosError;
logger.error("Error during GitHub gateway request:", axiosError.message, axiosError.response?.data);
throw error;
}
},
{
protocol: GatewayProxyProtocol.Tcp,
targetHost,
targetPort: 443,
relayHost,
relayPort: Number(relayPort),
identityId: relayDetails.identityId,
orgId: relayDetails.orgId,
tlsOptions: {
ca: relayDetails.certChain,
cert: relayDetails.certificate,
key: relayDetails.privateKey.toString()
}
}
);
};
export const getGitHubAppAuthToken = async (appConnection: TGitHubConnection) => {
const appCfg = getConfig();
const { method, credentials } = appConnection;
let client: Octokit;
const appId = appCfg.INF_APP_CONNECTION_GITHUB_APP_ID;
const appPrivateKey = appCfg.INF_APP_CONNECTION_GITHUB_APP_PRIVATE_KEY;
switch (method) {
case GitHubConnectionMethod.App:
if (!appId || !appPrivateKey) {
throw new InternalServerError({
message: `GitHub ${getAppConnectionMethodName(method).replace("GitHub", "")} has not been configured`
});
}
client = new Octokit({
authStrategy: createAppAuth,
auth: {
appId,
privateKey: appPrivateKey,
installationId: credentials.installationId
}
});
break;
case GitHubConnectionMethod.OAuth:
client = new Octokit({
auth: credentials.accessToken
});
break;
default:
throw new InternalServerError({
message: `Unhandled GitHub connection method: ${method as GitHubConnectionMethod}`
});
if (!appId || !appPrivateKey) {
throw new InternalServerError({
message: `GitHub App keys are not configured.`
});
}
return client;
if (appConnection.method !== GitHubConnectionMethod.App) {
throw new InternalServerError({ message: "Cannot generate GitHub App token for non-app connection" });
}
const appAuth = createAppAuth({
appId,
privateKey: appPrivateKey,
installationId: appConnection.credentials.installationId
});
const { token } = await appAuth({ type: "installation" });
return token;
};
function extractNextPageUrl(linkHeader: string | undefined): string | null {
if (!linkHeader) return null;
const links = linkHeader.split(",");
const nextLink = links.find((link) => link.includes('rel="next"'));
if (!nextLink) return null;
const match = new RE2(/<([^>]+)>/).exec(nextLink);
return match ? match[1] : null;
}
export const makePaginatedGitHubRequest = async <T, R = T[]>(
appConnection: TGitHubConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
path: string,
dataMapper?: (data: R) => T[]
): Promise<T[]> => {
const { credentials, method } = appConnection;
const token =
method === GitHubConnectionMethod.OAuth ? credentials.accessToken : await getGitHubAppAuthToken(appConnection);
let url: string | null = `https://api.${credentials.host || "github.com"}${path}`;
let results: T[] = [];
let i = 0;
while (url && i < 1000) {
// eslint-disable-next-line no-await-in-loop
const response: AxiosResponse<R> = await requestWithGitHubGateway<R>(appConnection, gatewayService, {
url,
method: "GET",
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28"
}
});
const items = dataMapper ? dataMapper(response.data) : (response.data as unknown as T[]);
results = results.concat(items);
url = extractNextPageUrl(response.headers.link as string | undefined);
i += 1;
}
return results;
};
type GitHubOrganization = {
login: string;
id: number;
type: string;
};
type GitHubRepository = {
id: number;
name: string;
owner: GitHubOrganization;
permissions?: {
admin: boolean;
maintain: boolean;
push: boolean;
triage: boolean;
pull: boolean;
};
};
export const getGitHubRepositories = async (appConnection: TGitHubConnection) => {
const client = getGitHubClient(appConnection);
type GitHubEnvironment = {
id: number;
name: string;
};
let repositories: GitHubRepository[];
switch (appConnection.method) {
case GitHubConnectionMethod.App:
repositories = await client.paginate("GET /installation/repositories");
break;
case GitHubConnectionMethod.OAuth:
default:
repositories = (await client.paginate("GET /user/repos")).filter((repo) => repo.permissions?.admin);
break;
export const getGitHubRepositories = async (
appConnection: TGitHubConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
if (appConnection.method === GitHubConnectionMethod.App) {
return makePaginatedGitHubRequest<GitHubRepository, { repositories: GitHubRepository[] }>(
appConnection,
gatewayService,
"/installation/repositories",
(data) => data.repositories
);
}
return repositories;
const repos = await makePaginatedGitHubRequest<GitHubRepository>(appConnection, gatewayService, "/user/repos");
return repos.filter((repo) => repo.permissions?.admin);
};
export const getGitHubOrganizations = async (appConnection: TGitHubConnection) => {
const client = getGitHubClient(appConnection);
export const getGitHubOrganizations = async (
appConnection: TGitHubConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
if (appConnection.method === GitHubConnectionMethod.App) {
const installationRepositories = await makePaginatedGitHubRequest<
GitHubRepository,
{ repositories: GitHubRepository[] }
>(appConnection, gatewayService, "/installation/repositories", (data) => data.repositories);
let organizations: GitHubOrganization[];
switch (appConnection.method) {
case GitHubConnectionMethod.App: {
const installationRepositories = await client.paginate("GET /installation/repositories");
const organizationMap: Record<string, GitHubOrganization> = {};
installationRepositories.forEach((repo) => {
if (repo.owner.type === "Organization") {
organizationMap[repo.owner.id] = repo.owner;
}
});
organizations = Object.values(organizationMap);
break;
}
case GitHubConnectionMethod.OAuth:
default:
organizations = await client.paginate("GET /user/orgs");
break;
}
return organizations;
};
export const getGitHubEnvironments = async (appConnection: TGitHubConnection, owner: string, repo: string) => {
const client = getGitHubClient(appConnection);
try {
const environments = await client.paginate("GET /repos/{owner}/{repo}/environments", {
owner,
repo
const organizationMap: Record<string, GitHubOrganization> = {};
installationRepositories.forEach((repo) => {
if (repo.owner.type === "Organization") {
organizationMap[repo.owner.id] = repo.owner;
}
});
return environments;
} catch (e) {
// repo doesn't have envs
if ((e as { status: number }).status === 404) {
return [];
}
return Object.values(organizationMap);
}
throw e;
return makePaginatedGitHubRequest<GitHubOrganization>(appConnection, gatewayService, "/user/orgs");
};
export const getGitHubEnvironments = async (
appConnection: TGitHubConnection,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
owner: string,
repo: string
) => {
try {
return await makePaginatedGitHubRequest<GitHubEnvironment, { environments: GitHubEnvironment[] }>(
appConnection,
gatewayService,
`/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/environments`,
(data) => data.environments
);
} catch (error) {
const axiosError = error as AxiosError;
if (axiosError.response?.status === 404) return [];
throw error;
}
};
@@ -159,9 +266,11 @@ export function isGithubErrorResponse(data: GithubTokenRespData): data is Github
return "error" in data;
}
export const validateGitHubConnectionCredentials = async (config: TGitHubConnectionConfig) => {
export const validateGitHubConnectionCredentials = async (
config: TGitHubConnectionConfig,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
const { credentials, method } = config;
const {
INF_APP_CONNECTION_GITHUB_OAUTH_CLIENT_ID,
INF_APP_CONNECTION_GITHUB_OAUTH_CLIENT_SECRET,
@@ -192,10 +301,13 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
}
let tokenResp: AxiosResponse<GithubTokenRespData>;
const host = credentials.host || "github.com";
try {
tokenResp = await request.get<GithubTokenRespData>("https://github.com/login/oauth/access_token", {
params: {
tokenResp = await requestWithGitHubGateway<GithubTokenRespData>(config, gatewayService, {
url: `https://${host}/login/oauth/access_token`,
method: "POST",
data: {
client_id: clientId,
client_secret: clientSecret,
code: credentials.code,
@@ -203,7 +315,7 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
},
headers: {
Accept: "application/json",
"Accept-Encoding": "application/json"
"Content-Type": "application/json"
}
});
@@ -233,7 +345,7 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
throw new InternalServerError({ message: `Missing access token: ${tokenResp.data.error}` });
}
const installationsResp = await request.get<{
const installationsResp = await requestWithGitHubGateway<{
installations: {
id: number;
account: {
@@ -242,7 +354,8 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
id: number;
};
}[];
}>(IntegrationUrls.GITHUB_USER_INSTALLATIONS, {
}>(config, gatewayService, {
url: IntegrationUrls.GITHUB_USER_INSTALLATIONS.replace("api.github.com", `api.${host}`),
headers: {
Accept: "application/json",
Authorization: `Bearer ${tokenResp.data.access_token}`,
@@ -11,20 +11,24 @@ import {
import { GitHubConnectionMethod } from "./github-connection-enums";
export const GitHubConnectionOAuthInputCredentialsSchema = z.object({
code: z.string().trim().min(1, "OAuth code required")
code: z.string().trim().min(1, "OAuth code required"),
host: z.string().trim().optional()
});
export const GitHubConnectionAppInputCredentialsSchema = z.object({
code: z.string().trim().min(1, "GitHub App code required"),
installationId: z.string().min(1, "GitHub App Installation ID required")
installationId: z.string().min(1, "GitHub App Installation ID required"),
host: z.string().trim().optional()
});
export const GitHubConnectionOAuthOutputCredentialsSchema = z.object({
accessToken: z.string()
accessToken: z.string(),
host: z.string().trim().optional()
});
export const GitHubConnectionAppOutputCredentialsSchema = z.object({
installationId: z.string()
installationId: z.string(),
host: z.string().trim().optional()
});
export const ValidateGitHubConnectionCredentialsSchema = z.discriminatedUnion("method", [
@@ -43,7 +47,9 @@ export const ValidateGitHubConnectionCredentialsSchema = z.discriminatedUnion("m
]);
export const CreateGitHubConnectionSchema = ValidateGitHubConnectionCredentialsSchema.and(
GenericCreateAppConnectionFieldsSchema(AppConnection.GitHub)
GenericCreateAppConnectionFieldsSchema(AppConnection.GitHub, {
supportsGateways: true
})
);
export const UpdateGitHubConnectionSchema = z
@@ -53,7 +59,11 @@ export const UpdateGitHubConnectionSchema = z
.optional()
.describe(AppConnections.UPDATE(AppConnection.GitHub).credentials)
})
.and(GenericUpdateAppConnectionFieldsSchema(AppConnection.GitHub));
.and(
GenericUpdateAppConnectionFieldsSchema(AppConnection.GitHub, {
supportsGateways: true
})
);
const BaseGitHubConnectionSchema = BaseAppConnectionSchema.extend({ app: z.literal(AppConnection.GitHub) });
@@ -1,3 +1,4 @@
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { OrgServiceActor } from "@app/lib/types";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import {
@@ -19,11 +20,14 @@ type TListGitHubEnvironmentsDTO = {
owner: string;
};
export const githubConnectionService = (getAppConnection: TGetAppConnectionFunc) => {
export const githubConnectionService = (
getAppConnection: TGetAppConnectionFunc,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
const listRepositories = async (connectionId: string, actor: OrgServiceActor) => {
const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);
const repositories = await getGitHubRepositories(appConnection);
const repositories = await getGitHubRepositories(appConnection, gatewayService);
return repositories;
};
@@ -31,7 +35,7 @@ export const githubConnectionService = (getAppConnection: TGetAppConnectionFunc)
const listOrganizations = async (connectionId: string, actor: OrgServiceActor) => {
const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);
const organizations = await getGitHubOrganizations(appConnection);
const organizations = await getGitHubOrganizations(appConnection, gatewayService);
return organizations;
};
@@ -42,7 +46,7 @@ export const githubConnectionService = (getAppConnection: TGetAppConnectionFunc)
) => {
const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);
const environments = await getGitHubEnvironments(appConnection, owner, repo);
const environments = await getGitHubEnvironments(appConnection, gatewayService, owner, repo);
return environments;
};
@@ -17,4 +17,7 @@ export type TGitHubConnectionInput = z.infer<typeof CreateGitHubConnectionSchema
export type TValidateGitHubConnectionCredentialsSchema = typeof ValidateGitHubConnectionCredentialsSchema;
export type TGitHubConnectionConfig = DiscriminativePick<TGitHubConnectionInput, "method" | "app" | "credentials">;
export type TGitHubConnectionConfig = DiscriminativePick<
TGitHubConnectionInput,
"method" | "app" | "credentials" | "gatewayId"
>;
@@ -645,7 +645,7 @@ export const projectServiceFactory = ({
const updateProject = async ({ actor, actorId, actorOrgId, actorAuthMethod, update, filter }: TUpdateProjectDTO) => {
const project = await projectDAL.findProjectByFilter(filter);
const { permission } = await permissionService.getProjectPermission({
const { permission, hasRole } = await permissionService.getProjectPermission({
actor,
actorId,
projectId: project.id,
@@ -667,6 +667,12 @@ export const projectServiceFactory = ({
}
}
if (update.secretDetectionIgnoreValues && !hasRole(ProjectMembershipRole.Admin)) {
throw new ForbiddenRequestError({
message: "Only admins can update secret detection ignore values"
});
}
const updatedProject = await projectDAL.updateById(project.id, {
name: update.name,
description: update.description,
@@ -676,7 +682,8 @@ export const projectServiceFactory = ({
slug: update.slug,
secretSharing: update.secretSharing,
defaultProduct: update.defaultProduct,
showSnapshotsLegacy: update.showSnapshotsLegacy
showSnapshotsLegacy: update.showSnapshotsLegacy,
secretDetectionIgnoreValues: update.secretDetectionIgnoreValues
});
return updatedProject;
@@ -96,6 +96,7 @@ export type TUpdateProjectDTO = {
slug?: string;
secretSharing?: boolean;
showSnapshotsLegacy?: boolean;
secretDetectionIgnoreValues?: string[];
};
} & Omit<TProjectPermission, "projectId">;
@@ -13,7 +13,7 @@ import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
import { TAzureAppConfigurationSyncWithCredentials } from "./azure-app-configuration-sync-types";
type TAzureAppConfigurationSyncFactoryDeps = {
appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "update">;
appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "updateById">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
};
@@ -12,7 +12,7 @@ import { SecretSyncError } from "../secret-sync-errors";
import { GetAzureKeyVaultSecret, TAzureKeyVaultSyncWithCredentials } from "./azure-key-vault-sync-types";
type TAzureKeyVaultSyncFactoryDeps = {
appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "update">;
appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "updateById">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
};
@@ -1,7 +1,12 @@
import { Octokit } from "@octokit/rest";
import sodium from "libsodium-wrappers";
import { getGitHubClient } from "@app/services/app-connection/github";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import {
getGitHubAppAuthToken,
GitHubConnectionMethod,
makePaginatedGitHubRequest,
requestWithGitHubGateway
} from "@app/services/app-connection/github";
import { GitHubSyncScope, GitHubSyncVisibility } from "@app/services/secret-sync/github/github-sync-enums";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import { matchesSchema } from "@app/services/secret-sync/secret-sync-fns";
@@ -12,155 +17,165 @@ import { TGitHubPublicKey, TGitHubSecret, TGitHubSecretPayload, TGitHubSyncWithC
// TODO: rate limit handling
const getEncryptedSecrets = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => {
let encryptedSecrets: TGitHubSecret[];
const { destinationConfig } = secretSync;
const getEncryptedSecrets = async (
secretSync: TGitHubSyncWithCredentials,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
const { destinationConfig, connection } = secretSync;
let path: string;
switch (destinationConfig.scope) {
case GitHubSyncScope.Organization: {
encryptedSecrets = await client.paginate("GET /orgs/{org}/actions/secrets", {
org: destinationConfig.org
});
path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets`;
break;
}
case GitHubSyncScope.Repository: {
encryptedSecrets = await client.paginate("GET /repos/{owner}/{repo}/actions/secrets", {
owner: destinationConfig.owner,
repo: destinationConfig.repo
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets`;
break;
}
case GitHubSyncScope.RepositoryEnvironment:
default: {
encryptedSecrets = await client.paginate("GET /repos/{owner}/{repo}/environments/{environment_name}/secrets", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
environment_name: destinationConfig.env
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets`;
break;
}
}
return encryptedSecrets;
return makePaginatedGitHubRequest<TGitHubSecret, { secrets: TGitHubSecret[] }>(
connection,
gatewayService,
path,
(data) => data.secrets
);
};
const getPublicKey = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => {
let publicKey: TGitHubPublicKey;
const { destinationConfig } = secretSync;
const getPublicKey = async (
secretSync: TGitHubSyncWithCredentials,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
token: string
) => {
const { destinationConfig, connection } = secretSync;
let path: string;
switch (destinationConfig.scope) {
case GitHubSyncScope.Organization: {
publicKey = (
await client.request("GET /orgs/{org}/actions/secrets/public-key", {
org: destinationConfig.org
})
).data;
path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets/public-key`;
break;
}
case GitHubSyncScope.Repository: {
publicKey = (
await client.request("GET /repos/{owner}/{repo}/actions/secrets/public-key", {
owner: destinationConfig.owner,
repo: destinationConfig.repo
})
).data;
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets/public-key`;
break;
}
case GitHubSyncScope.RepositoryEnvironment:
default: {
publicKey = (
await client.request("GET /repos/{owner}/{repo}/environments/{environment_name}/secrets/public-key", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
environment_name: destinationConfig.env
})
).data;
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets/public-key`;
break;
}
}
return publicKey;
const response = await requestWithGitHubGateway<TGitHubPublicKey>(connection, gatewayService, {
url: `https://api.${connection.credentials.host || "github.com"}${path}`,
method: "GET",
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28"
}
});
return response.data;
};
const deleteSecret = async (
client: Octokit,
secretSync: TGitHubSyncWithCredentials,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
token: string,
encryptedSecret: TGitHubSecret
) => {
const { destinationConfig } = secretSync;
const { destinationConfig, connection } = secretSync;
let path: string;
switch (destinationConfig.scope) {
case GitHubSyncScope.Organization: {
await client.request(`DELETE /orgs/{org}/actions/secrets/{secret_name}`, {
org: destinationConfig.org,
secret_name: encryptedSecret.name
});
path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets/${encodeURIComponent(encryptedSecret.name)}`;
break;
}
case GitHubSyncScope.Repository: {
await client.request("DELETE /repos/{owner}/{repo}/actions/secrets/{secret_name}", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
secret_name: encryptedSecret.name
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets/${encodeURIComponent(encryptedSecret.name)}`;
break;
}
case GitHubSyncScope.RepositoryEnvironment:
default: {
await client.request("DELETE /repos/{owner}/{repo}/environments/{environment_name}/secrets/{secret_name}", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
environment_name: destinationConfig.env,
secret_name: encryptedSecret.name
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets/${encodeURIComponent(encryptedSecret.name)}`;
break;
}
}
await requestWithGitHubGateway(connection, gatewayService, {
url: `https://api.${connection.credentials.host || "github.com"}${path}`,
method: "DELETE",
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28"
}
});
};
const putSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredentials, payload: TGitHubSecretPayload) => {
const { destinationConfig } = secretSync;
const putSecret = async (
secretSync: TGitHubSyncWithCredentials,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">,
token: string,
payload: TGitHubSecretPayload
) => {
const { destinationConfig, connection } = secretSync;
let path: string;
let body: Record<string, string | number[]> = payload;
switch (destinationConfig.scope) {
case GitHubSyncScope.Organization: {
const { visibility, selectedRepositoryIds } = destinationConfig;
await client.request(`PUT /orgs/{org}/actions/secrets/{secret_name}`, {
org: destinationConfig.org,
path = `/orgs/${encodeURIComponent(destinationConfig.org)}/actions/secrets/${encodeURIComponent(payload.secret_name)}`;
body = {
...payload,
visibility,
...(visibility === GitHubSyncVisibility.Selected && {
selected_repository_ids: selectedRepositoryIds
})
});
};
break;
}
case GitHubSyncScope.Repository: {
await client.request("PUT /repos/{owner}/{repo}/actions/secrets/{secret_name}", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
...payload
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/actions/secrets/${encodeURIComponent(payload.secret_name)}`;
break;
}
case GitHubSyncScope.RepositoryEnvironment:
default: {
await client.request("PUT /repos/{owner}/{repo}/environments/{environment_name}/secrets/{secret_name}", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
environment_name: destinationConfig.env,
...payload
});
path = `/repos/${encodeURIComponent(destinationConfig.owner)}/${encodeURIComponent(destinationConfig.repo)}/environments/${encodeURIComponent(destinationConfig.env)}/secrets/${encodeURIComponent(payload.secret_name)}`;
break;
}
}
await requestWithGitHubGateway(connection, gatewayService, {
url: `https://api.${connection.credentials.host || "github.com"}${path}`,
method: "PUT",
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28"
},
data: body
});
};
export const GithubSyncFns = {
syncSecrets: async (secretSync: TGitHubSyncWithCredentials, secretMap: TSecretMap) => {
syncSecrets: async (
secretSync: TGitHubSyncWithCredentials,
ogSecretMap: TSecretMap,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
const secretMap = Object.fromEntries(Object.entries(ogSecretMap).map(([i, v]) => [i.toUpperCase(), v]));
switch (secretSync.destinationConfig.scope) {
case GitHubSyncScope.Organization:
if (Object.values(secretMap).length > 1000) {
@@ -187,38 +202,40 @@ export const GithubSyncFns = {
);
}
const client = getGitHubClient(secretSync.connection);
const { connection } = secretSync;
const token =
connection.method === GitHubConnectionMethod.OAuth
? connection.credentials.accessToken
: await getGitHubAppAuthToken(connection);
const encryptedSecrets = await getEncryptedSecrets(client, secretSync);
const encryptedSecrets = await getEncryptedSecrets(secretSync, gatewayService);
const publicKey = await getPublicKey(secretSync, gatewayService, token);
const publicKey = await getPublicKey(client, secretSync);
await sodium.ready;
for await (const key of Object.keys(secretMap)) {
// convert secret & base64 key to Uint8Array.
const binaryKey = sodium.from_base64(publicKey.key, sodium.base64_variants.ORIGINAL);
const binarySecretValue = sodium.from_string(secretMap[key].value);
await sodium.ready.then(async () => {
for await (const key of Object.keys(secretMap)) {
// convert secret & base64 key to Uint8Array.
const binaryKey = sodium.from_base64(publicKey.key, sodium.base64_variants.ORIGINAL);
const binarySecretValue = sodium.from_string(secretMap[key].value);
// encrypt secret using libsodium
const encryptedBytes = sodium.crypto_box_seal(binarySecretValue, binaryKey);
// encrypt secret using libsodium
const encryptedBytes = sodium.crypto_box_seal(binarySecretValue, binaryKey);
// convert encrypted Uint8Array to base64
const encryptedSecretValue = sodium.to_base64(encryptedBytes, sodium.base64_variants.ORIGINAL);
// convert encrypted Uint8Array to base64
const encryptedSecretValue = sodium.to_base64(encryptedBytes, sodium.base64_variants.ORIGINAL);
try {
await putSecret(client, secretSync, {
secret_name: key,
encrypted_value: encryptedSecretValue,
key_id: publicKey.key_id
});
} catch (error) {
throw new SecretSyncError({
error,
secretKey: key
});
}
try {
await putSecret(secretSync, gatewayService, token, {
secret_name: key,
encrypted_value: encryptedSecretValue,
key_id: publicKey.key_id
});
} catch (error) {
throw new SecretSyncError({
error,
secretKey: key
});
}
});
}
if (secretSync.syncOptions.disableSecretDeletion) return;
@@ -228,21 +245,31 @@ export const GithubSyncFns = {
continue;
if (!(encryptedSecret.name in secretMap)) {
await deleteSecret(client, secretSync, encryptedSecret);
await deleteSecret(secretSync, gatewayService, token, encryptedSecret);
}
}
},
getSecrets: async (secretSync: TGitHubSyncWithCredentials) => {
throw new Error(`${SECRET_SYNC_NAME_MAP[secretSync.destination]} does not support importing secrets.`);
},
removeSecrets: async (secretSync: TGitHubSyncWithCredentials, secretMap: TSecretMap) => {
const client = getGitHubClient(secretSync.connection);
removeSecrets: async (
secretSync: TGitHubSyncWithCredentials,
ogSecretMap: TSecretMap,
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">
) => {
const secretMap = Object.fromEntries(Object.entries(ogSecretMap).map(([i, v]) => [i.toUpperCase(), v]));
const encryptedSecrets = await getEncryptedSecrets(client, secretSync);
const { connection } = secretSync;
const token =
connection.method === GitHubConnectionMethod.OAuth
? connection.credentials.accessToken
: await getGitHubAppAuthToken(connection);
const encryptedSecrets = await getEncryptedSecrets(secretSync, gatewayService);
for await (const encryptedSecret of encryptedSecrets) {
if (encryptedSecret.name in secretMap) {
await deleteSecret(client, secretSync, encryptedSecret);
await deleteSecret(secretSync, gatewayService, token, encryptedSecret);
}
}
}
@@ -1,6 +1,7 @@
import { AxiosError } from "axios";
import handlebars from "handlebars";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import { OCI_VAULT_SYNC_LIST_OPTION, OCIVaultSyncFns } from "@app/ee/services/secret-sync/oci-vault";
import { BadRequestError } from "@app/lib/errors";
@@ -97,6 +98,7 @@ export const listSecretSyncOptions = () => {
type TSyncSecretDeps = {
appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "update" | "updateById">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
};
// Add schema to secret keys
@@ -191,7 +193,7 @@ export const SecretSyncFns = {
syncSecrets: (
secretSync: TSecretSyncWithCredentials,
secretMap: TSecretMap,
{ kmsService, appConnectionDAL }: TSyncSecretDeps
{ kmsService, appConnectionDAL, gatewayService }: TSyncSecretDeps
): Promise<void> => {
const schemaSecretMap = addSchema(secretMap, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema);
@@ -201,7 +203,7 @@ export const SecretSyncFns = {
case SecretSync.AWSSecretsManager:
return AwsSecretsManagerSyncFns.syncSecrets(secretSync, schemaSecretMap);
case SecretSync.GitHub:
return GithubSyncFns.syncSecrets(secretSync, schemaSecretMap);
return GithubSyncFns.syncSecrets(secretSync, schemaSecretMap, gatewayService);
case SecretSync.GCPSecretManager:
return GcpSyncFns.syncSecrets(secretSync, schemaSecretMap);
case SecretSync.AzureKeyVault:
@@ -395,7 +397,7 @@ export const SecretSyncFns = {
removeSecrets: (
secretSync: TSecretSyncWithCredentials,
secretMap: TSecretMap,
{ kmsService, appConnectionDAL }: TSyncSecretDeps
{ kmsService, appConnectionDAL, gatewayService }: TSyncSecretDeps
): Promise<void> => {
const schemaSecretMap = addSchema(secretMap, secretSync.environment?.slug || "", secretSync.syncOptions.keySchema);
@@ -405,7 +407,7 @@ export const SecretSyncFns = {
case SecretSync.AWSSecretsManager:
return AwsSecretsManagerSyncFns.removeSecrets(secretSync, schemaSecretMap);
case SecretSync.GitHub:
return GithubSyncFns.removeSecrets(secretSync, schemaSecretMap);
return GithubSyncFns.removeSecrets(secretSync, schemaSecretMap, gatewayService);
case SecretSync.GCPSecretManager:
return GcpSyncFns.removeSecrets(secretSync, schemaSecretMap);
case SecretSync.AzureKeyVault:
@@ -4,6 +4,7 @@ import { Job } from "bullmq";
import { ProjectMembershipRole, SecretType } from "@app/db/schemas";
import { EventType, TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-types";
import { TGatewayServiceFactory } from "@app/ee/services/gateway/gateway-service";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
@@ -96,6 +97,7 @@ type TSecretSyncQueueFactoryDep = {
resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">;
folderCommitService: Pick<TFolderCommitServiceFactory, "createCommit">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
gatewayService: Pick<TGatewayServiceFactory, "fnGetGatewayClientTlsByGatewayId">;
};
type SecretSyncActionJob = Job<
@@ -138,7 +140,8 @@ export const secretSyncQueueFactory = ({
secretVersionTagV2BridgeDAL,
resourceMetadataDAL,
folderCommitService,
licenseService
licenseService,
gatewayService
}: TSecretSyncQueueFactoryDep) => {
const appCfg = getConfig();
@@ -353,7 +356,8 @@ export const secretSyncQueueFactory = ({
const importedSecrets = await SecretSyncFns.getSecrets(secretSync, {
appConnectionDAL,
kmsService
kmsService,
gatewayService
});
if (!Object.keys(importedSecrets).length) return {};
@@ -481,7 +485,8 @@ export const secretSyncQueueFactory = ({
await SecretSyncFns.syncSecrets(secretSyncWithCredentials, secretMap, {
appConnectionDAL,
kmsService
kmsService,
gatewayService
});
isSynced = true;
@@ -730,7 +735,8 @@ export const secretSyncQueueFactory = ({
secretMap,
{
appConnectionDAL,
kmsService
kmsService,
gatewayService
}
);
@@ -25,6 +25,7 @@ import {
import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal";
import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal";
import { scanSecretPolicyViolations } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
import { TKeyStoreFactory } from "@app/keystore/keystore";
import { DatabaseErrorCode } from "@app/lib/error-codes";
@@ -38,6 +39,7 @@ import { ActorType } from "../auth/auth-type";
import { TCommitResourceChangeDTO, TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service";
import { TKmsServiceFactory } from "../kms/kms-service";
import { KmsDataKey } from "../kms/kms-types";
import { TProjectDALFactory } from "../project/project-dal";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { TReminderServiceFactory } from "../reminder/reminder-types";
import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal";
@@ -88,6 +90,7 @@ import { TSecretVersionV2TagDALFactory } from "./secret-version-tag-dal";
type TSecretV2BridgeServiceFactoryDep = {
secretDAL: TSecretV2BridgeDALFactory;
projectDAL: Pick<TProjectDALFactory, "findById">;
secretVersionDAL: TSecretVersionV2DALFactory;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
secretVersionTagDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
@@ -126,6 +129,7 @@ export type TSecretV2BridgeServiceFactory = ReturnType<typeof secretV2BridgeServ
*/
export const secretV2BridgeServiceFactory = ({
secretDAL,
projectDAL,
projectEnvDAL,
secretTagDAL,
secretVersionDAL,
@@ -295,6 +299,19 @@ export const secretV2BridgeServiceFactory = ({
})
);
const project = await projectDAL.findById(projectId);
await scanSecretPolicyViolations(
projectId,
secretPath,
[
{
secretKey: inputSecret.secretName,
secretValue: inputSecret.secretValue
}
],
project.secretDetectionIgnoreValues || []
);
const { nestedReferences, localReferences } = getAllSecretReferences(inputSecret.secretValue);
const allSecretReferences = nestedReferences.concat(
localReferences.map((el) => ({ secretKey: el, secretPath, environment }))
@@ -506,6 +523,21 @@ export const secretV2BridgeServiceFactory = ({
const { secretName, secretValue } = inputSecret;
if (secretValue) {
const project = await projectDAL.findById(projectId);
await scanSecretPolicyViolations(
projectId,
secretPath,
[
{
secretKey: inputSecret.newSecretName || secretName,
secretValue
}
],
project.secretDetectionIgnoreValues || []
);
}
const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId
@@ -1585,6 +1617,9 @@ export const secretV2BridgeServiceFactory = ({
if (secrets.length)
throw new BadRequestError({ message: `Secret already exist: ${secrets.map((el) => el.key).join(",")}` });
const project = await projectDAL.findById(projectId);
await scanSecretPolicyViolations(projectId, secretPath, inputSecrets, project.secretDetectionIgnoreValues || []);
// get all tags
const sanitizedTagIds = inputSecrets.flatMap(({ tagIds = [] }) => tagIds);
const tags = sanitizedTagIds.length ? await secretTagDAL.findManyTagsById(projectId, sanitizedTagIds) : [];
@@ -1925,6 +1960,19 @@ export const secretV2BridgeServiceFactory = ({
});
await $validateSecretReferences(projectId, permission, secretReferences, tx);
const project = await projectDAL.findById(projectId);
await scanSecretPolicyViolations(
projectId,
secretPath,
secretsToUpdate
.filter((el) => el.secretValue)
.map((el) => ({
secretKey: el.newSecretName || el.secretKey,
secretValue: el.secretValue as string
})),
project.secretDetectionIgnoreValues || []
);
const bulkUpdatedSecrets = await fnSecretBulkUpdate({
folderId,
orgId: actorOrgId,