feature: secret sync base + aws parameter store & github

This commit is contained in:
Scott Wilson
2025-01-15 21:52:50 -08:00
parent b440e918ac
commit 473efa91f0
214 changed files with 10212 additions and 623 deletions
@@ -2,3 +2,50 @@ export enum AppConnection {
GitHub = "github",
AWS = "aws"
}
export enum AWSRegion {
// US
US_EAST_1 = "us-east-1", // N. Virginia
US_EAST_2 = "us-east-2", // Ohio
US_WEST_1 = "us-west-1", // N. California
US_WEST_2 = "us-west-2", // Oregon
// GovCloud
US_GOV_EAST_1 = "us-gov-east-1", // US-East
US_GOV_WEST_1 = "us-gov-west-1", // US-West
// Africa
AF_SOUTH_1 = "af-south-1", // Cape Town
// Asia Pacific
AP_EAST_1 = "ap-east-1", // Hong Kong
AP_SOUTH_1 = "ap-south-1", // Mumbai
AP_SOUTH_2 = "ap-south-2", // Hyderabad
AP_NORTHEAST_1 = "ap-northeast-1", // Tokyo
AP_NORTHEAST_2 = "ap-northeast-2", // Seoul
AP_NORTHEAST_3 = "ap-northeast-3", // Osaka
AP_SOUTHEAST_1 = "ap-southeast-1", // Singapore
AP_SOUTHEAST_2 = "ap-southeast-2", // Sydney
AP_SOUTHEAST_3 = "ap-southeast-3", // Jakarta
AP_SOUTHEAST_4 = "ap-southeast-4", // Melbourne
// Canada
CA_CENTRAL_1 = "ca-central-1", // Central
// Europe
EU_CENTRAL_1 = "eu-central-1", // Frankfurt
EU_CENTRAL_2 = "eu-central-2", // Zurich
EU_WEST_1 = "eu-west-1", // Ireland
EU_WEST_2 = "eu-west-2", // London
EU_WEST_3 = "eu-west-3", // Paris
EU_SOUTH_1 = "eu-south-1", // Milan
EU_SOUTH_2 = "eu-south-2", // Spain
EU_NORTH_1 = "eu-north-1", // Stockholm
// Middle East
ME_SOUTH_1 = "me-south-1", // Bahrain
ME_CENTRAL_1 = "me-central-1", // UAE
// South America
SA_EAST_1 = "sa-east-1" // Sao Paulo
}
@@ -1,3 +1,4 @@
import { TAppConnections } from "@app/db/schemas/app-connections";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { TAppConnectionServiceFactoryDep } from "@app/services/app-connection/app-connection-service";
import { TAppConnection, TAppConnectionConfig } from "@app/services/app-connection/app-connection-types";
@@ -64,9 +65,8 @@ export const validateAppConnectionCredentials = async (
): Promise<TAppConnection["credentials"]> => {
const { app } = appConnection;
switch (app) {
case AppConnection.AWS: {
case AppConnection.AWS:
return validateAwsConnectionCredentials(appConnection);
}
case AppConnection.GitHub:
return validateGitHubConnectionCredentials(appConnection);
default:
@@ -90,3 +90,17 @@ export const getAppConnectionMethodName = (method: TAppConnection["method"]) =>
throw new Error(`Unhandled App Connection Method: ${method}`);
}
};
export const decryptAppConnection = async (
appConnection: TAppConnections,
kmsService: TAppConnectionServiceFactoryDep["kmsService"]
) => {
return {
...appConnection,
credentials: await decryptAppConnectionCredentials({
encryptedCredentials: appConnection.encryptedCredentials,
orgId: appConnection.orgId,
kmsService
})
} as TAppConnection;
};
@@ -1,13 +1,12 @@
import { ForbiddenError } from "@casl/ability";
import { ForbiddenError, subject } from "@casl/ability";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission";
import { OrgPermissionAppConnectionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
import { DiscriminativePick, OrgServiceActor } from "@app/lib/types";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import {
decryptAppConnectionCredentials,
decryptAppConnection,
encryptAppConnectionCredentials,
getAppConnectionMethodName,
listAppConnectionOptions,
@@ -23,6 +22,7 @@ import {
} from "@app/services/app-connection/app-connection-types";
import { ValidateAwsConnectionCredentialsSchema } from "@app/services/app-connection/aws";
import { ValidateGitHubConnectionCredentialsSchema } from "@app/services/app-connection/github";
import { githubConnectionService } from "@app/services/app-connection/github/github-connection-service";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { TAppConnectionDALFactory } from "./app-connection-dal";
@@ -31,7 +31,6 @@ export type TAppConnectionServiceFactoryDep = {
appConnectionDAL: TAppConnectionDALFactory;
permissionService: Pick<TPermissionServiceFactory, "getOrgPermission">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">; // TODO: remove once launched
};
export type TAppConnectionServiceFactory = ReturnType<typeof appConnectionServiceFactory>;
@@ -44,19 +43,9 @@ const VALIDATE_APP_CONNECTION_CREDENTIALS_MAP: Record<AppConnection, TValidateAp
export const appConnectionServiceFactory = ({
appConnectionDAL,
permissionService,
kmsService,
licenseService
kmsService
}: TAppConnectionServiceFactoryDep) => {
// app connections are disabled for public until launch
const checkAppServicesAvailability = async (orgId: string) => {
const subscription = await licenseService.getPlan(orgId);
if (!subscription.appConnections) throw new BadRequestError({ message: "App Connections are not available yet." });
};
const listAppConnectionsByOrg = async (actor: OrgServiceActor, app?: AppConnection) => {
await checkAppServicesAvailability(actor.orgId);
const { permission } = await permissionService.getOrgPermission(
actor.type,
actor.id,
@@ -65,7 +54,10 @@ export const appConnectionServiceFactory = ({
actor.orgId
);
ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.AppConnections);
ForbiddenError.from(permission).throwUnlessCan(
OrgPermissionAppConnectionActions.Read,
OrgPermissionSubjects.AppConnections
);
const appConnections = await appConnectionDAL.find(
app
@@ -78,24 +70,11 @@ export const appConnectionServiceFactory = ({
return Promise.all(
appConnections
.sort((a, b) => a.name.toLowerCase().localeCompare(b.name.toLowerCase()))
.map(async ({ encryptedCredentials, ...connection }) => {
const credentials = await decryptAppConnectionCredentials({
encryptedCredentials,
kmsService,
orgId: connection.orgId
});
return {
...connection,
credentials
} as TAppConnection;
})
.map((appConnection) => decryptAppConnection(appConnection, kmsService))
);
};
const findAppConnectionById = async (app: AppConnection, connectionId: string, actor: OrgServiceActor) => {
await checkAppServicesAvailability(actor.orgId);
const appConnection = await appConnectionDAL.findById(connectionId);
if (!appConnection) throw new NotFoundError({ message: `Could not find App Connection with ID ${connectionId}` });
@@ -108,24 +87,18 @@ export const appConnectionServiceFactory = ({
appConnection.orgId
);
ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.AppConnections);
ForbiddenError.from(permission).throwUnlessCan(
OrgPermissionAppConnectionActions.Read,
OrgPermissionSubjects.AppConnections
);
if (appConnection.app !== app)
throw new BadRequestError({ message: `App Connection with ID ${connectionId} is not for App "${app}"` });
return {
...appConnection,
credentials: await decryptAppConnectionCredentials({
encryptedCredentials: appConnection.encryptedCredentials,
orgId: appConnection.orgId,
kmsService
})
} as TAppConnection;
return decryptAppConnection(appConnection, kmsService);
};
const findAppConnectionByName = async (app: AppConnection, connectionName: string, actor: OrgServiceActor) => {
await checkAppServicesAvailability(actor.orgId);
const appConnection = await appConnectionDAL.findOne({ name: connectionName, orgId: actor.orgId });
if (!appConnection)
@@ -139,27 +112,21 @@ export const appConnectionServiceFactory = ({
appConnection.orgId
);
ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.AppConnections);
ForbiddenError.from(permission).throwUnlessCan(
OrgPermissionAppConnectionActions.Read,
OrgPermissionSubjects.AppConnections
);
if (appConnection.app !== app)
throw new BadRequestError({ message: `App Connection with name ${connectionName} is not for App "${app}"` });
return {
...appConnection,
credentials: await decryptAppConnectionCredentials({
encryptedCredentials: appConnection.encryptedCredentials,
orgId: appConnection.orgId,
kmsService
})
} as TAppConnection;
return decryptAppConnection(appConnection, kmsService);
};
const createAppConnection = async (
{ method, app, credentials, ...params }: TCreateAppConnectionDTO,
actor: OrgServiceActor
) => {
await checkAppServicesAvailability(actor.orgId);
const { permission } = await permissionService.getOrgPermission(
actor.type,
actor.id,
@@ -168,7 +135,10 @@ export const appConnectionServiceFactory = ({
actor.orgId
);
ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.AppConnections);
ForbiddenError.from(permission).throwUnlessCan(
OrgPermissionAppConnectionActions.Create,
OrgPermissionSubjects.AppConnections
);
const appConnection = await appConnectionDAL.transaction(async (tx) => {
const isConflictingName = Boolean(
@@ -216,15 +186,13 @@ export const appConnectionServiceFactory = ({
};
});
return appConnection;
return appConnection as TAppConnection;
};
const updateAppConnection = async (
{ connectionId, credentials, ...params }: TUpdateAppConnectionDTO,
actor: OrgServiceActor
) => {
await checkAppServicesAvailability(actor.orgId);
const appConnection = await appConnectionDAL.findById(connectionId);
if (!appConnection) throw new NotFoundError({ message: `Could not find App Connection with ID ${connectionId}` });
@@ -237,7 +205,10 @@ export const appConnectionServiceFactory = ({
appConnection.orgId
);
ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.AppConnections);
ForbiddenError.from(permission).throwUnlessCan(
OrgPermissionAppConnectionActions.Edit,
OrgPermissionSubjects.AppConnections
);
const updatedAppConnection = await appConnectionDAL.transaction(async (tx) => {
if (params.name && appConnection.name !== params.name) {
@@ -304,19 +275,10 @@ export const appConnectionServiceFactory = ({
return updatedConnection;
});
return {
...updatedAppConnection,
credentials: await decryptAppConnectionCredentials({
encryptedCredentials: updatedAppConnection.encryptedCredentials,
orgId: updatedAppConnection.orgId,
kmsService
})
} as TAppConnection;
return decryptAppConnection(updatedAppConnection, kmsService);
};
const deleteAppConnection = async (app: AppConnection, connectionId: string, actor: OrgServiceActor) => {
await checkAppServicesAvailability(actor.orgId);
const appConnection = await appConnectionDAL.findById(connectionId);
if (!appConnection) throw new NotFoundError({ message: `Could not find App Connection with ID ${connectionId}` });
@@ -329,23 +291,85 @@ export const appConnectionServiceFactory = ({
appConnection.orgId
);
ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Delete, OrgPermissionSubjects.AppConnections);
ForbiddenError.from(permission).throwUnlessCan(
OrgPermissionAppConnectionActions.Delete,
OrgPermissionSubjects.AppConnections
);
if (appConnection.app !== app)
throw new BadRequestError({ message: `App Connection with ID ${connectionId} is not for App "${app}"` });
// TODO: specify delete error message if due to existing dependencies
// TODO (scott): add option to delete all dependencies
const deletedAppConnection = await appConnectionDAL.deleteById(connectionId);
try {
const deletedAppConnection = await appConnectionDAL.deleteById(connectionId);
return {
...deletedAppConnection,
credentials: await decryptAppConnectionCredentials({
encryptedCredentials: deletedAppConnection.encryptedCredentials,
orgId: deletedAppConnection.orgId,
kmsService
})
} as TAppConnection;
return await decryptAppConnection(deletedAppConnection, kmsService);
} catch (err) {
if (err instanceof DatabaseError && (err.error as { code: string })?.code === "23503") {
throw new BadRequestError({
message:
"Cannot delete App Connection with existing connections. Remove all existing connections and try again."
});
}
throw err;
}
};
const connectAppConnectionById = async <T extends TAppConnection>(
app: AppConnection,
connectionId: string,
actor: OrgServiceActor
) => {
const appConnection = await appConnectionDAL.findById(connectionId);
if (!appConnection) throw new NotFoundError({ message: `Could not find App Connection with ID ${connectionId}` });
const { permission: orgPermission } = await permissionService.getOrgPermission(
actor.type,
actor.id,
appConnection.orgId,
actor.authMethod,
actor.orgId
);
ForbiddenError.from(orgPermission).throwUnlessCan(
OrgPermissionAppConnectionActions.Connect,
subject(OrgPermissionSubjects.AppConnections, { connectionId: appConnection.id })
);
if (appConnection.app !== app)
throw new BadRequestError({
message: `${
APP_CONNECTION_NAME_MAP[appConnection.app as AppConnection]
} Connection with ID ${connectionId} cannot be used to connect to ${APP_CONNECTION_NAME_MAP[app]}`
});
const connection = await decryptAppConnection(appConnection, kmsService);
return connection as T;
};
const listAvailableAppConnectionsForUser = async (app: AppConnection, actor: OrgServiceActor) => {
const { permission: orgPermission } = await permissionService.getOrgPermission(
actor.type,
actor.id,
actor.orgId,
actor.authMethod,
actor.orgId
);
const appConnections = await appConnectionDAL.find({ app, orgId: actor.orgId });
const availableConnections = appConnections.filter((connection) =>
orgPermission.can(
OrgPermissionAppConnectionActions.Connect,
subject(OrgPermissionSubjects.AppConnections, { connectionId: connection.id })
)
);
return availableConnections as Omit<TAppConnection, "credentials">[];
};
return {
@@ -355,6 +379,9 @@ export const appConnectionServiceFactory = ({
findAppConnectionByName,
createAppConnection,
updateAppConnection,
deleteAppConnection
deleteAppConnection,
connectAppConnectionById,
listAvailableAppConnectionsForUser,
github: githubConnectionService(connectAppConnectionById)
};
};
@@ -4,7 +4,7 @@ import { randomUUID } from "crypto";
import { getConfig } from "@app/lib/config/env";
import { BadRequestError, InternalServerError } from "@app/lib/errors";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { AppConnection, AWSRegion } from "@app/services/app-connection/app-connection-enums";
import { AwsConnectionMethod } from "./aws-connection-enums";
import { TAwsConnectionConfig } from "./aws-connection-types";
@@ -20,7 +20,7 @@ export const getAwsAppConnectionListItem = () => {
};
};
export const getAwsConnectionConfig = async (appConnection: TAwsConnectionConfig, region = "us-east-1") => {
export const getAwsConnectionConfig = async (appConnection: TAwsConnectionConfig, region = AWSRegion.US_EAST_1) => {
const appCfg = getConfig();
let accessKeyId: string;
@@ -38,11 +38,11 @@ export const AwsConnectionSchema = z.intersection(
export const SanitizedAwsConnectionSchema = z.discriminatedUnion("method", [
BaseAwsConnectionSchema.extend({
method: z.literal(AwsConnectionMethod.AssumeRole),
credentials: AwsConnectionAssumeRoleCredentialsSchema.omit({ roleArn: true })
credentials: AwsConnectionAssumeRoleCredentialsSchema.pick({})
}),
BaseAwsConnectionSchema.extend({
method: z.literal(AwsConnectionMethod.AccessKey),
credentials: AwsConnectionAccessTokenCredentialsSchema.omit({ secretAccessKey: true })
credentials: AwsConnectionAccessTokenCredentialsSchema.pick({ accessKeyId: true })
})
]);
@@ -75,7 +75,7 @@ export const UpdateAwsConnectionSchema = z
export const AwsConnectionListItemSchema = z.object({
name: z.literal("AWS"),
app: z.literal(AppConnection.AWS),
// the below is preferable but currently breaks mintlify
// the below is preferable but currently breaks with our zod to json schema parser
// methods: z.tuple([z.literal(AwsConnectionMethod.AssumeRole), z.literal(AwsConnectionMethod.AccessKey)]),
methods: z.nativeEnum(AwsConnectionMethod).array(),
accessKeyId: z.string().optional()
@@ -1,3 +1,5 @@
import { createAppAuth } from "@octokit/auth-app";
import { Octokit } from "@octokit/rest";
import { AxiosResponse } from "axios";
import { getConfig } from "@app/lib/config/env";
@@ -8,7 +10,7 @@ import { IntegrationUrls } from "@app/services/integration-auth/integration-list
import { AppConnection } from "../app-connection-enums";
import { GitHubConnectionMethod } from "./github-connection-enums";
import { TGitHubConnectionConfig } from "./github-connection-types";
import { TGitHubConnection, TGitHubConnectionConfig } from "./github-connection-types";
export const getGitHubConnectionListItem = () => {
const { INF_APP_CONNECTION_GITHUB_OAUTH_CLIENT_ID, INF_APP_CONNECTION_GITHUB_APP_SLUG } = getConfig();
@@ -22,10 +24,131 @@ export const getGitHubConnectionListItem = () => {
};
};
export const getGitHubClient = (appConnection: TGitHubConnection) => {
const appCfg = getConfig();
const { method, credentials } = appConnection;
let client: Octokit;
switch (method) {
case GitHubConnectionMethod.App:
if (!appCfg.INF_APP_CONNECTION_GITHUB_APP_ID || !appCfg.INF_APP_CONNECTION_GITHUB_APP_PRIVATE_KEY) {
throw new InternalServerError({
message: `GitHub ${getAppConnectionMethodName(method).replace(
"GitHub",
""
)} environment variables have not been configured`
});
}
client = new Octokit({
authStrategy: createAppAuth,
auth: {
appId: appCfg.INF_APP_CONNECTION_GITHUB_APP_ID,
privateKey: appCfg.INF_APP_CONNECTION_GITHUB_APP_PRIVATE_KEY,
installationId: credentials.installationId
}
});
break;
case GitHubConnectionMethod.OAuth:
client = new Octokit({
auth: credentials.accessToken
});
break;
default:
throw new InternalServerError({
message: `Unhandled GitHub connection method: ${method as GitHubConnectionMethod}`
});
}
return client;
};
type GitHubOrganization = {
login: string;
id: number;
};
type GitHubRepository = {
id: number;
name: string;
owner: GitHubOrganization;
};
export const getGitHubRepositories = async (appConnection: TGitHubConnection) => {
const client = getGitHubClient(appConnection);
let repositories: GitHubRepository[];
switch (appConnection.method) {
case GitHubConnectionMethod.App:
repositories = await client.paginate("GET /installation/repositories");
break;
case GitHubConnectionMethod.OAuth:
default:
repositories = (await client.paginate("GET /user/repos")).filter((repo) => repo.permissions?.admin);
break;
}
return repositories;
};
export const getGitHubOrganizations = async (appConnection: TGitHubConnection) => {
const client = getGitHubClient(appConnection);
let organizations: GitHubOrganization[];
switch (appConnection.method) {
case GitHubConnectionMethod.App: {
const installationRepositories = await client.paginate("GET /installation/repositories");
const organizationMap: Record<string, GitHubOrganization> = {};
installationRepositories.forEach((repo) => {
if (repo.owner.type === "Organization") {
organizationMap[repo.owner.id] = repo.owner;
}
});
organizations = Object.values(organizationMap);
break;
}
case GitHubConnectionMethod.OAuth:
default:
organizations = await client.paginate("GET /user/orgs");
break;
}
return organizations;
};
export const getGitHubEnvironments = async (appConnection: TGitHubConnection, owner: string, repo: string) => {
const client = getGitHubClient(appConnection);
try {
const environments = await client.paginate("GET /repos/{owner}/{repo}/environments", {
owner,
repo
});
return environments;
} catch (e) {
// repo doesn't have envs
if ((e as { status: number }).status === 404) {
return [];
}
throw e;
}
};
type TokenRespData = {
access_token: string;
scope: string;
token_type: string;
error?: string;
};
export const validateGitHubConnectionCredentials = async (config: TGitHubConnectionConfig) => {
@@ -53,7 +176,10 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
if (!clientId || !clientSecret) {
throw new InternalServerError({
message: `GitHub ${getAppConnectionMethodName(method)} environment variables have not been configured`
message: `GitHub ${getAppConnectionMethodName(method).replace(
"GitHub",
""
)} environment variables have not been configured`
});
}
@@ -65,7 +191,7 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
client_id: clientId,
client_secret: clientSecret,
code: credentials.code,
redirect_uri: `${SITE_URL}/app-connections/github/oauth/callback`
redirect_uri: `${SITE_URL}/organization/app-connections/github/oauth/callback`
},
headers: {
Accept: "application/json",
@@ -90,6 +216,8 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
id: number;
account: {
login: string;
type: string;
id: number;
};
}[];
}>(IntegrationUrls.GITHUB_USER_INSTALLATIONS, {
@@ -111,10 +239,13 @@ export const validateGitHubConnectionCredentials = async (config: TGitHubConnect
}
}
if (!tokenResp.data.access_token) {
throw new InternalServerError({ message: `Missing access token: ${tokenResp.data.error}` });
}
switch (method) {
case GitHubConnectionMethod.App:
return {
// access token not needed for GitHub App
installationId: credentials.installationId
};
case GitHubConnectionMethod.OAuth:
@@ -57,7 +57,7 @@ export const UpdateGitHubConnectionSchema = z
const BaseGitHubConnectionSchema = BaseAppConnectionSchema.extend({ app: z.literal(AppConnection.GitHub) });
export const GitHubAppConnectionSchema = z.intersection(
export const GitHubConnectionSchema = z.intersection(
BaseGitHubConnectionSchema,
z.discriminatedUnion("method", [
z.object({
@@ -74,19 +74,19 @@ export const GitHubAppConnectionSchema = z.intersection(
export const SanitizedGitHubConnectionSchema = z.discriminatedUnion("method", [
BaseGitHubConnectionSchema.extend({
method: z.literal(GitHubConnectionMethod.App),
credentials: GitHubConnectionAppOutputCredentialsSchema.omit({ installationId: true })
credentials: GitHubConnectionAppOutputCredentialsSchema.pick({})
}),
BaseGitHubConnectionSchema.extend({
method: z.literal(GitHubConnectionMethod.OAuth),
credentials: GitHubConnectionOAuthOutputCredentialsSchema.omit({ accessToken: true })
credentials: GitHubConnectionOAuthOutputCredentialsSchema.pick({})
})
]);
export const GitHubConnectionListItemSchema = z.object({
name: z.literal("GitHub"),
app: z.literal(AppConnection.GitHub),
// the below is preferable but currently breaks mintlify
// methods: z.tuple([z.literal(GitHubConnectionMethod.GitHubApp), z.literal(GitHubConnectionMethod.OAuth)]),
// the below is preferable but currently breaks with our zod to json schema parser
// methods: z.tuple([z.literal(GitHubConnectionMethod.App), z.literal(GitHubConnectionMethod.OAuth)]),
methods: z.nativeEnum(GitHubConnectionMethod).array(),
oauthClientId: z.string().optional(),
appClientSlug: z.string().optional()
@@ -0,0 +1,55 @@
import { OrgServiceActor } from "@app/lib/types";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import {
getGitHubEnvironments,
getGitHubOrganizations,
getGitHubRepositories
} from "@app/services/app-connection/github/github-connection-fns";
import { TGitHubConnection } from "@app/services/app-connection/github/github-connection-types";
type TGetAppConnectionFunc = (
app: AppConnection,
connectionId: string,
actor: OrgServiceActor
) => Promise<TGitHubConnection>;
type TListGitHubEnvironmentsDTO = {
connectionId: string;
repo: string;
owner: string;
};
export const githubConnectionService = (getAppConnection: TGetAppConnectionFunc) => {
const listRepositories = async (connectionId: string, actor: OrgServiceActor) => {
const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);
const repositories = await getGitHubRepositories(appConnection);
return repositories;
};
const listOrganizations = async (connectionId: string, actor: OrgServiceActor) => {
const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);
const organizations = await getGitHubOrganizations(appConnection);
return organizations;
};
const listEnvironments = async (
{ connectionId, repo, owner }: TListGitHubEnvironmentsDTO,
actor: OrgServiceActor
) => {
const appConnection = await getAppConnection(AppConnection.GitHub, connectionId, actor);
const environments = await getGitHubEnvironments(appConnection, owner, repo);
return environments;
};
return {
listRepositories,
listOrganizations,
listEnvironments
};
};
@@ -5,11 +5,11 @@ import { DiscriminativePick } from "@app/lib/types";
import { AppConnection } from "../app-connection-enums";
import {
CreateGitHubConnectionSchema,
GitHubAppConnectionSchema,
GitHubConnectionSchema,
ValidateGitHubConnectionCredentialsSchema
} from "./github-connection-schemas";
export type TGitHubConnection = z.infer<typeof GitHubAppConnectionSchema>;
export type TGitHubConnection = z.infer<typeof GitHubConnectionSchema>;
export type TGitHubConnectionInput = z.infer<typeof CreateGitHubConnectionSchema> & {
app: AppConnection.GitHub;
@@ -0,0 +1,10 @@
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import { TSecretSyncListItem } from "@app/services/secret-sync/secret-sync-types";
export const AWS_PARAMETER_STORE_SYNC_LIST_OPTION: TSecretSyncListItem = {
name: "AWS Parameter Store",
destination: SecretSync.AWSParameterStore,
connection: AppConnection.AWS,
canImportSecrets: true
};
@@ -0,0 +1,199 @@
import AWS, { AWSError } from "aws-sdk";
import { getAwsConnectionConfig } from "@app/services/app-connection/aws/aws-connection-fns";
import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
import { TAwsParameterStoreSyncWithCredentials } from "./aws-parameter-store-sync-types";
type TAWSParameterStoreRecord = Record<string, AWS.SSM.Parameter>;
const MAX_RETRIES = 5;
const BATCH_SIZE = 10;
const getSSM = async (secretSync: TAwsParameterStoreSyncWithCredentials) => {
const { destinationConfig, connection } = secretSync;
const config = await getAwsConnectionConfig(connection, destinationConfig.region);
const ssm = new AWS.SSM({
apiVersion: "2014-11-06",
region: destinationConfig.region
});
ssm.config.update(config);
return ssm;
};
const sleep = async () =>
new Promise((resolve) => {
setTimeout(resolve, 1000);
});
const getParametersByPath = async (ssm: AWS.SSM, path: string): Promise<TAWSParameterStoreRecord> => {
const awsParameterStoreSecretsRecord: TAWSParameterStoreRecord = {};
let hasNext = true;
let nextToken: string | undefined;
let attempt = 0;
while (hasNext) {
try {
// eslint-disable-next-line no-await-in-loop
const parameters = await ssm
.getParametersByPath({
Path: path,
Recursive: false,
WithDecryption: true,
MaxResults: BATCH_SIZE,
NextToken: nextToken
})
.promise();
attempt = 0;
if (parameters.Parameters) {
parameters.Parameters.forEach((parameter) => {
if (parameter.Name) {
// no leading slash if path is '/'
const secKey = path.length > 1 ? parameter.Name.substring(path.length) : parameter.Name;
awsParameterStoreSecretsRecord[secKey] = parameter;
}
});
}
hasNext = Boolean(parameters.NextToken);
nextToken = parameters.NextToken;
} catch (e) {
if ((e as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) {
attempt += 1;
// eslint-disable-next-line no-await-in-loop
await sleep();
}
throw e;
}
}
return awsParameterStoreSecretsRecord;
};
const putParameter = async (
ssm: AWS.SSM,
params: AWS.SSM.PutParameterRequest,
attempt = 0
): Promise<AWS.SSM.PutParameterResult> => {
try {
return await ssm.putParameter(params).promise();
} catch (error) {
if ((error as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) {
await sleep();
// retry
return putParameter(ssm, params, attempt + 1);
}
throw error;
}
};
const deleteParametersBatch = async (
ssm: AWS.SSM,
parameters: AWS.SSM.Parameter[],
attempt = 0
): Promise<AWS.SSM.DeleteParameterResult[]> => {
const results: AWS.SSM.DeleteParameterResult[] = [];
let remainingParams = [...parameters];
while (remainingParams.length > 0) {
const batch = remainingParams.slice(0, BATCH_SIZE);
try {
// eslint-disable-next-line no-await-in-loop
const result = await ssm.deleteParameters({ Names: batch.map((param) => param.Name!) }).promise();
results.push(result);
remainingParams = remainingParams.slice(BATCH_SIZE);
} catch (error) {
if ((error as AWSError).code === "ThrottlingException" && attempt < MAX_RETRIES) {
// eslint-disable-next-line no-await-in-loop
await sleep();
// Retry the current batch
// eslint-disable-next-line no-await-in-loop
return [...results, ...(await deleteParametersBatch(ssm, remainingParams, attempt + 1))];
}
throw error;
}
}
return results;
};
export const AwsParameterStoreSyncFns = {
syncSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials, affixedSecretMap: TSecretMap) => {
const { destinationConfig } = secretSync;
const ssm = await getSSM(secretSync);
// TODO(scott): KMS Key ID, Tags
const awsParameterStoreSecretsRecord = await getParametersByPath(ssm, destinationConfig.path);
for await (const entry of Object.entries(affixedSecretMap)) {
const [key, { value }] = entry;
// skip empty values (not allowed by AWS) or secrets that haven't changed
if (!value || (key in awsParameterStoreSecretsRecord && awsParameterStoreSecretsRecord[key].Value === value)) {
// eslint-disable-next-line no-continue
continue;
}
await putParameter(ssm, {
Name: `${destinationConfig.path}${key}`,
Type: "SecureString",
Value: value,
Overwrite: true
});
}
const parametersToDelete: AWS.SSM.Parameter[] = [];
for (const entry of Object.entries(awsParameterStoreSecretsRecord)) {
const [key, parameter] = entry;
if (!(key in affixedSecretMap) || !affixedSecretMap[key].value) {
parametersToDelete.push(parameter);
}
}
await deleteParametersBatch(ssm, parametersToDelete);
},
importSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials): Promise<TSecretMap> => {
const { destinationConfig } = secretSync;
const ssm = await getSSM(secretSync);
const awsParameterStoreSecretsRecord = await getParametersByPath(ssm, destinationConfig.path);
return Object.fromEntries(
Object.entries(awsParameterStoreSecretsRecord).map(([key, value]) => [key, { value: value.Value ?? "" }])
);
},
removeSecrets: async (secretSync: TAwsParameterStoreSyncWithCredentials, affixedSecretMap: TSecretMap) => {
const { destinationConfig } = secretSync;
const ssm = await getSSM(secretSync);
const awsParameterStoreSecretsRecord = await getParametersByPath(ssm, destinationConfig.path);
const parametersToDelete: AWS.SSM.Parameter[] = [];
for (const entry of Object.entries(awsParameterStoreSecretsRecord)) {
const [key, param] = entry;
if (key in affixedSecretMap) {
parametersToDelete.push(param);
}
}
await deleteParametersBatch(ssm, parametersToDelete);
}
};
@@ -0,0 +1,59 @@
import { z } from "zod";
import { SecretSyncs } from "@app/lib/api-docs";
import { wrapWithSlashes } from "@app/lib/fn";
import { AppConnection, AWSRegion } from "@app/services/app-connection/app-connection-enums";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import {
BaseSecretSyncSchema,
GenericCreateSecretSyncFieldsSchema,
GenericUpdateSecretSyncFieldsSchema
} from "@app/services/secret-sync/secret-sync-schemas";
const AwsParameterStoreSyncDestinationConfigSchema = z.object({
region: z.nativeEnum(AWSRegion).describe(SecretSyncs.DESTINATION_CONFIG.AWS_PARAMETER_STORE.REGION),
path: z
.string()
.min(1, "Parameter Store Path Required")
.transform(wrapWithSlashes)
.superRefine((val, ctx) => {
if (!/^\/([/]|(([\w-]+\/)+))?$/.test(val)) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: `Invalid Parameter Store Path - must follow "/example/path/" format`
});
}
if (val.length > 2048) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: `Invalid Parameter Store Path - cannot exceed 2048 characters`
});
}
})
.describe(SecretSyncs.DESTINATION_CONFIG.AWS_PARAMETER_STORE.PATH)
});
export const AwsParameterStoreSyncSchema = BaseSecretSyncSchema(SecretSync.AWSParameterStore).extend({
destination: z.literal(SecretSync.AWSParameterStore),
destinationConfig: AwsParameterStoreSyncDestinationConfigSchema
});
export const CreateAwsParameterStoreSyncSchema = GenericCreateSecretSyncFieldsSchema(
SecretSync.AWSParameterStore
).extend({
destinationConfig: AwsParameterStoreSyncDestinationConfigSchema
});
export const UpdateAwsParameterStoreSyncSchema = GenericUpdateSecretSyncFieldsSchema(
SecretSync.AWSParameterStore
).extend({
destinationConfig: AwsParameterStoreSyncDestinationConfigSchema.optional()
});
export const AwsParameterStoreSyncListItemSchema = z.object({
name: z.literal("AWS Parameter Store"),
connection: z.literal(AppConnection.AWS),
destination: z.literal(SecretSync.AWSParameterStore),
canImportSecrets: z.literal(true)
});
@@ -0,0 +1,19 @@
import { z } from "zod";
import { TAwsConnection } from "@app/services/app-connection/aws";
import {
AwsParameterStoreSyncListItemSchema,
AwsParameterStoreSyncSchema,
CreateAwsParameterStoreSyncSchema
} from "./aws-parameter-store-sync-schemas";
export type TAwsParameterStoreSync = z.infer<typeof AwsParameterStoreSyncSchema>;
export type TAwsParameterStoreSyncInput = z.infer<typeof CreateAwsParameterStoreSyncSchema>;
export type TAwsParameterStoreSyncListItem = z.infer<typeof AwsParameterStoreSyncListItemSchema>;
export type TAwsParameterStoreSyncWithCredentials = TAwsParameterStoreSync & {
connection: TAwsConnection;
};
@@ -0,0 +1,4 @@
export * from "./aws-parameter-store-sync-constants";
export * from "./aws-parameter-store-sync-fns";
export * from "./aws-parameter-store-sync-schemas";
export * from "./aws-parameter-store-sync-types";
@@ -0,0 +1,10 @@
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import { TSecretSyncListItem } from "@app/services/secret-sync/secret-sync-types";
export const GITHUB_SYNC_LIST_OPTION: TSecretSyncListItem = {
name: "GitHub",
destination: SecretSync.GitHub,
connection: AppConnection.GitHub,
canImportSecrets: false
};
@@ -0,0 +1,11 @@
export enum GitHubSyncScope {
Repository = "repository",
Organization = "organization",
RepositoryEnvironment = "repository-environment"
}
export enum GitHubSyncVisibility {
All = "all",
Private = "private",
Selected = "selected"
}
@@ -0,0 +1,225 @@
import { Octokit } from "@octokit/rest";
import sodium from "libsodium-wrappers";
import { getGitHubClient } from "@app/services/app-connection/github";
import { GitHubSyncScope } from "@app/services/secret-sync/github/github-sync-enums";
import { SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";
import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
import { TGitHubSyncWithCredentials } from "./github-sync-types";
interface GitHubSecret {
name: string;
created_at: string;
updated_at: string;
visibility?: "all" | "private" | "selected";
selected_repositories_url?: string | undefined;
}
// TODO: rate limit handling
const getEncryptedSecrets = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => {
let encryptedSecrets: GitHubSecret[];
const { destinationConfig } = secretSync;
switch (destinationConfig.scope) {
case GitHubSyncScope.Organization: {
encryptedSecrets = await client.paginate("GET /orgs/{org}/actions/secrets", {
org: destinationConfig.org
});
break;
}
case GitHubSyncScope.Repository: {
encryptedSecrets = await client.paginate("GET /repos/{owner}/{repo}/actions/secrets", {
owner: destinationConfig.owner,
repo: destinationConfig.repo
});
break;
}
case GitHubSyncScope.RepositoryEnvironment:
default: {
encryptedSecrets = await client.paginate("GET /repos/{owner}/{repo}/environments/{environment_name}/secrets", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
environment_name: destinationConfig.env
});
break;
}
}
return encryptedSecrets;
};
interface GitHubPublicKey {
key_id: string;
key: string;
id?: number | undefined;
url?: string | undefined;
title?: string | undefined;
created_at?: string | undefined;
}
const getPublicKey = async (client: Octokit, secretSync: TGitHubSyncWithCredentials) => {
let publicKey: GitHubPublicKey;
const { destinationConfig } = secretSync;
switch (destinationConfig.scope) {
case GitHubSyncScope.Organization: {
publicKey = (
await client.request("GET /orgs/{org}/actions/secrets/public-key", {
org: destinationConfig.org
})
).data;
break;
}
case GitHubSyncScope.Repository: {
publicKey = (
await client.request("GET /repos/{owner}/{repo}/actions/secrets/public-key", {
owner: destinationConfig.owner,
repo: destinationConfig.repo
})
).data;
break;
}
case GitHubSyncScope.RepositoryEnvironment:
default: {
publicKey = (
await client.request("GET /repos/{owner}/{repo}/environments/{environment_name}/secrets/public-key", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
environment_name: destinationConfig.env
})
).data;
break;
}
}
return publicKey;
};
const deleteSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredentials, encryptedSecret: GitHubSecret) => {
const { destinationConfig } = secretSync;
switch (destinationConfig.scope) {
case GitHubSyncScope.Organization: {
await client.request(`DELETE /orgs/{org}/actions/secrets/{secret_name}`, {
org: destinationConfig.org,
secret_name: encryptedSecret.name
});
break;
}
case GitHubSyncScope.Repository: {
await client.request("DELETE /repos/{owner}/{repo}/actions/secrets/{secret_name}", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
secret_name: encryptedSecret.name
});
break;
}
case GitHubSyncScope.RepositoryEnvironment:
default: {
await client.request("DELETE /repos/{owner}/{repo}/environments/{environment_name}/secrets/{secret_name}", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
environment_name: destinationConfig.env,
secret_name: encryptedSecret.name
});
break;
}
}
};
interface GitHubSecretPayload {
key_id: string;
secret_name: string;
encrypted_value: string;
}
const putSecret = async (client: Octokit, secretSync: TGitHubSyncWithCredentials, payload: GitHubSecretPayload) => {
const { destinationConfig } = secretSync;
switch (destinationConfig.scope) {
case GitHubSyncScope.Organization: {
const { visibility, selectedRepositoryIds } = destinationConfig;
await client.request(`PUT /orgs/{org}/actions/secrets/{secret_name}`, {
org: destinationConfig.org,
...payload,
visibility,
selected_repository_ids: selectedRepositoryIds
});
break;
}
case GitHubSyncScope.Repository: {
await client.request("PUT /repos/{owner}/{repo}/actions/secrets/{secret_name}", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
...payload
});
break;
}
case GitHubSyncScope.RepositoryEnvironment:
default: {
await client.request("PUT /repos/{owner}/{repo}/environments/{environment_name}/secrets/{secret_name}", {
owner: destinationConfig.owner,
repo: destinationConfig.repo,
environment_name: destinationConfig.env,
...payload
});
break;
}
}
};
export const GithubSyncFns = {
syncSecrets: async (secretSync: TGitHubSyncWithCredentials, affixedSecretMap: TSecretMap) => {
const client = getGitHubClient(secretSync.connection);
const encryptedSecrets = await getEncryptedSecrets(client, secretSync);
const publicKey = await getPublicKey(client, secretSync);
for await (const encryptedSecret of encryptedSecrets) {
if (!(encryptedSecret.name in affixedSecretMap)) {
await deleteSecret(client, secretSync, encryptedSecret);
}
}
await sodium.ready.then(async () => {
for await (const key of Object.keys(affixedSecretMap)) {
// convert secret & base64 key to Uint8Array.
const binaryKey = sodium.from_base64(publicKey.key, sodium.base64_variants.ORIGINAL);
const binarySecretValue = sodium.from_string(affixedSecretMap[key].value);
// encrypt secret using libsodium
const encryptedBytes = sodium.crypto_box_seal(binarySecretValue, binaryKey);
// convert encrypted Uint8Array to base64
const encryptedSecretValue = sodium.to_base64(encryptedBytes, sodium.base64_variants.ORIGINAL);
await putSecret(client, secretSync, {
secret_name: key,
encrypted_value: encryptedSecretValue,
key_id: publicKey.key_id
});
}
});
},
importSecrets: async (secretSync: TGitHubSyncWithCredentials) => {
throw new Error(`${SECRET_SYNC_NAME_MAP[secretSync.destination]} does not support importing secrets.`);
},
removeSecrets: async (secretSync: TGitHubSyncWithCredentials, affixedSecretMap: TSecretMap) => {
const client = getGitHubClient(secretSync.connection);
const encryptedSecrets = await getEncryptedSecrets(client, secretSync);
for await (const encryptedSecret of encryptedSecrets) {
if (encryptedSecret.name in affixedSecretMap) {
await deleteSecret(client, secretSync, encryptedSecret);
}
}
}
};
@@ -0,0 +1,72 @@
import { z } from "zod";
import { SecretSyncs } from "@app/lib/api-docs";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { GitHubSyncScope, GitHubSyncVisibility } from "@app/services/secret-sync/github/github-sync-enums";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import {
BaseSecretSyncSchema,
GenericCreateSecretSyncFieldsSchema,
GenericUpdateSecretSyncFieldsSchema
} from "@app/services/secret-sync/secret-sync-schemas";
import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types";
const GitHubSyncDestinationConfigSchema = z
.discriminatedUnion("scope", [
z.object({
scope: z.literal(GitHubSyncScope.Organization),
org: z.string().min(1, "Organization name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.ORG),
visibility: z.nativeEnum(GitHubSyncVisibility),
selectedRepositoryIds: z.number().array().optional()
}),
z.object({
scope: z.literal(GitHubSyncScope.Repository),
owner: z.string().min(1, "Repository owner name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.OWNER),
repo: z.string().min(1, "Repository name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.REPO)
}),
z.object({
scope: z.literal(GitHubSyncScope.RepositoryEnvironment),
owner: z.string().min(1, "Repository owner name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.OWNER),
repo: z.string().min(1, "Repository name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.REPO),
env: z.string().min(1, "Environment name required").describe(SecretSyncs.DESTINATION_CONFIG.GITHUB.ENV)
})
])
.superRefine((options, ctx) => {
if (options.scope !== GitHubSyncScope.Organization) return;
if (options.visibility === GitHubSyncVisibility.Selected && !options.selectedRepositoryIds?.length) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: "Select at least 1 repository",
path: ["selectedRepositoryIds"]
});
}
});
const GitHubSyncOptionsConfig: TSyncOptionsConfig = { canImportSecrets: false };
export const GitHubSyncSchema = BaseSecretSyncSchema(SecretSync.GitHub, GitHubSyncOptionsConfig).extend({
destination: z.literal(SecretSync.GitHub),
destinationConfig: GitHubSyncDestinationConfigSchema
});
export const CreateGitHubSyncSchema = GenericCreateSecretSyncFieldsSchema(
SecretSync.GitHub,
GitHubSyncOptionsConfig
).extend({
destinationConfig: GitHubSyncDestinationConfigSchema
});
export const UpdateGitHubSyncSchema = GenericUpdateSecretSyncFieldsSchema(
SecretSync.GitHub,
GitHubSyncOptionsConfig
).extend({
destinationConfig: GitHubSyncDestinationConfigSchema.optional()
});
export const GitHubSyncListItemSchema = z.object({
name: z.literal("GitHub"),
connection: z.literal(AppConnection.GitHub),
destination: z.literal(SecretSync.GitHub),
canImportSecrets: z.literal(false)
});
@@ -0,0 +1,15 @@
import { z } from "zod";
import { TGitHubConnection } from "@app/services/app-connection/github";
import { CreateGitHubSyncSchema, GitHubSyncListItemSchema, GitHubSyncSchema } from "./github-sync-schemas";
export type TGitHubSync = z.infer<typeof GitHubSyncSchema>;
export type TGitHubSyncInput = z.infer<typeof CreateGitHubSyncSchema>;
export type TGitHubSyncListItem = z.infer<typeof GitHubSyncListItemSchema>;
export type TGitHubSyncWithCredentials = TGitHubSync & {
connection: TGitHubConnection;
};
@@ -0,0 +1,4 @@
export * from "./github-sync-constants";
export * from "./github-sync-fns";
export * from "./github-sync-schemas";
export * from "./github-sync-types";
@@ -0,0 +1,201 @@
import { Knex } from "knex";
import { TDbClient } from "@app/db";
import { TableName } from "@app/db/schemas";
import { TSecretSyncs } from "@app/db/schemas/secret-syncs";
import { DatabaseError } from "@app/lib/errors";
import { buildFindFilter, ormify, selectAllTableCols } from "@app/lib/knex";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
export type TSecretSyncDALFactory = ReturnType<typeof secretSyncDALFactory>;
type SecretSyncFindFilter = Parameters<typeof buildFindFilter<TSecretSyncs>>[0];
const baseSecretSyncQuery = ({ filter, db, tx }: { db: TDbClient; filter?: SecretSyncFindFilter; tx?: Knex }) => {
const query = (tx || db.replicaNode())(TableName.SecretSync)
.join(TableName.SecretFolder, `${TableName.SecretSync}.folderId`, `${TableName.SecretFolder}.id`)
.join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
.join(TableName.AppConnection, `${TableName.SecretSync}.connectionId`, `${TableName.AppConnection}.id`)
.select(selectAllTableCols(TableName.SecretSync))
.select(
// evironment
db.ref("name").withSchema(TableName.Environment).as("envName"),
db.ref("id").withSchema(TableName.Environment).as("envId"),
db.ref("slug").withSchema(TableName.Environment).as("envSlug"),
db.ref("projectId").withSchema(TableName.Environment),
// entire connection
db.ref("name").withSchema(TableName.AppConnection).as("connectionName"),
db.ref("method").withSchema(TableName.AppConnection).as("connectionMethod"),
db.ref("app").withSchema(TableName.AppConnection).as("connectionApp"),
db.ref("orgId").withSchema(TableName.AppConnection).as("connectionOrgId"),
db.ref("encryptedCredentials").withSchema(TableName.AppConnection).as("connectionEncryptedCredentials"),
db.ref("description").withSchema(TableName.AppConnection).as("connectionDescription"),
db.ref("version").withSchema(TableName.AppConnection).as("connectionVersion"),
db.ref("createdAt").withSchema(TableName.AppConnection).as("connectionCreatedAt"),
db.ref("updatedAt").withSchema(TableName.AppConnection).as("connectionUpdatedAt")
);
// prepends table name to filter keys to avoid ambiguous col references, skipping utility filters like $in, etc.
const prependTableName = (filterObj: object): SecretSyncFindFilter =>
Object.fromEntries(
Object.entries(filterObj).map(([key, value]) =>
key.startsWith("$") ? [key, prependTableName(value as object)] : [`${TableName.SecretSync}.${key}`, value]
)
);
if (filter) {
/* eslint-disable @typescript-eslint/no-misused-promises */
void query.where(buildFindFilter(prependTableName(filter)));
}
return query;
};
const expandSecretSync = (
secretSync: Awaited<ReturnType<typeof baseSecretSyncQuery>>[number],
folder: Awaited<ReturnType<TSecretFolderDALFactory["findSecretPathByFolderIds"]>>[number]
) => {
const {
envId,
envName,
envSlug,
connectionApp,
connectionName,
connectionId,
connectionOrgId,
connectionEncryptedCredentials,
connectionMethod,
connectionDescription,
connectionCreatedAt,
connectionUpdatedAt,
connectionVersion,
...el
} = secretSync;
return {
...el,
connectionId,
environment: { id: envId, name: envName, slug: envSlug },
connection: {
app: connectionApp,
id: connectionId,
name: connectionName,
orgId: connectionOrgId,
encryptedCredentials: connectionEncryptedCredentials,
method: connectionMethod,
description: connectionDescription,
createdAt: connectionCreatedAt,
updatedAt: connectionUpdatedAt,
version: connectionVersion
},
folder: {
id: folder!.id,
path: folder!.path
}
};
};
export const secretSyncDALFactory = (
db: TDbClient,
folderDAL: Pick<TSecretFolderDALFactory, "findSecretPathByFolderIds">
) => {
const secretSyncOrm = ormify(db, TableName.SecretSync);
const findById = async (id: string, tx?: Knex) => {
try {
const secretSync = await baseSecretSyncQuery({
filter: { id },
db,
tx
}).first();
if (secretSync) {
// TODO (scott): replace with cached folder path once implemented
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]);
return expandSecretSync(secretSync, folderWithPath);
}
} catch (error) {
throw new DatabaseError({ error, name: "Find by ID - Secret Sync" });
}
};
const create = async (data: Parameters<(typeof secretSyncOrm)["create"]>[0]) => {
try {
const secretSync = (await secretSyncOrm.transaction(async (tx) => {
const sync = await secretSyncOrm.create(data, tx);
return baseSecretSyncQuery({
filter: { id: sync.id },
db,
tx
}).first();
}))!;
// TODO (scott): replace with cached folder path once implemented
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]);
return expandSecretSync(secretSync, folderWithPath);
} catch (error) {
throw new DatabaseError({ error, name: "Create - Secret Sync" });
}
};
const updateById = async (syncId: string, data: Parameters<(typeof secretSyncOrm)["updateById"]>[1]) => {
try {
const secretSync = (await secretSyncOrm.transaction(async (tx) => {
const sync = await secretSyncOrm.updateById(syncId, data, tx);
return baseSecretSyncQuery({
filter: { id: sync.id },
db,
tx
}).first();
}))!;
// TODO (scott): replace with cached folder path once implemented
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]);
return expandSecretSync(secretSync, folderWithPath);
} catch (error) {
throw new DatabaseError({ error, name: "Update by ID - Secret Sync" });
}
};
const findOne = async (filter: Parameters<(typeof secretSyncOrm)["findOne"]>[0], tx?: Knex) => {
try {
const secretSync = await baseSecretSyncQuery({ filter, db, tx }).first();
if (secretSync) {
// TODO (scott): replace with cached folder path once implemented
const [folderWithPath] = await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId]);
return expandSecretSync(secretSync, folderWithPath);
}
} catch (error) {
throw new DatabaseError({ error, name: "Find One - Secret Sync" });
}
};
const find = async (filter: Parameters<(typeof secretSyncOrm)["find"]>[0], tx?: Knex) => {
try {
const secretSyncs = await baseSecretSyncQuery({ filter, db, tx });
if (!secretSyncs.length) return [];
const foldersWithPath = await folderDAL.findSecretPathByFolderIds(
secretSyncs[0].projectId,
secretSyncs.map((sync) => sync.folderId)
);
// TODO (scott): replace with cached folder path once implemented
const folderRecord: Record<string, (typeof foldersWithPath)[number]> = {};
foldersWithPath.forEach((folder) => {
if (folder) folderRecord[folder.id] = folder;
});
return secretSyncs.map((secretSync) => expandSecretSync(secretSync, folderRecord[secretSync.folderId]));
} catch (error) {
throw new DatabaseError({ error, name: "Find - Secret Sync" });
}
};
return { ...secretSyncOrm, findById, findOne, find, create, updateById };
};
@@ -0,0 +1,15 @@
export enum SecretSync {
AWSParameterStore = "aws-parameter-store",
GitHub = "github"
}
export enum SecretSyncInitialSyncBehavior {
OverwriteDestination = "overwrite-destination",
ImportPrioritizeSource = "import-prioritize-source",
ImportPrioritizeDestination = "import-prioritize-destination"
}
export enum SecretSyncImportBehavior {
PrioritizeSource = "prioritize-source",
PrioritizeDestination = "prioritize-destination"
}
@@ -0,0 +1,110 @@
import {
AWS_PARAMETER_STORE_SYNC_LIST_OPTION,
AwsParameterStoreSyncFns
} from "@app/services/secret-sync/aws-parameter-store";
import { GITHUB_SYNC_LIST_OPTION, GithubSyncFns } from "@app/services/secret-sync/github";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import {
TSecretMap,
TSecretSyncListItem,
TSecretSyncWithCredentials
} from "@app/services/secret-sync/secret-sync-types";
const SECRET_SYNC_LIST_OPTIONS: Record<SecretSync, TSecretSyncListItem> = {
[SecretSync.AWSParameterStore]: AWS_PARAMETER_STORE_SYNC_LIST_OPTION,
[SecretSync.GitHub]: GITHUB_SYNC_LIST_OPTION
};
export const listSecretSyncOptions = () => {
return Object.values(SECRET_SYNC_LIST_OPTIONS).sort((a, b) => a.name.localeCompare(b.name));
};
const addAffixes = (secretSync: TSecretSyncWithCredentials, unprocessedSecretMap: TSecretMap) => {
let secretMap = { ...unprocessedSecretMap };
const { appendSuffix, prependPrefix } = secretSync.syncOptions;
if (appendSuffix || prependPrefix) {
secretMap = {};
Object.entries(unprocessedSecretMap).forEach(([key, value]) => {
secretMap[`${prependPrefix || ""}${key}${appendSuffix || ""}`] = value;
});
}
return secretMap;
};
const stripAffixes = (secretSync: TSecretSyncWithCredentials, unprocessedSecretMap: TSecretMap) => {
let secretMap = { ...unprocessedSecretMap };
const { appendSuffix, prependPrefix } = secretSync.syncOptions;
if (appendSuffix || prependPrefix) {
secretMap = {};
Object.entries(unprocessedSecretMap).forEach(([key, value]) => {
let processedKey = key;
if (prependPrefix && processedKey.startsWith(prependPrefix)) {
processedKey = processedKey.slice(prependPrefix.length);
}
if (appendSuffix && processedKey.endsWith(appendSuffix)) {
processedKey = processedKey.slice(0, -appendSuffix.length);
}
secretMap[processedKey] = value;
});
}
return secretMap;
};
// TODO(scott): ideally do this in a map to reduce code but requires typescript trickery...
export const SecretSyncFns = {
syncSecrets: (secretSync: TSecretSyncWithCredentials, secretMap: TSecretMap): Promise<void> => {
const affixedSecretMap = addAffixes(secretSync, secretMap);
switch (secretSync.destination) {
case SecretSync.AWSParameterStore:
return AwsParameterStoreSyncFns.syncSecrets(secretSync, affixedSecretMap);
case SecretSync.GitHub:
return GithubSyncFns.syncSecrets(secretSync, affixedSecretMap);
default:
throw new Error(
`Unhandled sync destination for push secrets: ${(secretSync as TSecretSyncWithCredentials).destination}`
);
}
},
importSecrets: async (secretSync: TSecretSyncWithCredentials): Promise<TSecretMap> => {
let secretMap: TSecretMap;
switch (secretSync.destination) {
case SecretSync.AWSParameterStore:
secretMap = await AwsParameterStoreSyncFns.importSecrets(secretSync);
break;
case SecretSync.GitHub:
secretMap = await GithubSyncFns.importSecrets(secretSync);
break;
default:
throw new Error(
`Unhandled sync destination for push secrets: ${(secretSync as TSecretSyncWithCredentials).destination}`
);
}
return stripAffixes(secretSync, secretMap);
},
removeSecrets: (secretSync: TSecretSyncWithCredentials, secretMap: TSecretMap): Promise<void> => {
const affixedSecretMap = addAffixes(secretSync, secretMap);
switch (secretSync.destination) {
case SecretSync.AWSParameterStore:
return AwsParameterStoreSyncFns.removeSecrets(secretSync, affixedSecretMap);
case SecretSync.GitHub:
return GithubSyncFns.removeSecrets(secretSync, affixedSecretMap);
default:
throw new Error(
`Unhandled sync destination for removing secrets: ${(secretSync as TSecretSyncWithCredentials).destination}`
);
}
}
};
@@ -0,0 +1,12 @@
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
export const SECRET_SYNC_NAME_MAP: Record<SecretSync, string> = {
[SecretSync.AWSParameterStore]: "AWS Parameter Store",
[SecretSync.GitHub]: "GitHub"
};
export const SECRET_SYNC_CONNECTION_MAP: Record<SecretSync, AppConnection> = {
[SecretSync.AWSParameterStore]: AppConnection.AWS,
[SecretSync.GitHub]: AppConnection.GitHub
};
@@ -0,0 +1,939 @@
import opentelemetry from "@opentelemetry/api";
import { AxiosError } from "axios";
import { Job } from "bullmq";
import { ProjectMembershipRole, SecretType } from "@app/db/schemas";
import { TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-service";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { decryptAppConnectionCredentials } from "@app/services/app-connection/app-connection-fns";
import { ActorType } from "@app/services/auth/auth-type";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { KmsDataKey } from "@app/services/kms/kms-types";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal";
import { TProjectMembershipDALFactory } from "@app/services/project-membership/project-membership-dal";
import { TResourceMetadataDALFactory } from "@app/services/resource-metadata/resource-metadata-dal";
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
import { createManySecretsRawFnFactory, updateManySecretsRawFnFactory } from "@app/services/secret/secret-fns";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal";
import { fnSecretsV2FromImports } from "@app/services/secret-import/secret-import-fns";
import { TSecretSyncDALFactory } from "@app/services/secret-sync/secret-sync-dal";
import {
SecretSync,
SecretSyncImportBehavior,
SecretSyncInitialSyncBehavior
} from "@app/services/secret-sync/secret-sync-enums";
import { SecretSyncFns } from "@app/services/secret-sync/secret-sync-fns";
import { SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";
import {
SecretSyncAction,
SecretSyncStatus,
TQueueSecretSyncImportSecretsByIdDTO,
TQueueSecretSyncRemoveSecretsByIdDTO,
TQueueSecretSyncsByPathDTO,
TQueueSecretSyncSyncSecretsByIdDTO,
TQueueSendSecretSyncActionFailedNotificationsDTO,
TSecretMap,
TSecretSyncImportSecretsDTO,
TSecretSyncRaw,
TSecretSyncRemoveSecretsDTO,
TSecretSyncSyncSecretsDTO,
TSecretSyncWithCredentials,
TSendSecretSyncFailedNotificationsJobDTO
} from "@app/services/secret-sync/secret-sync-types";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";
import { expandSecretReferencesFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-fns";
import { TSecretVersionV2DALFactory } from "@app/services/secret-v2-bridge/secret-version-dal";
import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal";
import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
export type TSecretSyncQueueFactory = ReturnType<typeof secretSyncQueueFactory>;
type TSecretSyncQueueFactoryDep = {
queueService: Pick<TQueueServiceFactory, "queue" | "start">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
keyStore: Pick<TKeyStoreFactory, "acquireLock" | "setItemWithExpiry" | "getItem">;
folderDAL: TSecretFolderDALFactory;
secretV2BridgeDAL: Pick<
TSecretV2BridgeDALFactory,
| "findByFolderId"
| "find"
| "insertMany"
| "upsertSecretReferences"
| "findBySecretKeys"
| "bulkUpdate"
| "deleteMany"
>;
secretImportDAL: Pick<TSecretImportDALFactory, "find" | "findByFolderIds">;
secretSyncDAL: Pick<TSecretSyncDALFactory, "findById" | "find" | "updateById">;
auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
projectDAL: TProjectDALFactory;
smtpService: Pick<TSmtpService, "sendMail">;
projectBotDAL: TProjectBotDALFactory;
secretDAL: TSecretDALFactory;
secretVersionDAL: TSecretVersionDALFactory;
secretBlindIndexDAL: TSecretBlindIndexDALFactory;
secretTagDAL: TSecretTagDALFactory;
secretVersionTagDAL: TSecretVersionTagDALFactory;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">;
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">;
};
type SecretSyncActionJob = Job<
TQueueSecretSyncSyncSecretsByIdDTO | TQueueSecretSyncImportSecretsByIdDTO | TQueueSecretSyncRemoveSecretsByIdDTO
>;
const getRequeueDelay = (failureCount?: number) => {
if (!failureCount) return 0;
const baseDelay = 1000;
const maxDelay = 30000;
const delay = Math.min(baseDelay * 2 ** failureCount, maxDelay);
const jitter = delay * (0.5 + Math.random() * 0.5);
return jitter;
};
export const secretSyncQueueFactory = ({
queueService,
kmsService,
keyStore,
folderDAL,
secretV2BridgeDAL,
secretImportDAL,
secretSyncDAL,
auditLogService,
projectMembershipDAL,
projectDAL,
smtpService,
projectBotDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
secretVersionV2BridgeDAL,
secretVersionTagV2BridgeDAL,
resourceMetadataDAL
}: TSecretSyncQueueFactoryDep) => {
const appCfg = getConfig();
const integrationMeter = opentelemetry.metrics.getMeter("SecretSyncs");
const syncSecretsErrorHistogram = integrationMeter.createHistogram("secret_sync_sync_secrets_errors", {
description: "Secret Sync - sync secrets errors",
unit: "1"
});
const importSecretsErrorHistogram = integrationMeter.createHistogram("secret_sync_import_secrets_errors", {
description: "Secret Sync - import secrets errors",
unit: "1"
});
const removeSecretsErrorHistogram = integrationMeter.createHistogram("secret_sync_remove_secrets_errors", {
description: "Secret Sync - remove secrets errors",
unit: "1"
});
const $createManySecretsRawFn = createManySecretsRawFnFactory({
projectDAL,
projectBotDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL,
kmsService,
secretVersionV2BridgeDAL,
secretV2BridgeDAL,
secretVersionTagV2BridgeDAL,
resourceMetadataDAL
});
const $updateManySecretsRawFn = updateManySecretsRawFnFactory({
projectDAL,
projectBotDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL,
kmsService,
secretVersionV2BridgeDAL,
secretV2BridgeDAL,
secretVersionTagV2BridgeDAL,
resourceMetadataDAL
});
const $getSecrets = async (secretSync: TSecretSyncRaw | TSecretSyncWithCredentials, includeImports = true) => {
const {
projectId,
folderId,
environment: { slug: environmentSlug },
folder: { path: secretPath }
} = secretSync;
const secretMap: TSecretMap = {};
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId
});
const decryptSecretValue = (value?: Buffer | undefined | null) =>
value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : "";
const { expandSecretReferences } = expandSecretReferencesFactory({
decryptSecretValue,
secretDAL: secretV2BridgeDAL,
folderDAL,
projectId,
canExpandValue: () => true
});
const secrets = await secretV2BridgeDAL.findByFolderId(folderId);
await Promise.allSettled(
secrets.map(async (secret) => {
const secretKey = secret.key;
const secretValue = decryptSecretValue(secret.encryptedValue);
const expandedSecretValue = await expandSecretReferences({
environment: environmentSlug,
secretPath,
skipMultilineEncoding: secret.skipMultilineEncoding,
value: secretValue
});
secretMap[secretKey] = { value: expandedSecretValue || "" };
if (secret.encryptedComment) {
const commentValue = decryptSecretValue(secret.encryptedComment);
secretMap[secretKey].comment = commentValue;
}
secretMap[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding);
})
);
if (!includeImports) return secretMap;
const secretImports = await secretImportDAL.find({ folderId, isReplication: false });
if (secretImports.length) {
const importedSecrets = await fnSecretsV2FromImports({
decryptor: decryptSecretValue,
folderDAL,
secretDAL: secretV2BridgeDAL,
expandSecretReferences,
secretImportDAL,
secretImports,
hasSecretAccess: () => true
});
for (let i = importedSecrets.length - 1; i >= 0; i -= 1) {
for (let j = 0; j < importedSecrets[i].secrets.length; j += 1) {
const importedSecret = importedSecrets[i].secrets[j];
if (!secretMap[importedSecret.key]) {
secretMap[importedSecret.key] = {
skipMultilineEncoding: importedSecret.skipMultilineEncoding,
comment: importedSecret.secretComment,
value: importedSecret.secretValue || ""
};
}
}
}
}
return secretMap;
};
const queueSecretSyncSyncSecretsById = async (payload: TQueueSecretSyncSyncSecretsByIdDTO) =>
queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncSyncSecrets, payload, {
delay: getRequeueDelay(payload.failedToAcquireLockCount),
attempts: 5,
backoff: {
type: "exponential",
delay: 3000
},
removeOnComplete: true,
removeOnFail: true
});
const queueSecretSyncImportSecretsById = async (payload: TQueueSecretSyncImportSecretsByIdDTO) =>
queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncImportSecrets, payload, {
attempts: 1,
removeOnComplete: true,
removeOnFail: true
});
const queueSecretSyncRemoveSecretsById = async (payload: TQueueSecretSyncRemoveSecretsByIdDTO) =>
queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncRemoveSecrets, payload, {
attempts: 1,
removeOnComplete: true,
removeOnFail: true
});
const $queueSendSecretSyncFailedNotifications = async (payload: TQueueSendSecretSyncActionFailedNotificationsDTO) => {
if (!appCfg.isSmtpConfigured) return;
await queueService.queue(
QueueName.AppConnectionSecretSync,
QueueJobs.SecretSyncSendActionFailedNotifications,
payload,
{
jobId: `secret-sync-${payload.secretSync.id}-failed-notifications`,
attempts: 5,
delay: 1000 * 60,
backoff: {
type: "exponential",
delay: 3000
},
removeOnFail: true,
removeOnComplete: true
}
);
};
const $importSecrets = async (secretSync: TSecretSyncWithCredentials, importBehavior: SecretSyncImportBehavior) => {
const { projectId, environment } = secretSync;
const importedSecrets = await SecretSyncFns.importSecrets(secretSync);
if (Object.keys(importedSecrets).length) {
const secretMap = await $getSecrets(secretSync, false);
const secretsToCreate: Parameters<typeof $createManySecretsRawFn>[0]["secrets"] = [];
const secretsToUpdate: Parameters<typeof $updateManySecretsRawFn>[0]["secrets"] = [];
Object.entries(importedSecrets).forEach(([key, { value }]) => {
const secret = {
secretName: key,
secretValue: value,
type: SecretType.Shared,
secretComment: ""
};
if (Object.hasOwn(secretMap, key)) {
secretsToUpdate.push(secret);
} else {
secretsToCreate.push(secret);
}
});
if (secretsToCreate.length) {
await $createManySecretsRawFn({
projectId,
path: secretSync.folder.path,
environment: environment.slug,
secrets: secretsToCreate
});
}
if (importBehavior === SecretSyncImportBehavior.PrioritizeDestination && secretsToUpdate.length) {
await $updateManySecretsRawFn({
projectId,
path: secretSync.folder.path,
environment: environment.slug,
secrets: secretsToUpdate
});
}
}
};
const $handleSyncSecretsJob = async (job: TSecretSyncSyncSecretsDTO) => {
const {
data: { syncId, auditLogInfo }
} = job;
const secretSync = await secretSyncDAL.findById(syncId);
if (!secretSync) throw new Error(`Cannot find secret sync with ID ${syncId}`);
await secretSyncDAL.updateById(syncId, {
syncStatus: SecretSyncStatus.Running
});
logger.info(
`SecretSync Sync [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
);
let isSynced = false;
let syncMessage: string | null = null;
const isFinalAttempt = job.attemptsStarted === job.opts.attempts;
try {
const {
connection: { orgId, encryptedCredentials }
} = secretSync;
const credentials = await decryptAppConnectionCredentials({
orgId,
encryptedCredentials,
kmsService
});
const secretSyncWithCredentials = {
...secretSync,
connection: {
...secretSync.connection,
credentials
}
} as TSecretSyncWithCredentials;
const {
lastSyncedAt,
syncOptions: { initialSyncBehavior }
} = secretSyncWithCredentials;
if (!lastSyncedAt && initialSyncBehavior !== SecretSyncInitialSyncBehavior.OverwriteDestination) {
await $importSecrets(
secretSyncWithCredentials,
initialSyncBehavior === SecretSyncInitialSyncBehavior.ImportPrioritizeSource
? SecretSyncImportBehavior.PrioritizeSource
: SecretSyncImportBehavior.PrioritizeDestination
);
}
const secretMap = await $getSecrets(secretSync);
await SecretSyncFns.syncSecrets(secretSyncWithCredentials, secretMap);
isSynced = true;
} catch (err) {
logger.error(
err,
`SecretSync Sync Error [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
);
if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
syncSecretsErrorHistogram.record(1, {
version: 1,
destination: secretSync.destination,
syncId: secretSync.id,
projectId: secretSync.projectId,
type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError",
status: err instanceof AxiosError ? err.response?.status : undefined,
name: err instanceof Error ? err.name : undefined
});
}
syncMessage =
// eslint-disable-next-line no-nested-ternary
(err instanceof AxiosError
? err?.response?.data
? JSON.stringify(err?.response?.data)
: err?.message
: (err as Error)?.message) || "An unknown error occurred.";
// re-throw so job fails
throw err;
} finally {
const ranAt = new Date();
const syncStatus = isSynced ? SecretSyncStatus.Succeeded : SecretSyncStatus.Failed;
await auditLogService.createAuditLog({
projectId: secretSync.projectId,
...(auditLogInfo ?? {
actor: {
type: ActorType.PLATFORM,
metadata: {}
}
}),
event: {
type: EventType.SECRET_SYNC_SYNC_SECRETS,
metadata: {
syncId: secretSync.id,
syncOptions: secretSync.syncOptions,
destination: secretSync.destination,
destinationConfig: secretSync.destinationConfig,
folderId: secretSync.folderId,
connectionId: secretSync.connectionId,
jobRanAt: ranAt,
jobId: job.id!,
syncStatus,
syncMessage
}
}
});
if (isSynced || isFinalAttempt) {
const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, {
syncStatus,
lastSyncJobId: job.id,
lastSyncMessage: syncMessage,
lastSyncedAt: isSynced ? ranAt : undefined
});
if (!isSynced) {
await $queueSendSecretSyncFailedNotifications({
secretSync: updatedSecretSync,
action: SecretSyncAction.SyncSecrets,
auditLogInfo
});
}
}
}
logger.info("SecretSync Sync Job with ID %s Completed", job.id);
};
const $handleImportSecretsJob = async (job: TSecretSyncImportSecretsDTO) => {
const {
data: { syncId, auditLogInfo, importBehavior }
} = job;
const secretSync = await secretSyncDAL.findById(syncId);
if (!secretSync) throw new Error(`Cannot find secret sync with ID ${syncId}`);
await secretSyncDAL.updateById(syncId, {
importStatus: SecretSyncStatus.Running
});
logger.info(
`SecretSync Import [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
);
let isSuccess = false;
let importMessage: string | null = null;
const isFinalAttempt = job.attemptsStarted === job.opts.attempts;
try {
const {
connection: { orgId, encryptedCredentials }
} = secretSync;
const credentials = await decryptAppConnectionCredentials({
orgId,
encryptedCredentials,
kmsService
});
await $importSecrets(
{
...secretSync,
connection: {
...secretSync.connection,
credentials
}
} as TSecretSyncWithCredentials,
importBehavior
);
isSuccess = true;
} catch (err) {
logger.error(
err,
`SecretSync Import Error [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
);
if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
importSecretsErrorHistogram.record(1, {
version: 1,
destination: secretSync.destination,
syncId: secretSync.id,
projectId: secretSync.projectId,
type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError",
status: err instanceof AxiosError ? err.response?.status : undefined,
name: err instanceof Error ? err.name : undefined
});
}
importMessage =
// eslint-disable-next-line no-nested-ternary
(err instanceof AxiosError
? err?.response?.data
? JSON.stringify(err?.response?.data)
: err?.message
: (err as Error)?.message) || "An unknown error occurred.";
// re-throw so job fails
throw err;
} finally {
const ranAt = new Date();
const importStatus = isSuccess ? SecretSyncStatus.Succeeded : SecretSyncStatus.Failed;
await auditLogService.createAuditLog({
projectId: secretSync.projectId,
...(auditLogInfo ?? {
actor: {
type: ActorType.PLATFORM,
metadata: {}
}
}),
event: {
type: EventType.SECRET_SYNC_IMPORT_SECRETS,
metadata: {
syncId: secretSync.id,
syncOptions: secretSync.syncOptions,
destination: secretSync.destination,
destinationConfig: secretSync.destinationConfig,
folderId: secretSync.folderId,
connectionId: secretSync.connectionId,
jobRanAt: ranAt,
jobId: job.id!,
importStatus,
importMessage,
importBehavior
}
}
});
if (isSuccess || isFinalAttempt) {
const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, {
importStatus,
lastImportJobId: job.id,
lastImportMessage: importMessage,
lastImportedAt: isSuccess ? ranAt : undefined
});
if (!isSuccess) {
await $queueSendSecretSyncFailedNotifications({
secretSync: updatedSecretSync,
action: SecretSyncAction.ImportSecrets,
auditLogInfo
});
}
}
}
logger.info("SecretSync Import Job with ID %s Completed", job.id);
};
const $handleRemoveSecretsJob = async (job: TSecretSyncRemoveSecretsDTO) => {
const {
data: { syncId, auditLogInfo }
} = job;
const secretSync = await secretSyncDAL.findById(syncId);
if (!secretSync) throw new Error(`Cannot find secret sync with ID ${syncId}`);
await secretSyncDAL.updateById(syncId, {
removeStatus: SecretSyncStatus.Running
});
logger.info(
`SecretSync Remove [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
);
let isSuccess = false;
let removeMessage: string | null = null;
const isFinalAttempt = job.attemptsStarted === job.opts.attempts;
try {
const {
connection: { orgId, encryptedCredentials }
} = secretSync;
const credentials = await decryptAppConnectionCredentials({
orgId,
encryptedCredentials,
kmsService
});
const secretMap = await $getSecrets(secretSync);
await SecretSyncFns.removeSecrets(
{
...secretSync,
connection: {
...secretSync.connection,
credentials
}
} as TSecretSyncWithCredentials,
secretMap
);
isSuccess = true;
} catch (err) {
logger.error(
err,
`SecretSync Remove Error [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
);
if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
removeSecretsErrorHistogram.record(1, {
version: 1,
destination: secretSync.destination,
syncId: secretSync.id,
projectId: secretSync.projectId,
type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError",
status: err instanceof AxiosError ? err.response?.status : undefined,
name: err instanceof Error ? err.name : undefined
});
}
removeMessage =
// eslint-disable-next-line no-nested-ternary
(err instanceof AxiosError
? err?.response?.data
? JSON.stringify(err?.response?.data)
: err?.message
: (err as Error)?.message) || "An unknown error occurred.";
// re-throw so job fails
throw err;
} finally {
const ranAt = new Date();
const removeStatus = isSuccess ? SecretSyncStatus.Succeeded : SecretSyncStatus.Failed;
await auditLogService.createAuditLog({
projectId: secretSync.projectId,
...(auditLogInfo ?? {
actor: {
type: ActorType.PLATFORM,
metadata: {}
}
}),
event: {
type: EventType.SECRET_SYNC_REMOVE_SECRETS,
metadata: {
syncId: secretSync.id,
syncOptions: secretSync.syncOptions,
destination: secretSync.destination,
destinationConfig: secretSync.destinationConfig,
folderId: secretSync.folderId,
connectionId: secretSync.connectionId,
jobRanAt: ranAt,
jobId: job.id!,
removeStatus,
removeMessage
}
}
});
if (isSuccess || isFinalAttempt) {
const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, {
removeStatus,
lastRemoveJobId: job.id,
lastRemoveMessage: removeMessage,
lastRemovedAt: isSuccess ? ranAt : undefined
});
if (!isSuccess) {
await $queueSendSecretSyncFailedNotifications({
secretSync: updatedSecretSync,
action: SecretSyncAction.RemoveSecrets,
auditLogInfo
});
}
}
}
logger.info("SecretSync Remove Job with ID %s Completed", job.id);
};
const $sendSecretSyncFailedNotifications = async (job: TSendSecretSyncFailedNotificationsJobDTO) => {
const {
data: { secretSync, auditLogInfo, action }
} = job;
const { projectId, destination, name, folder, lastSyncMessage, lastRemoveMessage, lastImportMessage, environment } =
secretSync;
const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
const project = await projectDAL.findById(projectId);
let projectAdmins = projectMembers.filter((member) =>
member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
);
const triggeredByUserId =
auditLogInfo && auditLogInfo.actor.type === ActorType.USER && auditLogInfo.actor.metadata.userId;
// only notify triggering user if triggered by admin
if (triggeredByUserId && projectAdmins.map((admin) => admin.userId).includes(triggeredByUserId)) {
projectAdmins = projectAdmins.filter((admin) => admin.userId === triggeredByUserId);
}
const syncDestination = SECRET_SYNC_NAME_MAP[destination as SecretSync];
let actionLabel: string;
let failureMessage: string | null | undefined;
switch (action) {
case SecretSyncAction.ImportSecrets:
actionLabel = "Import";
failureMessage = lastImportMessage;
break;
case SecretSyncAction.RemoveSecrets:
actionLabel = "Remove";
failureMessage = lastRemoveMessage;
break;
case SecretSyncAction.SyncSecrets:
default:
actionLabel = `Sync`;
failureMessage = lastSyncMessage;
break;
}
await smtpService.sendMail({
recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
template: SmtpTemplates.SecretSyncFailed,
subjectLine: `Secret Sync Failed to ${actionLabel} Secrets`,
substitutions: {
syncName: name,
syncDestination,
content: `Your ${syncDestination} Sync named "${name}" failed while attempting to ${action.toLowerCase()} secrets.`,
failureMessage,
secretPath: folder.path,
environment: environment.name,
projectName: project.name,
syncUrl: `${appCfg.SITE_URL}/integrations/secret-syncs/${destination}/${secretSync.id}`
}
});
};
const queueSecretSyncsSyncSecretsByPath = async ({
secretPath,
projectId,
environmentSlug
}: TQueueSecretSyncsByPathDTO) => {
const folder = await folderDAL.findBySecretPath(projectId, environmentSlug, secretPath);
if (!folder)
throw new Error(
`Could not find folder at path "${secretPath}" for environment with slug "${environmentSlug}" in project with ID "${projectId}"`
);
const secretSyncs = await secretSyncDAL.find({ folderId: folder.id, isEnabled: true });
await Promise.all(secretSyncs.map((secretSync) => queueSecretSyncSyncSecretsById({ syncId: secretSync.id })));
};
const $handleAcquireLockFailure = async (job: SecretSyncActionJob) => {
const { syncId, auditLogInfo } = job.data;
switch (job.name) {
case QueueJobs.SecretSyncSyncSecrets: {
const { failedToAcquireLockCount = 0, ...rest } = job.data as TQueueSecretSyncSyncSecretsByIdDTO;
//
if (failedToAcquireLockCount < 10) {
await queueSecretSyncSyncSecretsById({ ...rest, failedToAcquireLockCount: failedToAcquireLockCount + 1 });
return;
}
const secretSync = await secretSyncDAL.updateById(syncId, {
syncStatus: SecretSyncStatus.Failed,
lastSyncMessage:
"Failed to run job. This typically happens when a sync is already in progress. Please try again.",
lastSyncJobId: job.id
});
await $queueSendSecretSyncFailedNotifications({
secretSync,
action: SecretSyncAction.SyncSecrets,
auditLogInfo
});
break;
}
// Scott: the two cases below are unlikely to happen as we check the lock at the API level but including this as a fallback
case QueueJobs.SecretSyncImportSecrets: {
const secretSync = await secretSyncDAL.updateById(syncId, {
importStatus: SecretSyncStatus.Failed,
lastImportMessage:
"Failed to run job. This typically happens when a sync is already in progress. Please try again.",
lastImportJobId: job.id
});
await $queueSendSecretSyncFailedNotifications({
secretSync,
action: SecretSyncAction.ImportSecrets,
auditLogInfo
});
break;
}
case QueueJobs.SecretSyncRemoveSecrets: {
const secretSync = await secretSyncDAL.updateById(syncId, {
removeStatus: SecretSyncStatus.Failed,
lastRemoveMessage:
"Failed to run job. This typically happens when a sync is already in progress. Please try again.",
lastRemoveJobId: job.id
});
await $queueSendSecretSyncFailedNotifications({
secretSync,
action: SecretSyncAction.RemoveSecrets,
auditLogInfo
});
break;
}
default:
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
throw new Error(`Unhandled Secret Sync Job ${job.name}`);
}
};
queueService.start(QueueName.AppConnectionSecretSync, async (job) => {
if (job.name === QueueJobs.SecretSyncSendActionFailedNotifications) {
await $sendSecretSyncFailedNotifications(job as TSendSecretSyncFailedNotificationsJobDTO);
return;
}
const { syncId } = job.data as
| TQueueSecretSyncSyncSecretsByIdDTO
| TQueueSecretSyncImportSecretsByIdDTO
| TQueueSecretSyncRemoveSecretsByIdDTO;
let lock: Awaited<ReturnType<typeof keyStore.acquireLock>>;
try {
lock = await keyStore.acquireLock(
[KeyStorePrefixes.SecretSyncLock(syncId)],
// scott: not sure on this duration; syncs can take excessive amounts of time so we need to keep it locked,
// but should always release below...
5 * 60 * 1000
);
} catch (e) {
logger.info(`SecretSync Failed to acquire lock [syncId=${syncId}] [job=${job.name}]`);
await $handleAcquireLockFailure(job as SecretSyncActionJob);
return;
}
try {
switch (job.name) {
case QueueJobs.SecretSyncSyncSecrets:
await $handleSyncSecretsJob(job as TSecretSyncSyncSecretsDTO);
break;
case QueueJobs.SecretSyncImportSecrets:
await $handleImportSecretsJob(job as TSecretSyncImportSecretsDTO);
break;
case QueueJobs.SecretSyncRemoveSecrets:
await $handleRemoveSecretsJob(job as TSecretSyncRemoveSecretsDTO);
break;
default:
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
throw new Error(`Unhandled Secret Sync Job ${job.name}`);
}
} finally {
await lock.release();
}
});
return {
queueSecretSyncSyncSecretsById,
queueSecretSyncImportSecretsById,
queueSecretSyncRemoveSecretsById,
queueSecretSyncsSyncSecretsByPath
};
};
@@ -0,0 +1,95 @@
import { z } from "zod";
import { SecretSyncsSchema } from "@app/db/schemas/secret-syncs";
import { SecretSyncs } from "@app/lib/api-docs";
import { removeTrailingSlash } from "@app/lib/fn";
import { slugSchema } from "@app/server/lib/schemas";
import { SecretSync, SecretSyncInitialSyncBehavior } from "@app/services/secret-sync/secret-sync-enums";
import { SECRET_SYNC_CONNECTION_MAP } from "@app/services/secret-sync/secret-sync-maps";
import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types";
const SyncOptionsSchema = (secretSync: SecretSync, options: TSyncOptionsConfig = { canImportSecrets: true }) =>
z.object({
initialSyncBehavior: (options.canImportSecrets
? z.nativeEnum(SecretSyncInitialSyncBehavior)
: z.literal(SecretSyncInitialSyncBehavior.OverwriteDestination)
).describe(SecretSyncs.SYNC_OPTIONS(secretSync).INITIAL_SYNC_BEHAVIOR),
prependPrefix: z
.string()
.trim()
.transform((str) => str.toUpperCase())
.optional()
.describe(SecretSyncs.SYNC_OPTIONS(secretSync).PREPEND_PREFIX),
appendSuffix: z
.string()
.trim()
.transform((str) => str.toUpperCase())
.optional()
.describe(SecretSyncs.SYNC_OPTIONS(secretSync).APPEND_SUFFIX)
});
export const BaseSecretSyncSchema = (destination: SecretSync, syncOptionsConfig?: TSyncOptionsConfig) =>
SecretSyncsSchema.omit({
destination: true,
destinationConfig: true,
syncOptions: true
}).extend({
// destination needs to be on the extended object for type differentiation
syncOptions: SyncOptionsSchema(destination, syncOptionsConfig),
// join properties
projectId: z.string(),
connection: z.object({
app: z.literal(SECRET_SYNC_CONNECTION_MAP[destination]),
name: z.string(),
id: z.string().uuid()
}),
environment: z.object({ slug: z.string(), name: z.string(), id: z.string().uuid() }),
folder: z.object({ id: z.string(), path: z.string() })
});
export const GenericCreateSecretSyncFieldsSchema = (destination: SecretSync, syncOptionsConfig?: TSyncOptionsConfig) =>
z.object({
name: slugSchema({ field: "name" }).describe(SecretSyncs.CREATE(destination).name),
projectId: z.string().trim().min(1, "Project ID required").describe(SecretSyncs.CREATE(destination).projectId),
description: z
.string()
.trim()
.max(256, "Description cannot exceed 256 characters")
.nullish()
.describe(SecretSyncs.CREATE(destination).description),
connectionId: z.string().uuid().describe(SecretSyncs.CREATE(destination).connectionId),
environment: slugSchema({ field: "environment", max: 64 }).describe(SecretSyncs.CREATE(destination).environment),
secretPath: z
.string()
.trim()
.min(1, "Secret path required")
.transform(removeTrailingSlash)
.describe(SecretSyncs.CREATE(destination).secretPath),
isEnabled: z.boolean().default(true).describe(SecretSyncs.CREATE(destination).isEnabled),
syncOptions: SyncOptionsSchema(destination, syncOptionsConfig).describe(SecretSyncs.CREATE(destination).syncOptions)
});
export const GenericUpdateSecretSyncFieldsSchema = (destination: SecretSync, syncOptionsConfig?: TSyncOptionsConfig) =>
z.object({
name: slugSchema({ field: "name" }).describe(SecretSyncs.UPDATE(destination).name).optional(),
description: z
.string()
.trim()
.max(256, "Description cannot exceed 256 characters")
.nullish()
.describe(SecretSyncs.UPDATE(destination).description),
environment: slugSchema({ field: "environment", max: 64 })
.optional()
.describe(SecretSyncs.UPDATE(destination).environment),
secretPath: z
.string()
.trim()
.min(1, "Invalid secret path")
.transform(removeTrailingSlash)
.optional()
.describe(SecretSyncs.UPDATE(destination).secretPath),
isEnabled: z.boolean().optional().describe(SecretSyncs.UPDATE(destination).isEnabled),
syncOptions: SyncOptionsSchema(destination, syncOptionsConfig)
.optional()
.describe(SecretSyncs.UPDATE(destination).syncOptions)
});
@@ -0,0 +1,485 @@
import { ForbiddenError, subject } from "@casl/ability";
import { ActionProjectType } from "@app/db/schemas";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { OrgServiceActor } from "@app/lib/types";
import { TAppConnectionServiceFactory } from "@app/services/app-connection/app-connection-service";
import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { listSecretSyncOptions } from "@app/services/secret-sync/secret-sync-fns";
import {
SecretSyncStatus,
TCreateSecretSyncDTO,
TDeleteSecretSyncDTO,
TFindSecretSyncByIdDTO,
TFindSecretSyncByNameDTO,
TListSecretSyncsByProjectId,
TSecretSync,
TTriggerSecretSyncImportSecretsByIdDTO,
TTriggerSecretSyncRemoveSecretsByIdDTO,
TTriggerSecretSyncSyncSecretsByIdDTO,
TUpdateSecretSyncDTO
} from "@app/services/secret-sync/secret-sync-types";
import { TSecretSyncDALFactory } from "./secret-sync-dal";
import { SECRET_SYNC_CONNECTION_MAP, SECRET_SYNC_NAME_MAP } from "./secret-sync-maps";
import { TSecretSyncQueueFactory } from "./secret-sync-queue";
type TSecretSyncServiceFactoryDep = {
secretSyncDAL: TSecretSyncDALFactory;
appConnectionService: Pick<TAppConnectionServiceFactory, "connectAppConnectionById">;
permissionService: Pick<TPermissionServiceFactory, "getProjectPermission" | "getOrgPermission">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
folderDAL: Pick<TSecretFolderDALFactory, "findByProjectId" | "findById" | "findBySecretPath">;
keyStore: Pick<TKeyStoreFactory, "getItem">;
secretSyncQueue: Pick<
TSecretSyncQueueFactory,
"queueSecretSyncSyncSecretsById" | "queueSecretSyncImportSecretsById" | "queueSecretSyncRemoveSecretsById"
>;
};
export type TSecretSyncServiceFactory = ReturnType<typeof secretSyncServiceFactory>;
export const secretSyncServiceFactory = ({
secretSyncDAL,
folderDAL,
permissionService,
appConnectionService,
projectBotService,
secretSyncQueue,
keyStore
}: TSecretSyncServiceFactoryDep) => {
const listSecretSyncsByProjectId = async (
{ projectId, destination }: TListSecretSyncsByProjectId,
actor: OrgServiceActor
) => {
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretManager,
projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
const folders = await folderDAL.findByProjectId(projectId);
const secretSyncs = await secretSyncDAL.find({
...(destination && { destination }),
$in: {
folderId: folders.map((folder) => folder.id)
}
});
return secretSyncs as TSecretSync[];
};
const findSecretSyncById = async ({ destination, syncId }: TFindSecretSyncByIdDTO, actor: OrgServiceActor) => {
const secretSync = await secretSyncDAL.findById(syncId);
if (!secretSync)
throw new NotFoundError({
message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID "${syncId}"`
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretManager,
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
});
return secretSync as TSecretSync;
};
const findSecretSyncByName = async (
{ destination, syncName, projectId }: TFindSecretSyncByNameDTO,
actor: OrgServiceActor
) => {
const folders = await folderDAL.findByProjectId(projectId);
// we prevent conflicting names within a project so this will only return one at most
const [secretSync] = await secretSyncDAL.find({
name: syncName,
$in: {
folderId: folders.map((folder) => folder.id)
}
});
if (!secretSync)
throw new NotFoundError({
message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with name "${syncName}"`
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretManager,
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
});
return secretSync as TSecretSync;
};
const createSecretSync = async (
{ projectId, secretPath, environment, ...params }: TCreateSecretSyncDTO,
actor: OrgServiceActor
) => {
const { permission: projectPermission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretManager,
projectId
});
const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
if (!shouldUseSecretV2Bridge)
throw new BadRequestError({ message: "Project version does not support Secret Syncs" });
ForbiddenError.from(projectPermission).throwUnlessCan(
ProjectPermissionActions.Create,
ProjectPermissionSub.SecretSyncs
);
ForbiddenError.from(projectPermission).throwUnlessCan(
ProjectPermissionActions.Read,
subject(ProjectPermissionSub.Secrets, {
environment,
secretPath
})
);
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder)
throw new BadRequestError({
message: `Could not find folder with path "${secretPath}" in environment "${environment}" for project with ID "${projectId}"`
});
const destinationApp = SECRET_SYNC_CONNECTION_MAP[params.destination];
// validates permission to connect and app is valid for sync destination
await appConnectionService.connectAppConnectionById(destinationApp, params.connectionId, actor);
const projectFolders = await folderDAL.findByProjectId(folder.projectId);
const secretSync = await secretSyncDAL.transaction(async (tx) => {
const isConflictingName = Boolean(
(
await secretSyncDAL.find(
{
name: params.name,
$in: {
folderId: projectFolders.map((f) => f.id)
}
},
tx
)
).length
);
if (isConflictingName)
throw new BadRequestError({
message: `A Secret Sync with the name "${params.name}" already exists for the project with ID "${folder.projectId}"`
});
const sync = await secretSyncDAL.create({
folderId: folder.id,
...params,
...(params.isEnabled && { syncStatus: SecretSyncStatus.Pending })
});
return sync;
});
if (secretSync.isEnabled) await secretSyncQueue.queueSecretSyncSyncSecretsById({ syncId: secretSync.id });
return secretSync as TSecretSync;
};
const updateSecretSync = async (
{ destination, syncId, secretPath, environment, ...params }: TUpdateSecretSyncDTO,
actor: OrgServiceActor
) => {
const secretSync = await secretSyncDAL.findById(syncId);
if (!secretSync)
throw new NotFoundError({
message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID ${syncId}`
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretManager,
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Edit, ProjectPermissionSub.SecretSyncs);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
});
const updatedSecretSync = await secretSyncDAL.transaction(async (tx) => {
if (
(secretPath && secretPath !== secretSync.folder.path) ||
(environment && environment !== secretSync.environment.slug)
) {
const updatedEnvironment = environment ?? secretSync.environment.slug;
const updatedSecretPath = secretPath ?? secretSync.folder.path;
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
subject(ProjectPermissionSub.Secrets, {
environment: updatedEnvironment,
secretPath: updatedSecretPath
})
);
const newFolder = await folderDAL.findBySecretPath(secretSync.projectId, updatedEnvironment, updatedSecretPath);
if (!newFolder)
throw new BadRequestError({
message: `Could not find folder with path "${secretPath}" in environment "${environment}" for project with ID "${secretSync.projectId}"`
});
}
if (params.name && secretSync.name !== params.name) {
const projectFolders = await folderDAL.findByProjectId(secretSync.projectId);
const isConflictingName = Boolean(
(
await secretSyncDAL.find(
{
name: params.name,
$in: {
folderId: projectFolders.map((f) => f.id)
}
},
tx
)
).length
);
if (isConflictingName)
throw new BadRequestError({
message: `A Secret Sync with the name "${params.name}" already exists for project with ID "${secretSync.projectId}"`
});
}
const isEnabled = params.isEnabled ?? secretSync.isEnabled;
const updatedSync = await secretSyncDAL.updateById(syncId, {
...params,
...(isEnabled && { syncStatus: SecretSyncStatus.Pending })
});
return updatedSync;
});
if (updatedSecretSync.isEnabled) await secretSyncQueue.queueSecretSyncSyncSecretsById({ syncId: secretSync.id });
return updatedSecretSync as TSecretSync;
};
const deleteSecretSync = async ({ destination, syncId }: TDeleteSecretSyncDTO, actor: OrgServiceActor) => {
const secretSync = await secretSyncDAL.findById(syncId);
if (!secretSync)
throw new NotFoundError({
message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID "${syncId}"`
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretManager,
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Delete, ProjectPermissionSub.SecretSyncs);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
});
await secretSyncDAL.deleteById(syncId);
return secretSync as TSecretSync;
};
const triggerSecretSyncSyncSecretsById = async (
{ syncId, destination, ...params }: TTriggerSecretSyncSyncSecretsByIdDTO,
actor: OrgServiceActor
) => {
const secretSync = await secretSyncDAL.findById(syncId);
if (!secretSync)
throw new NotFoundError({
message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID "${syncId}"`
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretManager,
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
});
const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId)));
if (isSyncJobRunning)
throw new BadRequestError({ message: `A job for this sync is already in progress. Please try again shortly.` });
await secretSyncQueue.queueSecretSyncSyncSecretsById({ syncId, ...params });
const updatedSecretSync = await secretSyncDAL.updateById(syncId, {
syncStatus: SecretSyncStatus.Pending
});
return updatedSecretSync as TSecretSync;
};
const triggerSecretSyncImportSecretsById = async (
{ syncId, destination, ...params }: TTriggerSecretSyncImportSecretsByIdDTO,
actor: OrgServiceActor
) => {
if (!listSecretSyncOptions().find((option) => option.destination === destination)?.canImportSecrets) {
throw new BadRequestError({
message: `${SECRET_SYNC_NAME_MAP[destination]} does not support importing secrets.`
});
}
const secretSync = await secretSyncDAL.findById(syncId);
if (!secretSync)
throw new NotFoundError({
message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID "${syncId}"`
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretManager,
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
});
const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId)));
if (isSyncJobRunning)
throw new BadRequestError({ message: `A job for this sync is already in progress. Please try again shortly.` });
await secretSyncQueue.queueSecretSyncImportSecretsById({ syncId, ...params });
const updatedSecretSync = await secretSyncDAL.updateById(syncId, {
importStatus: SecretSyncStatus.Pending
});
return updatedSecretSync as TSecretSync;
};
const triggerSecretSyncRemoveSecretsById = async (
{ syncId, destination, ...params }: TTriggerSecretSyncRemoveSecretsByIdDTO,
actor: OrgServiceActor
) => {
const secretSync = await secretSyncDAL.findById(syncId);
if (!secretSync)
throw new NotFoundError({
message: `Could not find ${SECRET_SYNC_NAME_MAP[destination]} Sync with ID "${syncId}"`
});
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.SecretManager,
projectId: secretSync.projectId
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SecretSyncs);
if (secretSync.connection.app !== SECRET_SYNC_CONNECTION_MAP[destination])
throw new BadRequestError({
message: `Secret sync with ID "${secretSync.id}" is not configured for ${SECRET_SYNC_NAME_MAP[destination]}`
});
const isSyncJobRunning = Boolean(await keyStore.getItem(KeyStorePrefixes.SecretSyncLock(syncId)));
if (isSyncJobRunning)
throw new BadRequestError({ message: `A job for this sync is already in progress. Please try again shortly.` });
await secretSyncQueue.queueSecretSyncRemoveSecretsById({ syncId, ...params });
const updatedSecretSync = await secretSyncDAL.updateById(syncId, {
removeStatus: SecretSyncStatus.Pending
});
return updatedSecretSync as TSecretSync;
};
return {
listSecretSyncOptions,
listSecretSyncsByProjectId,
findSecretSyncById,
findSecretSyncByName,
createSecretSync,
updateSecretSync,
deleteSecretSync,
triggerSecretSyncSyncSecretsById,
triggerSecretSyncImportSecretsById,
triggerSecretSyncRemoveSecretsById
};
};
@@ -0,0 +1,146 @@
import { Job } from "bullmq";
import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types";
import { QueueJobs } from "@app/queue";
import {
TGitHubSync,
TGitHubSyncInput,
TGitHubSyncListItem,
TGitHubSyncWithCredentials
} from "@app/services/secret-sync/github";
import { TSecretSyncDALFactory } from "@app/services/secret-sync/secret-sync-dal";
import { SecretSync, SecretSyncImportBehavior } from "@app/services/secret-sync/secret-sync-enums";
import {
TAwsParameterStoreSync,
TAwsParameterStoreSyncInput,
TAwsParameterStoreSyncListItem,
TAwsParameterStoreSyncWithCredentials
} from "./aws-parameter-store";
export type TSecretSync = TAwsParameterStoreSync | TGitHubSync;
export type TSecretSyncWithCredentials = TAwsParameterStoreSyncWithCredentials | TGitHubSyncWithCredentials;
export type TSecretSyncInput = TAwsParameterStoreSyncInput | TGitHubSyncInput;
export type TSecretSyncListItem = TAwsParameterStoreSyncListItem | TGitHubSyncListItem;
export type TSyncOptionsConfig = {
canImportSecrets: boolean;
};
export type TListSecretSyncsByProjectId = {
projectId: string;
destination?: SecretSync;
};
export type TFindSecretSyncByIdDTO = {
syncId: string;
destination: SecretSync;
};
export type TFindSecretSyncByNameDTO = {
syncName: string;
projectId: string;
destination: SecretSync;
};
export type TCreateSecretSyncDTO = Pick<TSecretSync, "syncOptions" | "destinationConfig" | "name" | "connectionId"> & {
destination: SecretSync;
projectId: string;
secretPath: string;
environment: string;
isEnabled?: boolean;
};
export type TUpdateSecretSyncDTO = Partial<Omit<TCreateSecretSyncDTO, "connectionId" | "projectId">> & {
syncId: string;
destination: SecretSync;
};
export type TDeleteSecretSyncDTO = {
destination: SecretSync;
syncId: string;
};
type AuditLogInfo = Pick<TCreateAuditLogDTO, "userAgent" | "userAgentType" | "ipAddress" | "actor">;
export enum SecretSyncStatus {
Pending = "pending",
Running = "running",
Succeeded = "succeeded",
Failed = "failed"
}
export enum SecretSyncAction {
SyncSecrets = "sync-secrets",
ImportSecrets = "import-secrets",
RemoveSecrets = "remove-secrets"
}
export type TSecretSyncRaw = NonNullable<Awaited<ReturnType<TSecretSyncDALFactory["findById"]>>>;
export type TQueueSecretSyncsByPathDTO = {
secretPath: string;
environmentSlug: string;
projectId: string;
};
export type TQueueSecretSyncSyncSecretsByIdDTO = {
syncId: string;
failedToAcquireLockCount?: number;
auditLogInfo?: AuditLogInfo;
};
export type TTriggerSecretSyncSyncSecretsByIdDTO = {
destination: SecretSync;
} & TQueueSecretSyncSyncSecretsByIdDTO;
export type TQueueSecretSyncImportSecretsByIdDTO = {
syncId: string;
importBehavior: SecretSyncImportBehavior;
auditLogInfo?: AuditLogInfo;
};
export type TTriggerSecretSyncImportSecretsByIdDTO = {
destination: SecretSync;
} & TQueueSecretSyncImportSecretsByIdDTO;
export type TQueueSecretSyncRemoveSecretsByIdDTO = {
syncId: string;
auditLogInfo?: AuditLogInfo;
};
export type TTriggerSecretSyncRemoveSecretsByIdDTO = {
destination: SecretSync;
} & TQueueSecretSyncRemoveSecretsByIdDTO;
export type TQueueSendSecretSyncActionFailedNotificationsDTO = {
secretSync: TSecretSyncRaw;
auditLogInfo?: AuditLogInfo;
action: SecretSyncAction;
};
export type TSecretSyncSyncSecretsDTO = Job<TQueueSecretSyncSyncSecretsByIdDTO, void, QueueJobs.SecretSyncSyncSecrets>;
export type TSecretSyncImportSecretsDTO = Job<
TQueueSecretSyncImportSecretsByIdDTO,
void,
QueueJobs.SecretSyncSyncSecrets
>;
export type TSecretSyncRemoveSecretsDTO = Job<
TQueueSecretSyncRemoveSecretsByIdDTO,
void,
QueueJobs.SecretSyncSyncSecrets
>;
export type TSendSecretSyncFailedNotificationsJobDTO = Job<
TQueueSendSecretSyncActionFailedNotificationsDTO,
void,
QueueJobs.SecretSyncSendActionFailedNotifications
>;
export type TSecretMap = Record<
string,
{ value: string; comment?: string; skipMultilineEncoding?: boolean | null | undefined }
>;
+7 -1
View File
@@ -29,6 +29,7 @@ import { createManySecretsRawFnFactory, updateManySecretsRawFnFactory } from "@a
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
import { TSecretSyncQueueFactory } from "@app/services/secret-sync/secret-sync-queue";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { ActorType } from "../auth/auth-type";
@@ -107,6 +108,7 @@ type TSecretQueueFactoryDep = {
orgService: Pick<TOrgServiceFactory, "addGhostUser">;
projectUserMembershipRoleDAL: Pick<TProjectUserMembershipRoleDALFactory, "create">;
resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">;
secretSyncQueue: Pick<TSecretSyncQueueFactory, "queueSecretSyncsSyncSecretsByPath">;
};
export type TGetSecrets = {
@@ -166,7 +168,8 @@ export const secretQueueFactory = ({
orgService,
projectUserMembershipRoleDAL,
projectKeyDAL,
resourceMetadataDAL
resourceMetadataDAL,
secretSyncQueue
}: TSecretQueueFactoryDep) => {
const integrationMeter = opentelemetry.metrics.getMeter("Integrations");
const errorHistogram = integrationMeter.createHistogram("integration_secret_sync_errors", {
@@ -633,6 +636,9 @@ export const secretQueueFactory = ({
}
}
);
await secretSyncQueue.queueSecretSyncsSyncSecretsByPath({ projectId, environmentSlug: environment, secretPath });
await syncIntegrations({ secretPath, projectId, environment, deDupeQueue, isManual: false });
if (!excludeReplication) {
await replicateSecrets({
@@ -35,6 +35,7 @@ export enum SmtpTemplates {
ScimUserProvisioned = "scimUserProvisioned.handlebars",
PkiExpirationAlert = "pkiExpirationAlert.handlebars",
IntegrationSyncFailed = "integrationSyncFailed.handlebars",
SecretSyncFailed = "secretSyncFailed.handlebars",
ExternalImportSuccessful = "externalImportSuccessful.handlebars",
ExternalImportFailed = "externalImportFailed.handlebars",
ExternalImportStarted = "externalImportStarted.handlebars"
@@ -0,0 +1,35 @@
<html>
<head>
<meta charset="utf-8" />
<meta http-equiv="x-ua-compatible" content="ie=edge" />
<title>{{syncDestination}} Sync "{{syncName}}" Failed</title>
</head>
<body>
<h2>Infisical</h2>
<div>
<p>{{content}}</p>
<a href="{{syncUrl}}">
View in Infisical.
</a>
</div>
<br />
<div>
<p><strong>Name</strong>: {{syncName}}</p>
<p><strong>Destination</strong>: {{syncDestination}}</p>
<p><strong>Project</strong>: {{projectName}}</p>
<p><strong>Environment</strong>: {{environment}}</p>
<p><strong>Secret Path</strong>: {{secretPath}}</p>
</div>
{{#if failureMessage}}
<p><b>Reason: </b>{{failureMessage}}</p>
{{/if}}
{{emailFooter}}
</body>
</html>