Merge branch 'main' into feat/showWarningOnImportedSecretDeletion

This commit is contained in:
carlosmonastyrski
2025-04-14 09:04:47 -03:00
423 changed files with 14637 additions and 1130 deletions
@@ -6,8 +6,11 @@ export enum AppConnection {
AzureKeyVault = "azure-key-vault",
AzureAppConfiguration = "azure-app-configuration",
Humanitec = "humanitec",
TerraformCloud = "terraform-cloud",
Vercel = "vercel",
Postgres = "postgres",
MsSql = "mssql"
MsSql = "mssql",
Camunda = "camunda"
}
export enum AWSRegion {
@@ -27,6 +27,7 @@ import {
getAzureKeyVaultConnectionListItem,
validateAzureKeyVaultConnectionCredentials
} from "./azure-key-vault";
import { CamundaConnectionMethod, getCamundaConnectionListItem, validateCamundaConnectionCredentials } from "./camunda";
import {
DatabricksConnectionMethod,
getDatabricksConnectionListItem,
@@ -41,6 +42,13 @@ import {
} from "./humanitec";
import { getMsSqlConnectionListItem, MsSqlConnectionMethod } from "./mssql";
import { getPostgresConnectionListItem, PostgresConnectionMethod } from "./postgres";
import {
getTerraformCloudConnectionListItem,
TerraformCloudConnectionMethod,
validateTerraformCloudConnectionCredentials
} from "./terraform-cloud";
import { VercelConnectionMethod } from "./vercel";
import { getVercelConnectionListItem, validateVercelConnectionCredentials } from "./vercel/vercel-connection-fns";
export const listAppConnectionOptions = () => {
return [
@@ -51,8 +59,11 @@ export const listAppConnectionOptions = () => {
getAzureAppConfigurationConnectionListItem(),
getDatabricksConnectionListItem(),
getHumanitecConnectionListItem(),
getTerraformCloudConnectionListItem(),
getVercelConnectionListItem(),
getPostgresConnectionListItem(),
getMsSqlConnectionListItem()
getMsSqlConnectionListItem(),
getCamundaConnectionListItem()
].sort((a, b) => a.name.localeCompare(b.name));
};
@@ -108,7 +119,10 @@ const VALIDATE_APP_CONNECTION_CREDENTIALS_MAP: Record<AppConnection, TAppConnect
validateAzureAppConfigurationConnectionCredentials as TAppConnectionCredentialsValidator,
[AppConnection.Humanitec]: validateHumanitecConnectionCredentials as TAppConnectionCredentialsValidator,
[AppConnection.Postgres]: validateSqlConnectionCredentials as TAppConnectionCredentialsValidator,
[AppConnection.MsSql]: validateSqlConnectionCredentials as TAppConnectionCredentialsValidator
[AppConnection.MsSql]: validateSqlConnectionCredentials as TAppConnectionCredentialsValidator,
[AppConnection.TerraformCloud]: validateTerraformCloudConnectionCredentials as TAppConnectionCredentialsValidator,
[AppConnection.Camunda]: validateCamundaConnectionCredentials as TAppConnectionCredentialsValidator,
[AppConnection.Vercel]: validateVercelConnectionCredentials as TAppConnectionCredentialsValidator
};
export const validateAppConnectionCredentials = async (
@@ -131,7 +145,11 @@ export const getAppConnectionMethodName = (method: TAppConnection["method"]) =>
return "Service Account Impersonation";
case DatabricksConnectionMethod.ServicePrincipal:
return "Service Principal";
case CamundaConnectionMethod.ClientCredentials:
return "Client Credentials";
case HumanitecConnectionMethod.ApiToken:
case TerraformCloudConnectionMethod.ApiToken:
case VercelConnectionMethod.ApiToken:
return "API Token";
case PostgresConnectionMethod.UsernameAndPassword:
case MsSqlConnectionMethod.UsernameAndPassword:
@@ -175,5 +193,8 @@ export const TRANSITION_CONNECTION_CREDENTIALS_TO_PLATFORM: Record<
[AppConnection.AzureAppConfiguration]: platformManagedCredentialsNotSupported,
[AppConnection.Humanitec]: platformManagedCredentialsNotSupported,
[AppConnection.Postgres]: transferSqlConnectionCredentialsToPlatform as TAppConnectionTransitionCredentialsToPlatform,
[AppConnection.MsSql]: transferSqlConnectionCredentialsToPlatform as TAppConnectionTransitionCredentialsToPlatform
[AppConnection.MsSql]: transferSqlConnectionCredentialsToPlatform as TAppConnectionTransitionCredentialsToPlatform,
[AppConnection.TerraformCloud]: platformManagedCredentialsNotSupported,
[AppConnection.Camunda]: platformManagedCredentialsNotSupported,
[AppConnection.Vercel]: platformManagedCredentialsNotSupported
};
@@ -8,6 +8,9 @@ export const APP_CONNECTION_NAME_MAP: Record<AppConnection, string> = {
[AppConnection.AzureAppConfiguration]: "Azure App Configuration",
[AppConnection.Databricks]: "Databricks",
[AppConnection.Humanitec]: "Humanitec",
[AppConnection.TerraformCloud]: "Terraform Cloud",
[AppConnection.Vercel]: "Vercel",
[AppConnection.Postgres]: "PostgreSQL",
[AppConnection.MsSql]: "Microsoft SQL Server"
[AppConnection.MsSql]: "Microsoft SQL Server",
[AppConnection.Camunda]: "Camunda"
};
@@ -31,6 +31,8 @@ import { ValidateAwsConnectionCredentialsSchema } from "./aws";
import { awsConnectionService } from "./aws/aws-connection-service";
import { ValidateAzureAppConfigurationConnectionCredentialsSchema } from "./azure-app-configuration";
import { ValidateAzureKeyVaultConnectionCredentialsSchema } from "./azure-key-vault";
import { ValidateCamundaConnectionCredentialsSchema } from "./camunda";
import { camundaConnectionService } from "./camunda/camunda-connection-service";
import { ValidateDatabricksConnectionCredentialsSchema } from "./databricks";
import { databricksConnectionService } from "./databricks/databricks-connection-service";
import { ValidateGcpConnectionCredentialsSchema } from "./gcp";
@@ -41,6 +43,10 @@ import { ValidateHumanitecConnectionCredentialsSchema } from "./humanitec";
import { humanitecConnectionService } from "./humanitec/humanitec-connection-service";
import { ValidateMsSqlConnectionCredentialsSchema } from "./mssql";
import { ValidatePostgresConnectionCredentialsSchema } from "./postgres";
import { ValidateTerraformCloudConnectionCredentialsSchema } from "./terraform-cloud";
import { terraformCloudConnectionService } from "./terraform-cloud/terraform-cloud-connection-service";
import { ValidateVercelConnectionCredentialsSchema } from "./vercel";
import { vercelConnectionService } from "./vercel/vercel-connection-service";
export type TAppConnectionServiceFactoryDep = {
appConnectionDAL: TAppConnectionDALFactory;
@@ -58,8 +64,11 @@ const VALIDATE_APP_CONNECTION_CREDENTIALS_MAP: Record<AppConnection, TValidateAp
[AppConnection.AzureAppConfiguration]: ValidateAzureAppConfigurationConnectionCredentialsSchema,
[AppConnection.Databricks]: ValidateDatabricksConnectionCredentialsSchema,
[AppConnection.Humanitec]: ValidateHumanitecConnectionCredentialsSchema,
[AppConnection.TerraformCloud]: ValidateTerraformCloudConnectionCredentialsSchema,
[AppConnection.Vercel]: ValidateVercelConnectionCredentialsSchema,
[AppConnection.Postgres]: ValidatePostgresConnectionCredentialsSchema,
[AppConnection.MsSql]: ValidateMsSqlConnectionCredentialsSchema
[AppConnection.MsSql]: ValidateMsSqlConnectionCredentialsSchema,
[AppConnection.Camunda]: ValidateCamundaConnectionCredentialsSchema
};
export const appConnectionServiceFactory = ({
@@ -430,6 +439,9 @@ export const appConnectionServiceFactory = ({
gcp: gcpConnectionService(connectAppConnectionById),
databricks: databricksConnectionService(connectAppConnectionById, appConnectionDAL, kmsService),
aws: awsConnectionService(connectAppConnectionById),
humanitec: humanitecConnectionService(connectAppConnectionById)
humanitec: humanitecConnectionService(connectAppConnectionById),
terraformCloud: terraformCloudConnectionService(connectAppConnectionById),
camunda: camundaConnectionService(connectAppConnectionById, appConnectionDAL, kmsService),
vercel: vercelConnectionService(connectAppConnectionById)
};
};
@@ -21,6 +21,12 @@ import {
TAzureKeyVaultConnectionInput,
TValidateAzureKeyVaultConnectionCredentialsSchema
} from "./azure-key-vault";
import {
TCamundaConnection,
TCamundaConnectionConfig,
TCamundaConnectionInput,
TValidateCamundaConnectionCredentialsSchema
} from "./camunda";
import {
TDatabricksConnection,
TDatabricksConnectionConfig,
@@ -51,6 +57,18 @@ import {
TPostgresConnectionInput,
TValidatePostgresConnectionCredentialsSchema
} from "./postgres";
import {
TTerraformCloudConnection,
TTerraformCloudConnectionConfig,
TTerraformCloudConnectionInput,
TValidateTerraformCloudConnectionCredentialsSchema
} from "./terraform-cloud";
import {
TValidateVercelConnectionCredentialsSchema,
TVercelConnection,
TVercelConnectionConfig,
TVercelConnectionInput
} from "./vercel";
export type TAppConnection = { id: string } & (
| TAwsConnection
@@ -60,8 +78,11 @@ export type TAppConnection = { id: string } & (
| TAzureAppConfigurationConnection
| TDatabricksConnection
| THumanitecConnection
| TTerraformCloudConnection
| TVercelConnection
| TPostgresConnection
| TMsSqlConnection
| TCamundaConnection
);
export type TAppConnectionRaw = NonNullable<Awaited<ReturnType<TAppConnectionDALFactory["findById"]>>>;
@@ -76,8 +97,11 @@ export type TAppConnectionInput = { id: string } & (
| TAzureAppConfigurationConnectionInput
| TDatabricksConnectionInput
| THumanitecConnectionInput
| TTerraformCloudConnectionInput
| TVercelConnectionInput
| TPostgresConnectionInput
| TMsSqlConnectionInput
| TCamundaConnectionInput
);
export type TSqlConnectionInput = TPostgresConnectionInput | TMsSqlConnectionInput;
@@ -99,7 +123,10 @@ export type TAppConnectionConfig =
| TAzureAppConfigurationConnectionConfig
| TDatabricksConnectionConfig
| THumanitecConnectionConfig
| TSqlConnectionConfig;
| TTerraformCloudConnectionConfig
| TVercelConnectionConfig
| TSqlConnectionConfig
| TCamundaConnectionConfig;
export type TValidateAppConnectionCredentialsSchema =
| TValidateAwsConnectionCredentialsSchema
@@ -110,7 +137,10 @@ export type TValidateAppConnectionCredentialsSchema =
| TValidateDatabricksConnectionCredentialsSchema
| TValidateHumanitecConnectionCredentialsSchema
| TValidatePostgresConnectionCredentialsSchema
| TValidateMsSqlConnectionCredentialsSchema;
| TValidateMsSqlConnectionCredentialsSchema
| TValidateCamundaConnectionCredentialsSchema
| TValidateTerraformCloudConnectionCredentialsSchema
| TValidateVercelConnectionCredentialsSchema;
export type TListAwsConnectionKmsKeys = {
connectionId: string;
@@ -0,0 +1,3 @@
export enum CamundaConnectionMethod {
ClientCredentials = "client-credentials"
}
@@ -0,0 +1,88 @@
import { request } from "@app/lib/config/request";
import { BadRequestError } from "@app/lib/errors";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { encryptAppConnectionCredentials } from "@app/services/app-connection/app-connection-fns";
import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { TAppConnectionDALFactory } from "../app-connection-dal";
import { CamundaConnectionMethod } from "./camunda-connection-enums";
import { TAuthorizeCamundaConnection, TCamundaConnection, TCamundaConnectionConfig } from "./camunda-connection-types";
export const getCamundaConnectionListItem = () => {
return {
name: "Camunda" as const,
app: AppConnection.Camunda as const,
methods: Object.values(CamundaConnectionMethod) as [CamundaConnectionMethod.ClientCredentials]
};
};
const authorizeCamundaConnection = async ({
clientId,
clientSecret
}: Pick<TCamundaConnection["credentials"], "clientId" | "clientSecret">) => {
const { data } = await request.post<TAuthorizeCamundaConnection>(
IntegrationUrls.CAMUNDA_TOKEN_URL,
{
grant_type: "client_credentials",
client_id: clientId,
client_secret: clientSecret,
audience: "api.cloud.camunda.io"
},
{
headers: {
"Content-Type": "application/json"
}
}
);
return { accessToken: data.access_token, expiresAt: data.expires_in * 1000 + Date.now() };
};
export const getCamundaConnectionAccessToken = async (
{ id, orgId, credentials }: TCamundaConnection,
appConnectionDAL: Pick<TAppConnectionDALFactory, "updateById">,
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">
) => {
const { clientSecret, clientId, accessToken, expiresAt } = credentials;
// get new token if less than 30 seconds from expiry
if (Date.now() < expiresAt - 30_000) {
return accessToken;
}
const authData = await authorizeCamundaConnection({ clientId, clientSecret });
const updatedCredentials: TCamundaConnection["credentials"] = {
...credentials,
...authData
};
const encryptedCredentials = await encryptAppConnectionCredentials({
credentials: updatedCredentials,
orgId,
kmsService
});
await appConnectionDAL.updateById(id, { encryptedCredentials });
return authData.accessToken;
};
export const validateCamundaConnectionCredentials = async (appConnection: TCamundaConnectionConfig) => {
const { credentials } = appConnection;
try {
const { accessToken, expiresAt } = await authorizeCamundaConnection(appConnection.credentials);
return {
...credentials,
accessToken,
expiresAt
};
} catch (e: unknown) {
throw new BadRequestError({
message: `Unable to validate connection: verify credentials`
});
}
};
@@ -0,0 +1,77 @@
import { z } from "zod";
import { AppConnections } from "@app/lib/api-docs";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import {
BaseAppConnectionSchema,
GenericCreateAppConnectionFieldsSchema,
GenericUpdateAppConnectionFieldsSchema
} from "@app/services/app-connection/app-connection-schemas";
import { CamundaConnectionMethod } from "./camunda-connection-enums";
const BaseCamundaConnectionSchema = BaseAppConnectionSchema.extend({ app: z.literal(AppConnection.Camunda) });
export const CamundaConnectionClientCredentialsInputCredentialsSchema = z.object({
clientId: z.string().trim().min(1, "Client ID required").describe(AppConnections.CREDENTIALS.CAMUNDA.clientId),
clientSecret: z
.string()
.trim()
.min(1, "Client Secret required")
.describe(AppConnections.CREDENTIALS.CAMUNDA.clientSecret)
});
export const CamundaConnectionClientCredentialsOutputCredentialsSchema = z
.object({
accessToken: z.string(),
expiresAt: z.number()
})
.merge(CamundaConnectionClientCredentialsInputCredentialsSchema);
export const CamundaConnectionSchema = z.intersection(
BaseCamundaConnectionSchema,
z.discriminatedUnion("method", [
z.object({
method: z.literal(CamundaConnectionMethod.ClientCredentials),
credentials: CamundaConnectionClientCredentialsOutputCredentialsSchema
})
])
);
export const SanitizedCamundaConnectionSchema = z.discriminatedUnion("method", [
BaseCamundaConnectionSchema.extend({
method: z.literal(CamundaConnectionMethod.ClientCredentials),
credentials: CamundaConnectionClientCredentialsOutputCredentialsSchema.pick({
clientId: true
})
})
]);
export const ValidateCamundaConnectionCredentialsSchema = z.discriminatedUnion("method", [
z.object({
method: z
.literal(CamundaConnectionMethod.ClientCredentials)
.describe(AppConnections.CREATE(AppConnection.Camunda).method),
credentials: CamundaConnectionClientCredentialsInputCredentialsSchema.describe(
AppConnections.CREATE(AppConnection.Camunda).credentials
)
})
]);
export const CreateCamundaConnectionSchema = ValidateCamundaConnectionCredentialsSchema.and(
GenericCreateAppConnectionFieldsSchema(AppConnection.Camunda)
);
export const UpdateCamundaConnectionSchema = z
.object({
credentials: CamundaConnectionClientCredentialsInputCredentialsSchema.optional().describe(
AppConnections.UPDATE(AppConnection.Camunda).credentials
)
})
.and(GenericUpdateAppConnectionFieldsSchema(AppConnection.Camunda));
export const CamundaConnectionListItemSchema = z.object({
name: z.literal("Camunda"),
app: z.literal(AppConnection.Camunda),
methods: z.nativeEnum(CamundaConnectionMethod).array()
});
@@ -0,0 +1,50 @@
import { request } from "@app/lib/config/request";
import { OrgServiceActor } from "@app/lib/types";
import { TAppConnectionDALFactory } from "@app/services/app-connection/app-connection-dal";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { getCamundaConnectionAccessToken } from "./camunda-connection-fns";
import { TCamundaConnection, TCamundaListClustersResponse } from "./camunda-connection-types";
type TGetAppConnectionFunc = (
app: AppConnection,
connectionId: string,
actor: OrgServiceActor
) => Promise<TCamundaConnection>;
const listCamundaClusters = async (
appConnection: TCamundaConnection,
appConnectionDAL: Pick<TAppConnectionDALFactory, "updateById">,
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">
) => {
const accessToken = await getCamundaConnectionAccessToken(appConnection, appConnectionDAL, kmsService);
const { data } = await request.get<TCamundaListClustersResponse>(`${IntegrationUrls.CAMUNDA_API_URL}/clusters`, {
headers: {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json"
}
});
return data ?? [];
};
export const camundaConnectionService = (
getAppConnection: TGetAppConnectionFunc,
appConnectionDAL: Pick<TAppConnectionDALFactory, "updateById">,
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">
) => {
const listClusters = async (connectionId: string, actor: OrgServiceActor) => {
const appConnection = await getAppConnection(AppConnection.Camunda, connectionId, actor);
const clusters = await listCamundaClusters(appConnection, appConnectionDAL, kmsService);
return clusters;
};
return {
listClusters
};
};
@@ -0,0 +1,31 @@
import { z } from "zod";
import { DiscriminativePick } from "@app/lib/types";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import {
CamundaConnectionSchema,
CreateCamundaConnectionSchema,
ValidateCamundaConnectionCredentialsSchema
} from "./camunda-connection-schema";
export type TCamundaConnection = z.infer<typeof CamundaConnectionSchema>;
export type TCamundaConnectionInput = z.infer<typeof CreateCamundaConnectionSchema> & {
app: AppConnection.Camunda;
};
export type TValidateCamundaConnectionCredentialsSchema = typeof ValidateCamundaConnectionCredentialsSchema;
export type TCamundaConnectionConfig = DiscriminativePick<TCamundaConnectionInput, "method" | "app" | "credentials"> & {
orgId: string;
};
export type TAuthorizeCamundaConnection = {
access_token: string;
scope: string;
token_type: string;
expires_in: number;
};
export type TCamundaListClustersResponse = { uuid: string; name: string }[];
@@ -0,0 +1,4 @@
export * from "./camunda-connection-enums";
export * from "./camunda-connection-fns";
export * from "./camunda-connection-schema";
export * from "./camunda-connection-types";
@@ -0,0 +1,4 @@
export * from "./terraform-cloud-connection-enums";
export * from "./terraform-cloud-connection-fns";
export * from "./terraform-cloud-connection-schemas";
export * from "./terraform-cloud-connection-types";
@@ -0,0 +1,3 @@
export enum TerraformCloudConnectionMethod {
ApiToken = "api-token"
}
@@ -0,0 +1,135 @@
import { AxiosError, AxiosResponse } from "axios";
import { request } from "@app/lib/config/request";
import { BadRequestError, InternalServerError } from "@app/lib/errors";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
import { TerraformCloudConnectionMethod } from "./terraform-cloud-connection-enums";
import {
TTerraformCloudConnection,
TTerraformCloudConnectionConfig,
TTerraformCloudOrganization,
TTerraformCloudVariableSet,
TTerraformCloudWorkspace
} from "./terraform-cloud-connection-types";
export const getTerraformCloudConnectionListItem = () => {
return {
name: "Terraform Cloud" as const,
app: AppConnection.TerraformCloud as const,
methods: Object.values(TerraformCloudConnectionMethod) as [TerraformCloudConnectionMethod.ApiToken]
};
};
export const validateTerraformCloudConnectionCredentials = async (config: TTerraformCloudConnectionConfig) => {
const { credentials: inputCredentials } = config;
let response: AxiosResponse<{ data: TTerraformCloudOrganization[] }> | null = null;
try {
response = await request.get<{ data: TTerraformCloudOrganization[] }>(
`${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/organizations`,
{
headers: {
Authorization: `Bearer ${inputCredentials.apiToken}`,
"Content-Type": "application/vnd.api+json"
}
}
);
} catch (error: unknown) {
if (error instanceof AxiosError) {
throw new BadRequestError({
message: `Failed to validate credentials: ${error.message || "Unknown error"}`
});
}
throw new BadRequestError({
message: "Unable to validate connection - verify credentials"
});
}
if (!response?.data) {
throw new InternalServerError({
message: "Failed to get organizations: Response was empty"
});
}
return inputCredentials;
};
export const listOrganizations = async (
appConnection: TTerraformCloudConnection
): Promise<TTerraformCloudOrganization[]> => {
const {
credentials: { apiToken }
} = appConnection;
const headers = {
Authorization: `Bearer ${apiToken}`,
"Content-Type": "application/vnd.api+json"
};
const fetchAllPages = async <T>(url: string): Promise<T[]> => {
let results: T[] = [];
let nextUrl: string | null = url;
while (nextUrl) {
// eslint-disable-next-line no-await-in-loop
const res: AxiosResponse<{ data: T[]; links?: { next?: string } }> = await request.get(nextUrl, { headers });
results = results.concat(res.data.data);
nextUrl = res.data.links?.next || null;
}
return results;
};
const orgEntities = await fetchAllPages<{ id: string; attributes: { name: string } }>(
`${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/organizations`
);
const orgsWithVariableSetsAndWorkspaces: TTerraformCloudOrganization[] = [];
const variableSetPromises = orgEntities.map((org) =>
fetchAllPages<{ id: string; attributes: { name: string; description?: string; global?: boolean } }>(
`${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/organizations/${org.id}/varsets`
).catch(() => [])
);
const workspacePromises = orgEntities.map((org) =>
fetchAllPages<{ id: string; attributes: { name: string } }>(
`${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/organizations/${org.id}/workspaces`
).catch(() => [])
);
const [variableSetResults, workspaceResults] = await Promise.all([
Promise.all(variableSetPromises),
Promise.all(workspacePromises)
]);
for (let i = 0; i < orgEntities.length; i += 1) {
const org = orgEntities[i];
const variableSetsData = variableSetResults[i];
const workspacesData = workspaceResults[i];
const variableSets: TTerraformCloudVariableSet[] = variableSetsData.map((varSet) => ({
id: varSet.id,
name: varSet.attributes.name,
description: varSet.attributes.description,
global: varSet.attributes.global
}));
const workspaces: TTerraformCloudWorkspace[] = workspacesData.map((workspace) => ({
id: workspace.id,
name: workspace.attributes.name
}));
orgsWithVariableSetsAndWorkspaces.push({
id: org.id,
name: org.attributes.name,
variableSets,
workspaces
});
}
return orgsWithVariableSetsAndWorkspaces;
};
@@ -0,0 +1,60 @@
import z from "zod";
import { AppConnections } from "@app/lib/api-docs";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import {
BaseAppConnectionSchema,
GenericCreateAppConnectionFieldsSchema,
GenericUpdateAppConnectionFieldsSchema
} from "@app/services/app-connection/app-connection-schemas";
import { TerraformCloudConnectionMethod } from "./terraform-cloud-connection-enums";
export const TerraformCloudConnectionAccessTokenCredentialsSchema = z.object({
apiToken: z.string().trim().min(1, "API Token required").describe(AppConnections.CREDENTIALS.TERRAFORM_CLOUD.apiToken)
});
const BaseTerraformCloudConnectionSchema = BaseAppConnectionSchema.extend({
app: z.literal(AppConnection.TerraformCloud)
});
export const TerraformCloudConnectionSchema = BaseTerraformCloudConnectionSchema.extend({
method: z.literal(TerraformCloudConnectionMethod.ApiToken),
credentials: TerraformCloudConnectionAccessTokenCredentialsSchema
});
export const SanitizedTerraformCloudConnectionSchema = z.discriminatedUnion("method", [
BaseTerraformCloudConnectionSchema.extend({
method: z.literal(TerraformCloudConnectionMethod.ApiToken),
credentials: TerraformCloudConnectionAccessTokenCredentialsSchema.pick({})
})
]);
export const ValidateTerraformCloudConnectionCredentialsSchema = z.discriminatedUnion("method", [
z.object({
method: z
.literal(TerraformCloudConnectionMethod.ApiToken)
.describe(AppConnections?.CREATE(AppConnection.TerraformCloud).method),
credentials: TerraformCloudConnectionAccessTokenCredentialsSchema.describe(
AppConnections.CREATE(AppConnection.TerraformCloud).credentials
)
})
]);
export const CreateTerraformCloudConnectionSchema = ValidateTerraformCloudConnectionCredentialsSchema.and(
GenericCreateAppConnectionFieldsSchema(AppConnection.TerraformCloud)
);
export const UpdateTerraformCloudConnectionSchema = z
.object({
credentials: TerraformCloudConnectionAccessTokenCredentialsSchema.optional().describe(
AppConnections.UPDATE(AppConnection.TerraformCloud).credentials
)
})
.and(GenericUpdateAppConnectionFieldsSchema(AppConnection.TerraformCloud));
export const TerraformCloudConnectionListItemSchema = z.object({
name: z.literal("Terraform Cloud"),
app: z.literal(AppConnection.TerraformCloud),
methods: z.nativeEnum(TerraformCloudConnectionMethod).array()
});
@@ -0,0 +1,29 @@
import { logger } from "@app/lib/logger";
import { OrgServiceActor } from "@app/lib/types";
import { AppConnection } from "../app-connection-enums";
import { listOrganizations as getTerraformCloudOrganizations } from "./terraform-cloud-connection-fns";
import { TTerraformCloudConnection } from "./terraform-cloud-connection-types";
type TGetAppConnectionFunc = (
app: AppConnection,
connectionId: string,
actor: OrgServiceActor
) => Promise<TTerraformCloudConnection>;
export const terraformCloudConnectionService = (getAppConnection: TGetAppConnectionFunc) => {
const listOrganizations = async (connectionId: string, actor: OrgServiceActor) => {
const appConnection = await getAppConnection(AppConnection.TerraformCloud, connectionId, actor);
try {
const organizations = await getTerraformCloudOrganizations(appConnection);
return organizations;
} catch (error) {
logger.error(error, "Failed to establish connection with Terraform Cloud");
return [];
}
};
return {
listOrganizations
};
};
@@ -0,0 +1,45 @@
import z from "zod";
import { DiscriminativePick } from "@app/lib/types";
import { AppConnection } from "../app-connection-enums";
import {
CreateTerraformCloudConnectionSchema,
TerraformCloudConnectionSchema,
ValidateTerraformCloudConnectionCredentialsSchema
} from "./terraform-cloud-connection-schemas";
export type TTerraformCloudConnection = z.infer<typeof TerraformCloudConnectionSchema>;
export type TTerraformCloudConnectionInput = z.infer<typeof CreateTerraformCloudConnectionSchema> & {
app: AppConnection.TerraformCloud;
};
export type TValidateTerraformCloudConnectionCredentialsSchema =
typeof ValidateTerraformCloudConnectionCredentialsSchema;
export type TTerraformCloudConnectionConfig = DiscriminativePick<
TTerraformCloudConnectionInput,
"method" | "app" | "credentials"
> & {
orgId: string;
};
export type TTerraformCloudVariableSet = {
id: string;
name: string;
description?: string;
global?: boolean;
};
export type TTerraformCloudWorkspace = {
id: string;
name: string;
};
export type TTerraformCloudOrganization = {
id: string;
name: string;
variableSets: TTerraformCloudVariableSet[];
workspaces: TTerraformCloudWorkspace[];
};
@@ -0,0 +1,4 @@
export * from "./vercel-connection-enums";
export * from "./vercel-connection-fns";
export * from "./vercel-connection-schemas";
export * from "./vercel-connection-types";
@@ -0,0 +1,3 @@
export enum VercelConnectionMethod {
ApiToken = "api-token"
}
@@ -0,0 +1,273 @@
/* eslint-disable no-await-in-loop */
import { AxiosError, AxiosResponse } from "axios";
import { request } from "@app/lib/config/request";
import { BadRequestError, InternalServerError } from "@app/lib/errors";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { TVercelBranches } from "@app/services/integration-auth/integration-auth-types";
import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
import { VercelConnectionMethod } from "./vercel-connection-enums";
import {
TVercelConnection,
TVercelConnectionConfig,
VercelApp,
VercelEnvironment,
VercelOrgWithApps
} from "./vercel-connection-types";
export const getVercelConnectionListItem = () => {
return {
name: "Vercel" as const,
app: AppConnection.Vercel as const,
methods: Object.values(VercelConnectionMethod) as [VercelConnectionMethod.ApiToken]
};
};
export const validateVercelConnectionCredentials = async (config: TVercelConnectionConfig) => {
const { credentials: inputCredentials } = config;
let response: AxiosResponse<VercelApp[]> | null = null;
try {
response = await request.get<VercelApp[]>(`${IntegrationUrls.VERCEL_API_URL}/v9/projects`, {
headers: {
Authorization: `Bearer ${inputCredentials.apiToken}`
}
});
} catch (error: unknown) {
if (error instanceof AxiosError) {
throw new BadRequestError({
message: `Failed to validate credentials: ${error.message || "Unknown error"}`
});
}
throw new BadRequestError({
message: "Unable to validate connection - verify credentials"
});
}
if (!response?.data) {
throw new InternalServerError({
message: "Failed to get organizations: Response was empty"
});
}
return inputCredentials;
};
interface ApiResponse<T> {
pagination?: {
count: number;
next: number;
};
data: T[];
[key: string]: unknown;
}
async function fetchAllPages<T>(
apiUrl: string,
apiToken: string,
initialParams: Record<string, string | number> = {},
dataPath?: string
): Promise<T[]> {
const allItems: T[] = [];
let hasMoreItems = true;
let params: Record<string, string | number> = { ...initialParams, limit: 100 };
while (hasMoreItems) {
try {
const response = await request.get<ApiResponse<T>>(apiUrl, {
params,
headers: {
Authorization: `Bearer ${apiToken}`,
"Accept-Encoding": "application/json"
}
});
if (!response?.data) {
throw new InternalServerError({
message: `Failed to fetch data from ${apiUrl}: Response was empty or malformed`
});
}
let itemsData: T[];
if (dataPath && dataPath in response.data) {
itemsData = response.data[dataPath] as T[];
} else {
itemsData = response.data.data;
}
if (!Array.isArray(itemsData)) {
throw new InternalServerError({
message: `Failed to fetch data from ${apiUrl}: Expected array but got ${typeof itemsData}`
});
}
allItems.push(...itemsData);
if (response.data.pagination?.next) {
params = { ...params, since: response.data.pagination.next };
} else {
hasMoreItems = false;
}
} catch (error) {
if (error instanceof AxiosError) {
throw new BadRequestError({
message: `Failed to fetch data from ${apiUrl}: ${error.message || "Unknown error"}`
});
}
throw error;
}
}
return allItems;
}
async function fetchOrgProjects(orgId: string, apiToken: string): Promise<VercelApp[]> {
return fetchAllPages<VercelApp>(
`${IntegrationUrls.VERCEL_API_URL}/v9/projects`,
apiToken,
{ teamId: orgId },
"projects"
);
}
async function fetchProjectEnvironments(
projectId: string,
teamId: string,
apiToken: string
): Promise<VercelEnvironment[]> {
try {
return await fetchAllPages<VercelEnvironment>(
`${IntegrationUrls.VERCEL_API_URL}/v9/projects/${projectId}/custom-environments?teamId=${teamId}`,
apiToken,
{},
"environments"
);
} catch (error) {
return [];
}
}
async function fetchPreviewBranches(projectId: string, apiToken: string): Promise<string[]> {
try {
const { data } = await request.get<TVercelBranches[]>(
`${IntegrationUrls.VERCEL_API_URL}/v1/integrations/git-branches`,
{
params: {
projectId
},
headers: {
Authorization: `Bearer ${apiToken}`,
"Accept-Encoding": "application/json"
}
}
);
return data.filter((b) => b.ref !== "main").map((b) => b.ref);
} catch (error) {
return [];
}
}
type VercelTeam = {
id: string;
name: string;
slug: string;
};
type VercelUserResponse = {
user: {
id: string;
name: string;
username: string;
};
};
export const listProjects = async (appConnection: TVercelConnection): Promise<VercelOrgWithApps[]> => {
const { credentials } = appConnection;
const { apiToken } = credentials;
const orgs = await fetchAllPages<VercelTeam>(`${IntegrationUrls.VERCEL_API_URL}/v2/teams`, apiToken, {}, "teams");
const personalAccountResponse = await request.get<VercelUserResponse>(`${IntegrationUrls.VERCEL_API_URL}/v2/user`, {
headers: {
Authorization: `Bearer ${apiToken}`,
"Accept-Encoding": "application/json"
}
});
if (personalAccountResponse?.data?.user) {
const { user } = personalAccountResponse.data;
orgs.push({
id: user.id,
name: user.name || "Personal Account",
slug: user.username || "personal"
});
}
const orgsWithApps: VercelOrgWithApps[] = [];
const orgPromises = orgs.map(async (org) => {
try {
const projects = await fetchOrgProjects(org.id, apiToken);
const enhancedProjectsPromises = projects.map(async (project) => {
try {
const [environments, previewBranches] = await Promise.all([
fetchProjectEnvironments(project.name, org.id, apiToken),
fetchPreviewBranches(project.id, apiToken)
]);
return {
name: project.name,
id: project.id,
envs: environments,
previewBranches
};
} catch (error) {
return {
name: project.name,
id: project.id,
envs: [],
previewBranches: []
};
}
});
const enhancedProjects = await Promise.all(enhancedProjectsPromises);
return {
...org,
apps: enhancedProjects
};
} catch (error) {
return null;
}
});
const results = await Promise.all(orgPromises);
results.forEach((result) => {
if (result !== null) {
orgsWithApps.push(result);
}
});
return orgsWithApps;
};
export const getProjectEnvironmentVariables = (project: VercelApp): Record<string, string> => {
const envVars: Record<string, string> = {};
if (!project.envs) return envVars;
project.envs.forEach((env) => {
if (env.slug && env.type !== "gitBranch") {
const { id, slug } = env;
envVars[id] = slug;
}
});
return envVars;
};
@@ -0,0 +1,58 @@
import z from "zod";
import { AppConnections } from "@app/lib/api-docs";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import {
BaseAppConnectionSchema,
GenericCreateAppConnectionFieldsSchema,
GenericUpdateAppConnectionFieldsSchema
} from "@app/services/app-connection/app-connection-schemas";
import { VercelConnectionMethod } from "./vercel-connection-enums";
export const VercelConnectionAccessTokenCredentialsSchema = z.object({
apiToken: z.string().trim().min(1, "API Token required").describe(AppConnections.CREDENTIALS.VERCEL.apiToken)
});
const BaseVercelConnectionSchema = BaseAppConnectionSchema.extend({
app: z.literal(AppConnection.Vercel)
});
export const VercelConnectionSchema = BaseVercelConnectionSchema.extend({
method: z.literal(VercelConnectionMethod.ApiToken),
credentials: VercelConnectionAccessTokenCredentialsSchema
});
export const SanitizedVercelConnectionSchema = z.discriminatedUnion("method", [
BaseVercelConnectionSchema.extend({
method: z.literal(VercelConnectionMethod.ApiToken),
credentials: VercelConnectionAccessTokenCredentialsSchema.pick({})
})
]);
export const ValidateVercelConnectionCredentialsSchema = z.discriminatedUnion("method", [
z.object({
method: z.literal(VercelConnectionMethod.ApiToken).describe(AppConnections.CREATE(AppConnection.Vercel).method),
credentials: VercelConnectionAccessTokenCredentialsSchema.describe(
AppConnections.CREATE(AppConnection.Vercel).credentials
)
})
]);
export const CreateVercelConnectionSchema = ValidateVercelConnectionCredentialsSchema.and(
GenericCreateAppConnectionFieldsSchema(AppConnection.Vercel)
);
export const UpdateVercelConnectionSchema = z
.object({
credentials: VercelConnectionAccessTokenCredentialsSchema.optional().describe(
AppConnections.UPDATE(AppConnection.Vercel).credentials
)
})
.and(GenericUpdateAppConnectionFieldsSchema(AppConnection.Vercel));
export const VercelConnectionListItemSchema = z.object({
name: z.literal("Vercel"),
app: z.literal(AppConnection.Vercel),
methods: z.nativeEnum(VercelConnectionMethod).array()
});
@@ -0,0 +1,29 @@
import { logger } from "@app/lib/logger";
import { OrgServiceActor } from "@app/lib/types";
import { AppConnection } from "../app-connection-enums";
import { listProjects as getVercelProjects } from "./vercel-connection-fns";
import { TVercelConnection } from "./vercel-connection-types";
type TGetAppConnectionFunc = (
app: AppConnection,
connectionId: string,
actor: OrgServiceActor
) => Promise<TVercelConnection>;
export const vercelConnectionService = (getAppConnection: TGetAppConnectionFunc) => {
const listProjects = async (connectionId: string, actor: OrgServiceActor) => {
const appConnection = await getAppConnection(AppConnection.Vercel, connectionId, actor);
try {
const projects = await getVercelProjects(appConnection);
return projects;
} catch (error) {
logger.error(error, "Failed to establish connection with Vercel");
return [];
}
};
return {
listProjects
};
};
@@ -0,0 +1,73 @@
import z from "zod";
import { DiscriminativePick } from "@app/lib/types";
import { AppConnection } from "../app-connection-enums";
import {
CreateVercelConnectionSchema,
ValidateVercelConnectionCredentialsSchema,
VercelConnectionSchema
} from "./vercel-connection-schemas";
export type TVercelConnection = z.infer<typeof VercelConnectionSchema>;
export type TVercelConnectionInput = z.infer<typeof CreateVercelConnectionSchema> & {
app: AppConnection.Vercel;
};
export type TValidateVercelConnectionCredentialsSchema = typeof ValidateVercelConnectionCredentialsSchema;
export type TVercelConnectionConfig = DiscriminativePick<TVercelConnectionInput, "method" | "app" | "credentials"> & {
orgId: string;
};
export type VercelTeam = {
id: string;
name: string;
slug: string;
};
export type VercelEnvironment = {
id: string;
slug: string;
type: string;
target?: string[];
gitBranch?: string;
createdAt?: number;
updatedAt?: number;
};
export type VercelAppMeta = {
githubCommitRef?: string;
githubCommitSha?: string;
githubCommitMessage?: string;
githubCommitAuthorName?: string;
};
export type VercelDeployment = {
id: string;
name: string;
url: string;
created: number;
meta?: VercelAppMeta;
target?: "production" | "preview" | "development";
};
export type VercelApp = {
name: string;
id: string;
envs?: VercelEnvironment[];
previewBranches?: string[];
};
export type VercelOrgWithApps = VercelTeam & {
apps: VercelApp[];
};
export type VercelUserResponse = {
user: {
id: string;
name: string;
username: string;
};
};
+161 -3
View File
@@ -3,12 +3,18 @@ import { ForbiddenError } from "@casl/ability";
import { ActionProjectType, ProjectType } from "@app/db/schemas";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
import { ProjectPermissionCmekActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import { SigningAlgorithm } from "@app/lib/crypto/sign";
import { DatabaseErrorCode } from "@app/lib/error-codes";
import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
import { OrgServiceActor } from "@app/lib/types";
import {
TCmekDecryptDTO,
TCmekEncryptDTO,
TCmekGetPublicKeyDTO,
TCmekKeyEncryptionAlgorithm,
TCmekListSigningAlgorithmsDTO,
TCmekSignDTO,
TCmekVerifyDTO,
TCreateCmekDTO,
TListCmeksByProjectIdDTO,
TUpdabteCmekByIdDTO
@@ -16,6 +22,7 @@ import {
import { TKmsKeyDALFactory } from "@app/services/kms/kms-key-dal";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { KmsKeyUsage } from "../kms/kms-types";
import { TProjectDALFactory } from "../project/project-dal";
type TCmekServiceFactoryDep = {
@@ -221,7 +228,151 @@ export const cmekServiceFactory = ({ kmsService, kmsDAL, permissionService, proj
const { cipherTextBlob } = await encrypt({ plainText: Buffer.from(plaintext, "base64") });
return cipherTextBlob.toString("base64");
return {
ciphertext: cipherTextBlob.toString("base64"),
projectId: key.projectId
};
};
const listSigningAlgorithms = async ({ keyId }: TCmekListSigningAlgorithmsDTO, actor: OrgServiceActor) => {
const key = await kmsDAL.findCmekById(keyId);
if (!key) throw new NotFoundError({ message: `Key with ID "${keyId}" not found` });
if (!key.projectId || key.isReserved) throw new BadRequestError({ message: "Key is not customer managed" });
if (key.isDisabled) throw new BadRequestError({ message: "Key is disabled" });
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
projectId: key.projectId,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.KMS
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionCmekActions.Read, ProjectPermissionSub.Cmek);
if (key.keyUsage !== KmsKeyUsage.SIGN_VERIFY) {
throw new BadRequestError({ message: `Key with ID '${keyId}' is not intended for signing` });
}
const encryptionAlgorithm = key.encryptionAlgorithm as TCmekKeyEncryptionAlgorithm;
const algos = [
{
keyAlgorithm: "rsa",
signingAlgorithms: Object.values(SigningAlgorithm).filter((algorithm) =>
algorithm.toLowerCase().startsWith("rsa")
)
},
{
keyAlgorithm: "ecc",
signingAlgorithms: Object.values(SigningAlgorithm).filter((algorithm) =>
algorithm.toLowerCase().startsWith("ecdsa")
)
}
];
const selectedAlgorithm = algos.find((algo) => encryptionAlgorithm.toLowerCase().startsWith(algo.keyAlgorithm));
if (!selectedAlgorithm) {
throw new BadRequestError({ message: `Unsupported encryption algorithm: ${encryptionAlgorithm}` });
}
return { signingAlgorithms: selectedAlgorithm.signingAlgorithms, projectId: key.projectId };
};
const getPublicKey = async ({ keyId }: TCmekGetPublicKeyDTO, actor: OrgServiceActor) => {
const key = await kmsDAL.findCmekById(keyId);
if (!key) throw new NotFoundError({ message: `Key with ID "${keyId}" not found` });
if (!key.projectId || key.isReserved) throw new BadRequestError({ message: "Key is not customer managed" });
if (key.isDisabled) throw new BadRequestError({ message: "Key is disabled" });
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
projectId: key.projectId,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.KMS
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionCmekActions.Read, ProjectPermissionSub.Cmek);
const publicKey = await kmsService.getPublicKey({ kmsId: keyId });
return { publicKey: publicKey.toString("base64"), projectId: key.projectId };
};
const cmekSign = async ({ keyId, data, signingAlgorithm, isDigest }: TCmekSignDTO, actor: OrgServiceActor) => {
const key = await kmsDAL.findCmekById(keyId);
if (!key) throw new NotFoundError({ message: `Key with ID "${keyId}" not found` });
if (!key.projectId || key.isReserved) throw new BadRequestError({ message: "Key is not customer managed" });
if (key.isDisabled) throw new BadRequestError({ message: "Key is disabled" });
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
projectId: key.projectId,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.KMS
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionCmekActions.Sign, ProjectPermissionSub.Cmek);
const sign = await kmsService.signWithKmsKey({ kmsId: keyId });
const { signature, algorithm } = await sign({ data: Buffer.from(data, "base64"), signingAlgorithm, isDigest });
return {
signature: signature.toString("base64"),
keyId: key.id,
projectId: key.projectId,
signingAlgorithm: algorithm
};
};
const cmekVerify = async (
{ keyId, data, signature, signingAlgorithm, isDigest }: TCmekVerifyDTO,
actor: OrgServiceActor
) => {
const key = await kmsDAL.findCmekById(keyId);
if (!key) throw new NotFoundError({ message: `Key with ID "${keyId}" not found` });
if (!key.projectId || key.isReserved) throw new BadRequestError({ message: "Key is not customer managed" });
if (key.isDisabled) throw new BadRequestError({ message: "Key is disabled" });
const { permission } = await permissionService.getProjectPermission({
actor: actor.type,
actorId: actor.id,
projectId: key.projectId,
actorAuthMethod: actor.authMethod,
actorOrgId: actor.orgId,
actionProjectType: ActionProjectType.KMS
});
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionCmekActions.Verify, ProjectPermissionSub.Cmek);
const verify = await kmsService.verifyWithKmsKey({ kmsId: keyId, signingAlgorithm });
const { signatureValid, algorithm } = await verify({
isDigest,
data: Buffer.from(data, "base64"),
signature: Buffer.from(signature, "base64")
});
return {
signatureValid,
keyId: key.id,
projectId: key.projectId,
signingAlgorithm: algorithm
};
};
const cmekDecrypt = async ({ keyId, ciphertext }: TCmekDecryptDTO, actor: OrgServiceActor) => {
@@ -248,7 +399,10 @@ export const cmekServiceFactory = ({ kmsService, kmsDAL, permissionService, proj
const plaintextBlob = await decrypt({ cipherTextBlob: Buffer.from(ciphertext, "base64") });
return plaintextBlob.toString("base64");
return {
plaintext: plaintextBlob.toString("base64"),
projectId: key.projectId
};
};
return {
@@ -259,6 +413,10 @@ export const cmekServiceFactory = ({ kmsService, kmsDAL, permissionService, proj
cmekEncrypt,
cmekDecrypt,
findCmekById,
findCmekByName
findCmekByName,
cmekSign,
cmekVerify,
listSigningAlgorithms,
getPublicKey
};
};
+31 -2
View File
@@ -1,12 +1,18 @@
import { SymmetricEncryption } from "@app/lib/crypto/cipher";
import { SymmetricKeyAlgorithm } from "@app/lib/crypto/cipher";
import { AsymmetricKeyAlgorithm, SigningAlgorithm } from "@app/lib/crypto/sign";
import { OrderByDirection } from "@app/lib/types";
import { KmsKeyUsage } from "../kms/kms-types";
export type TCmekKeyEncryptionAlgorithm = SymmetricKeyAlgorithm | AsymmetricKeyAlgorithm;
export type TCreateCmekDTO = {
orgId: string;
projectId: string;
name: string;
description?: string;
encryptionAlgorithm: SymmetricEncryption;
encryptionAlgorithm: TCmekKeyEncryptionAlgorithm;
keyUsage: KmsKeyUsage;
};
export type TUpdabteCmekByIdDTO = {
@@ -38,3 +44,26 @@ export type TCmekDecryptDTO = {
export enum CmekOrderBy {
Name = "name"
}
export type TCmekListSigningAlgorithmsDTO = {
keyId: string;
};
export type TCmekGetPublicKeyDTO = {
keyId: string;
};
export type TCmekSignDTO = {
keyId: string;
data: string;
signingAlgorithm: SigningAlgorithm;
isDigest: boolean;
};
export type TCmekVerifyDTO = {
keyId: string;
data: string;
signature: string;
signingAlgorithm: SigningAlgorithm;
isDigest: boolean;
};
@@ -21,6 +21,7 @@ import {
TCreateProjectIdentityDTO,
TDeleteProjectIdentityDTO,
TGetProjectIdentityByIdentityIdDTO,
TGetProjectIdentityByMembershipIdDTO,
TListProjectIdentityDTO,
TUpdateProjectIdentityDTO
} from "./identity-project-types";
@@ -370,11 +371,48 @@ export const identityProjectServiceFactory = ({
return identityMembership;
};
const getProjectIdentityByMembershipId = async ({
identityMembershipId,
actor,
actorId,
actorAuthMethod,
actorOrgId
}: TGetProjectIdentityByMembershipIdDTO) => {
const membership = await identityProjectDAL.findOne({ id: identityMembershipId });
if (!membership) {
throw new NotFoundError({
message: `Project membership with ID '${identityMembershipId}' not found`
});
}
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
projectId: membership.projectId,
actorAuthMethod,
actorOrgId,
actionProjectType: ActionProjectType.Any
});
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionIdentityActions.Read,
subject(ProjectPermissionSub.Identity, { identityId: membership.identityId })
);
const [identityMembership] = await identityProjectDAL.findByProjectId(membership.projectId, {
identityId: membership.identityId
});
return identityMembership;
};
return {
createProjectIdentity,
updateProjectIdentity,
deleteProjectIdentity,
listProjectIdentities,
getProjectIdentityByIdentityId
getProjectIdentityByIdentityId,
getProjectIdentityByMembershipId
};
};
@@ -52,6 +52,10 @@ export type TGetProjectIdentityByIdentityIdDTO = {
identityId: string;
} & TProjectPermission;
export type TGetProjectIdentityByMembershipIdDTO = {
identityMembershipId: string;
} & Omit<TProjectPermission, "projectId">;
export enum ProjectIdentityOrderBy {
Name = "name"
}
@@ -14,10 +14,15 @@ import {
TIdentityUniversalAuths,
TOrgRoles
} from "@app/db/schemas";
import { DatabaseError } from "@app/lib/errors";
import { BadRequestError, DatabaseError } from "@app/lib/errors";
import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
import { buildKnexFilterForSearchResource } from "@app/lib/search-resource/db";
import { OrderByDirection } from "@app/lib/types";
import { OrgIdentityOrderBy, TListOrgIdentitiesByOrgIdDTO } from "@app/services/identity/identity-types";
import {
OrgIdentityOrderBy,
TListOrgIdentitiesByOrgIdDTO,
TSearchOrgIdentitiesByOrgIdDAL
} from "@app/services/identity/identity-types";
import { buildAuthMethods } from "./identity-fns";
@@ -195,7 +200,6 @@ export const identityOrgDALFactory = (db: TDbClient) => {
"paginatedIdentity.identityId",
`${TableName.IdentityJwtAuth}.identityId`
)
.select(
db.ref("id").withSchema("paginatedIdentity"),
db.ref("role").withSchema("paginatedIdentity"),
@@ -309,6 +313,214 @@ export const identityOrgDALFactory = (db: TDbClient) => {
}
};
const searchIdentities = async (
{
limit,
offset = 0,
orderBy = OrgIdentityOrderBy.Name,
orderDirection = OrderByDirection.ASC,
searchFilter,
orgId
}: TSearchOrgIdentitiesByOrgIdDAL,
tx?: Knex
) => {
try {
const searchQuery = (tx || db.replicaNode())(TableName.IdentityOrgMembership)
.join(TableName.Identity, `${TableName.Identity}.id`, `${TableName.IdentityOrgMembership}.identityId`)
.where(`${TableName.IdentityOrgMembership}.orgId`, orgId)
.leftJoin(TableName.OrgRoles, `${TableName.IdentityOrgMembership}.roleId`, `${TableName.OrgRoles}.id`)
.orderBy(`${TableName.Identity}.${orderBy}`, orderDirection)
.select(`${TableName.IdentityOrgMembership}.id`)
.select<{ id: string; total_count: string }>(
db.raw(
`count(${TableName.IdentityOrgMembership}."identityId") OVER(PARTITION BY ${TableName.IdentityOrgMembership}."orgId") as total_count`
)
)
.as("searchedIdentities");
if (searchFilter) {
buildKnexFilterForSearchResource(searchQuery, searchFilter, (attr) => {
switch (attr) {
case "role":
return [`${TableName.OrgRoles}.slug`, `${TableName.IdentityOrgMembership}.role`];
case "name":
return `${TableName.Identity}.name`;
default:
throw new BadRequestError({ message: `Invalid ${String(attr)} provided` });
}
});
}
if (limit) {
void searchQuery.offset(offset).limit(limit);
}
type TSubquery = Awaited<typeof searchQuery>;
const query = (tx || db.replicaNode())(TableName.IdentityOrgMembership)
.where(`${TableName.IdentityOrgMembership}.orgId`, orgId)
.join<TSubquery>(searchQuery, `${TableName.IdentityOrgMembership}.id`, "searchedIdentities.id")
.join(TableName.Identity, `${TableName.IdentityOrgMembership}.identityId`, `${TableName.Identity}.id`)
.leftJoin(TableName.OrgRoles, `${TableName.IdentityOrgMembership}.roleId`, `${TableName.OrgRoles}.id`)
.leftJoin(TableName.IdentityMetadata, (queryBuilder) => {
void queryBuilder
.on(`${TableName.IdentityOrgMembership}.identityId`, `${TableName.IdentityMetadata}.identityId`)
.andOn(`${TableName.IdentityOrgMembership}.orgId`, `${TableName.IdentityMetadata}.orgId`);
})
.leftJoin(
TableName.IdentityUniversalAuth,
`${TableName.IdentityOrgMembership}.identityId`,
`${TableName.IdentityUniversalAuth}.identityId`
)
.leftJoin(
TableName.IdentityGcpAuth,
`${TableName.IdentityOrgMembership}.identityId`,
`${TableName.IdentityGcpAuth}.identityId`
)
.leftJoin(
TableName.IdentityAwsAuth,
`${TableName.IdentityOrgMembership}.identityId`,
`${TableName.IdentityAwsAuth}.identityId`
)
.leftJoin(
TableName.IdentityKubernetesAuth,
`${TableName.IdentityOrgMembership}.identityId`,
`${TableName.IdentityKubernetesAuth}.identityId`
)
.leftJoin(
TableName.IdentityOidcAuth,
`${TableName.IdentityOrgMembership}.identityId`,
`${TableName.IdentityOidcAuth}.identityId`
)
.leftJoin(
TableName.IdentityAzureAuth,
`${TableName.IdentityOrgMembership}.identityId`,
`${TableName.IdentityAzureAuth}.identityId`
)
.leftJoin(
TableName.IdentityTokenAuth,
`${TableName.IdentityOrgMembership}.identityId`,
`${TableName.IdentityTokenAuth}.identityId`
)
.leftJoin(
TableName.IdentityJwtAuth,
`${TableName.IdentityOrgMembership}.identityId`,
`${TableName.IdentityJwtAuth}.identityId`
)
.select(
db.ref("id").withSchema(TableName.IdentityOrgMembership),
db.ref("total_count").withSchema("searchedIdentities"),
db.ref("role").withSchema(TableName.IdentityOrgMembership),
db.ref("roleId").withSchema(TableName.IdentityOrgMembership),
db.ref("orgId").withSchema(TableName.IdentityOrgMembership),
db.ref("createdAt").withSchema(TableName.IdentityOrgMembership),
db.ref("updatedAt").withSchema(TableName.IdentityOrgMembership),
db.ref("identityId").withSchema(TableName.IdentityOrgMembership).as("identityId"),
db.ref("name").withSchema(TableName.Identity).as("identityName"),
db.ref("id").as("uaId").withSchema(TableName.IdentityUniversalAuth),
db.ref("id").as("gcpId").withSchema(TableName.IdentityGcpAuth),
db.ref("id").as("awsId").withSchema(TableName.IdentityAwsAuth),
db.ref("id").as("kubernetesId").withSchema(TableName.IdentityKubernetesAuth),
db.ref("id").as("oidcId").withSchema(TableName.IdentityOidcAuth),
db.ref("id").as("azureId").withSchema(TableName.IdentityAzureAuth),
db.ref("id").as("tokenId").withSchema(TableName.IdentityTokenAuth),
db.ref("id").as("jwtId").withSchema(TableName.IdentityJwtAuth)
)
// cr stands for custom role
.select(db.ref("id").as("crId").withSchema(TableName.OrgRoles))
.select(db.ref("name").as("crName").withSchema(TableName.OrgRoles))
.select(db.ref("slug").as("crSlug").withSchema(TableName.OrgRoles))
.select(db.ref("description").as("crDescription").withSchema(TableName.OrgRoles))
.select(db.ref("permissions").as("crPermission").withSchema(TableName.OrgRoles))
.select(db.ref("permissions").as("crPermission").withSchema(TableName.OrgRoles))
.select(
db.ref("id").withSchema(TableName.IdentityMetadata).as("metadataId"),
db.ref("key").withSchema(TableName.IdentityMetadata).as("metadataKey"),
db.ref("value").withSchema(TableName.IdentityMetadata).as("metadataValue")
);
if (orderBy === OrgIdentityOrderBy.Name) {
void query.orderBy("identityName", orderDirection);
}
const docs = await query;
const formattedDocs = sqlNestRelationships({
data: docs,
key: "id",
parentMapper: ({
crId,
crDescription,
crSlug,
crPermission,
crName,
identityId,
identityName,
role,
roleId,
total_count,
id,
uaId,
awsId,
gcpId,
jwtId,
kubernetesId,
oidcId,
azureId,
tokenId,
createdAt,
updatedAt
}) => ({
role,
roleId,
identityId,
id,
total_count: total_count as string,
orgId,
createdAt,
updatedAt,
customRole: roleId
? {
id: crId,
name: crName,
slug: crSlug,
permissions: crPermission,
description: crDescription
}
: undefined,
identity: {
id: identityId,
name: identityName,
authMethods: buildAuthMethods({
uaId,
awsId,
gcpId,
kubernetesId,
oidcId,
azureId,
tokenId,
jwtId
})
}
}),
childrenMapper: [
{
key: "metadataId",
label: "metadata" as const,
mapper: ({ metadataKey, metadataValue, metadataId }) => ({
id: metadataId,
key: metadataKey,
value: metadataValue
})
}
]
});
return { docs: formattedDocs, totalCount: Number(formattedDocs?.[0]?.total_count ?? 0) };
} catch (error) {
throw new DatabaseError({ error, name: "FindByOrgId" });
}
};
const countAllOrgIdentities = async (
{ search, ...filter }: Partial<TIdentityOrgMemberships> & Pick<TListOrgIdentitiesByOrgIdDTO, "search">,
tx?: Knex
@@ -331,5 +543,5 @@ export const identityOrgDALFactory = (db: TDbClient) => {
}
};
return { ...identityOrgOrm, find, findOne, countAllOrgIdentities };
return { ...identityOrgOrm, find, findOne, countAllOrgIdentities, searchIdentities };
};
@@ -21,6 +21,7 @@ import {
TGetIdentityByIdDTO,
TListOrgIdentitiesByOrgIdDTO,
TListProjectIdentitiesByIdentityIdDTO,
TSearchOrgIdentitiesByOrgIdDTO,
TUpdateIdentityDTO
} from "./identity-types";
@@ -288,6 +289,33 @@ export const identityServiceFactory = ({
return { identityMemberships, totalCount };
};
const searchOrgIdentities = async ({
orgId,
actor,
actorId,
actorAuthMethod,
actorOrgId,
limit,
offset,
orderBy,
orderDirection,
searchFilter = {}
}: TSearchOrgIdentitiesByOrgIdDTO) => {
const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorAuthMethod, actorOrgId);
ForbiddenError.from(permission).throwUnlessCan(OrgPermissionIdentityActions.Read, OrgPermissionSubjects.Identity);
const { totalCount, docs } = await identityOrgMembershipDAL.searchIdentities({
orgId,
limit,
offset,
orderBy,
orderDirection,
searchFilter
});
return { identityMemberships: docs, totalCount };
};
const listProjectIdentitiesByIdentityId = async ({
identityId,
actor,
@@ -317,6 +345,7 @@ export const identityServiceFactory = ({
deleteIdentity,
listOrgIdentities,
getIdentityById,
searchOrgIdentities,
listProjectIdentitiesByIdentityId
};
};
@@ -1,4 +1,5 @@
import { IPType } from "@app/lib/ip";
import { TSearchResourceOperator } from "@app/lib/search-resource/search";
import { OrderByDirection, TOrgPermission } from "@app/lib/types";
export type TCreateIdentityDTO = {
@@ -46,3 +47,17 @@ export enum OrgIdentityOrderBy {
Name = "name"
// Role = "role"
}
export type TSearchOrgIdentitiesByOrgIdDAL = {
limit?: number;
offset?: number;
orderBy?: OrgIdentityOrderBy;
orderDirection?: OrderByDirection;
orgId: string;
searchFilter?: Partial<{
name: Omit<TSearchResourceOperator, "number">;
role: Omit<TSearchResourceOperator, "number">;
}>;
};
export type TSearchOrgIdentitiesByOrgIdDTO = TSearchOrgIdentitiesByOrgIdDAL & TOrgPermission;
@@ -50,7 +50,7 @@ const getIntegrationSecretsV2 = async (
}
// process secrets in current folder
const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId);
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId: dto.folderId, projectId: dto.projectId });
secrets.forEach((secret) => {
const secretKey = secret.key;
@@ -63,6 +63,7 @@ const getIntegrationSecretsV2 = async (
// if no imports then return secrets in the current folder
if (!secretImports.length) return content;
const importedSecrets = await fnSecretsV2FromImports({
projectId: dto.projectId,
decryptor: dto.decryptor,
folderDAL,
secretDAL: secretV2BridgeDAL,
@@ -63,6 +63,7 @@ export enum IntegrationUrls {
GITHUB_TOKEN_URL = "https://github.com/login/oauth/access_token",
GITLAB_TOKEN_URL = "https://gitlab.com/oauth/token",
BITBUCKET_TOKEN_URL = "https://bitbucket.org/site/oauth2/access_token",
CAMUNDA_TOKEN_URL = "https://login.cloud.camunda.io/oauth/token",
// integration apps endpoints
GCP_API_URL = "https://cloudresourcemanager.googleapis.com",
@@ -94,6 +95,7 @@ export enum IntegrationUrls {
HASURA_CLOUD_API_URL = "https://data.pro.hasura.io/v1/graphql",
AZURE_DEVOPS_API_URL = "https://dev.azure.com",
HUMANITEC_API_URL = "https://api.humanitec.io",
CAMUNDA_API_URL = "https://api.cloud.camunda.io",
GCP_SECRET_MANAGER_SERVICE_NAME = "secretmanager.googleapis.com",
GCP_SECRET_MANAGER_URL = `https://${GCP_SECRET_MANAGER_SERVICE_NAME}`,
+46 -4
View File
@@ -1,13 +1,55 @@
import { SymmetricEncryption } from "@app/lib/crypto/cipher";
import { SymmetricKeyAlgorithm } from "@app/lib/crypto/cipher";
import { AsymmetricKeyAlgorithm } from "@app/lib/crypto/sign";
import { BadRequestError } from "@app/lib/errors";
import { KmsKeyUsage } from "./kms-types";
export const KMS_ROOT_CONFIG_UUID = "00000000-0000-0000-0000-000000000000";
export const getByteLengthForAlgorithm = (encryptionAlgorithm: SymmetricEncryption) => {
export const getByteLengthForSymmetricEncryptionAlgorithm = (encryptionAlgorithm: SymmetricKeyAlgorithm) => {
switch (encryptionAlgorithm) {
case SymmetricEncryption.AES_GCM_128:
case SymmetricKeyAlgorithm.AES_GCM_128:
return 16;
case SymmetricEncryption.AES_GCM_256:
case SymmetricKeyAlgorithm.AES_GCM_256:
default:
return 32;
}
};
export const verifyKeyTypeAndAlgorithm = (
keyUsage: KmsKeyUsage,
algorithm: SymmetricKeyAlgorithm | AsymmetricKeyAlgorithm,
extra?: {
forceType?: KmsKeyUsage;
}
) => {
if (extra?.forceType && keyUsage !== extra.forceType) {
throw new BadRequestError({
message: `Unsupported key type, expected ${extra.forceType} but got ${keyUsage}`
});
}
if (keyUsage === KmsKeyUsage.ENCRYPT_DECRYPT) {
if (!Object.values(SymmetricKeyAlgorithm).includes(algorithm as SymmetricKeyAlgorithm)) {
throw new BadRequestError({
message: `Unsupported encryption algorithm for encrypt/decrypt key: ${algorithm as string}`
});
}
return true;
}
if (keyUsage === KmsKeyUsage.SIGN_VERIFY) {
if (!Object.values(AsymmetricKeyAlgorithm).includes(algorithm as AsymmetricKeyAlgorithm)) {
throw new BadRequestError({
message: `Unsupported sign/verify algorithm for sign/verify key: ${algorithm as string}`
});
}
return true;
}
throw new BadRequestError({
message: `Unsupported key type: ${keyUsage as string}`
});
};
+141 -22
View File
@@ -15,12 +15,17 @@ import { THsmServiceFactory } from "@app/ee/services/hsm/hsm-service";
import { KeyStorePrefixes, PgSqlLock, TKeyStoreFactory } from "@app/keystore/keystore";
import { TEnvConfig } from "@app/lib/config/env";
import { randomSecureBytes } from "@app/lib/crypto";
import { symmetricCipherService, SymmetricEncryption } from "@app/lib/crypto/cipher";
import { symmetricCipherService, SymmetricKeyAlgorithm } from "@app/lib/crypto/cipher";
import { generateHash } from "@app/lib/crypto/encryption";
import { AsymmetricKeyAlgorithm, signingService } from "@app/lib/crypto/sign";
import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { getByteLengthForAlgorithm, KMS_ROOT_CONFIG_UUID } from "@app/services/kms/kms-fns";
import {
getByteLengthForSymmetricEncryptionAlgorithm,
KMS_ROOT_CONFIG_UUID,
verifyKeyTypeAndAlgorithm
} from "@app/services/kms/kms-fns";
import { TOrgDALFactory } from "../org/org-dal";
import { TProjectDALFactory } from "../project/project-dal";
@@ -29,6 +34,7 @@ import { TKmsKeyDALFactory } from "./kms-key-dal";
import { TKmsRootConfigDALFactory } from "./kms-root-config-dal";
import {
KmsDataKey,
KmsKeyUsage,
KmsType,
RootKeyEncryptionStrategy,
TDecryptWithKeyDTO,
@@ -38,8 +44,11 @@ import {
TEncryptWithKmsDTO,
TGenerateKMSDTO,
TGetKeyMaterialDTO,
TGetPublicKeyDTO,
TImportKeyMaterialDTO,
TUpdateProjectSecretManagerKmsKeyDTO
TSignWithKmsDTO,
TUpdateProjectSecretManagerKmsKeyDTO,
TVerifyWithKmsDTO
} from "./kms-types";
type TKmsServiceFactoryDep = {
@@ -83,19 +92,42 @@ export const kmsServiceFactory = ({
tx,
name,
projectId,
encryptionAlgorithm = SymmetricEncryption.AES_GCM_256,
encryptionAlgorithm = SymmetricKeyAlgorithm.AES_GCM_256,
keyUsage = KmsKeyUsage.ENCRYPT_DECRYPT,
description
}: TGenerateKMSDTO) => {
const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
// daniel: ensure that the key type (sign/encrypt) and the encryption algorithm are compatible.
verifyKeyTypeAndAlgorithm(keyUsage, encryptionAlgorithm);
const kmsKeyMaterial = randomSecureBytes(getByteLengthForAlgorithm(encryptionAlgorithm));
let kmsKeyMaterial: Buffer | null = null;
if (keyUsage === KmsKeyUsage.ENCRYPT_DECRYPT) {
kmsKeyMaterial = randomSecureBytes(
getByteLengthForSymmetricEncryptionAlgorithm(encryptionAlgorithm as SymmetricKeyAlgorithm)
);
} else if (keyUsage === KmsKeyUsage.SIGN_VERIFY) {
const { generateAsymmetricPrivateKey, getPublicKeyFromPrivateKey } = signingService(
encryptionAlgorithm as AsymmetricKeyAlgorithm
);
kmsKeyMaterial = await generateAsymmetricPrivateKey();
// daniel: safety check to ensure we're able to extract the public key from the private key before we proceed to key creation
getPublicKeyFromPrivateKey(kmsKeyMaterial);
}
if (!kmsKeyMaterial) {
throw new BadRequestError({
message: `Invalid KMS key type. No key material was created for key usage '${keyUsage}' using algorithm '${encryptionAlgorithm}'`
});
}
const cipher = symmetricCipherService(SymmetricKeyAlgorithm.AES_GCM_256);
const encryptedKeyMaterial = cipher.encrypt(kmsKeyMaterial, ROOT_ENCRYPTION_KEY);
const sanitizedName = name ? slugify(name) : slugify(alphaNumericNanoId(8).toLowerCase());
const dbQuery = async (db: Knex) => {
const kmsDoc = await kmsDAL.create(
{
name: sanitizedName,
keyUsage,
orgId,
isReserved,
projectId,
@@ -115,6 +147,7 @@ export const kmsServiceFactory = ({
);
return kmsDoc;
};
if (tx) return dbQuery(tx);
const doc = await kmsDAL.transaction(async (tx2) => dbQuery(tx2));
return doc;
@@ -134,7 +167,7 @@ export const kmsServiceFactory = ({
*/
const encryptWithInputKey = async ({ key }: Omit<TEncryptionWithKeyDTO, "plainText">) => {
// akhilmhdh: as more encryption are added do a check here on kmsDoc.encryptionAlgorithm
const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
const cipher = symmetricCipherService(SymmetricKeyAlgorithm.AES_GCM_256);
return ({ plainText }: Pick<TEncryptWithKmsDTO, "plainText">) => {
const encryptedPlainTextBlob = cipher.encrypt(plainText, key);
// Buffer#1 encrypted text + Buffer#2 version number
@@ -149,7 +182,7 @@ export const kmsServiceFactory = ({
* This can be even later exposed directly as api for encryption as function
*/
const decryptWithInputKey = async ({ key }: Omit<TDecryptWithKeyDTO, "cipherTextBlob">) => {
const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
const cipher = symmetricCipherService(SymmetricKeyAlgorithm.AES_GCM_256);
return ({ cipherTextBlob: versionedCipherTextBlob }: Pick<TDecryptWithKeyDTO, "cipherTextBlob">) => {
const cipherTextBlob = versionedCipherTextBlob.subarray(0, -KMS_VERSION_BLOB_LENGTH);
@@ -227,7 +260,7 @@ export const kmsServiceFactory = ({
};
const encryptWithRootKey = () => {
const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
const cipher = symmetricCipherService(SymmetricKeyAlgorithm.AES_GCM_256);
return (plainTextBuffer: Buffer) => {
const encryptedBuffer = cipher.encrypt(plainTextBuffer, ROOT_ENCRYPTION_KEY);
@@ -236,7 +269,7 @@ export const kmsServiceFactory = ({
};
const decryptWithRootKey = () => {
const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
const cipher = symmetricCipherService(SymmetricKeyAlgorithm.AES_GCM_256);
return (cipherTextBuffer: Buffer) => {
return cipher.decrypt(cipherTextBuffer, ROOT_ENCRYPTION_KEY);
@@ -255,6 +288,11 @@ export const kmsServiceFactory = ({
throw new NotFoundError({ message: `KMS with ID '${kmsId}' not found` });
}
const encryptionAlgorithm = kmsDoc.internalKms?.encryptionAlgorithm as SymmetricKeyAlgorithm;
verifyKeyTypeAndAlgorithm(kmsDoc.keyUsage as KmsKeyUsage, encryptionAlgorithm, {
forceType: KmsKeyUsage.ENCRYPT_DECRYPT
});
if (kmsDoc.externalKms) {
let externalKms: TExternalKmsProviderFns;
@@ -316,8 +354,8 @@ export const kmsServiceFactory = ({
}
// internal KMS
const keyCipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
const dataCipher = symmetricCipherService(kmsDoc.internalKms?.encryptionAlgorithm as SymmetricEncryption);
const keyCipher = symmetricCipherService(SymmetricKeyAlgorithm.AES_GCM_256);
const dataCipher = symmetricCipherService(encryptionAlgorithm);
const kmsKey = keyCipher.decrypt(kmsDoc.internalKms?.encryptedKey as Buffer, ROOT_ENCRYPTION_KEY);
return ({ cipherTextBlob: versionedCipherTextBlob }: Pick<TDecryptWithKmsDTO, "cipherTextBlob">) => {
@@ -345,19 +383,22 @@ export const kmsServiceFactory = ({
});
}
const keyCipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
const keyCipher = symmetricCipherService(SymmetricKeyAlgorithm.AES_GCM_256);
const kmsKey = keyCipher.decrypt(kmsDoc.internalKms?.encryptedKey as Buffer, ROOT_ENCRYPTION_KEY);
return kmsKey;
};
const importKeyMaterial = async (
{ key, algorithm, name, isReserved, projectId, orgId }: TImportKeyMaterialDTO,
{ key, algorithm, name, isReserved, projectId, orgId, keyUsage }: TImportKeyMaterialDTO,
tx?: Knex
) => {
const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
// daniel: currently we only support imports for encrypt/decrypt keys
verifyKeyTypeAndAlgorithm(keyUsage, algorithm, { forceType: KmsKeyUsage.ENCRYPT_DECRYPT });
const expectedByteLength = getByteLengthForAlgorithm(algorithm);
const cipher = symmetricCipherService(SymmetricKeyAlgorithm.AES_GCM_256);
const expectedByteLength = getByteLengthForSymmetricEncryptionAlgorithm(algorithm as SymmetricKeyAlgorithm);
if (key.byteLength !== expectedByteLength) {
throw new BadRequestError({
message: `Invalid key length for ${algorithm}. Expected ${expectedByteLength} bytes but got ${key.byteLength} bytes`
@@ -370,6 +411,7 @@ export const kmsServiceFactory = ({
const kmsDoc = await kmsDAL.create(
{
name: sanitizedName,
keyUsage: KmsKeyUsage.ENCRYPT_DECRYPT,
orgId,
isReserved,
projectId
@@ -393,12 +435,85 @@ export const kmsServiceFactory = ({
return doc;
};
const getPublicKey = async ({ kmsId }: TGetPublicKeyDTO) => {
const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId);
if (!kmsDoc) {
throw new NotFoundError({ message: `KMS with ID '${kmsId}' not found` });
}
const encryptionAlgorithm = kmsDoc.internalKms?.encryptionAlgorithm as AsymmetricKeyAlgorithm;
verifyKeyTypeAndAlgorithm(kmsDoc.keyUsage as KmsKeyUsage, encryptionAlgorithm, {
forceType: KmsKeyUsage.SIGN_VERIFY
});
const keyCipher = symmetricCipherService(SymmetricKeyAlgorithm.AES_GCM_256);
const kmsKey = keyCipher.decrypt(kmsDoc.internalKms?.encryptedKey as Buffer, ROOT_ENCRYPTION_KEY);
return signingService(encryptionAlgorithm).getPublicKeyFromPrivateKey(kmsKey);
};
const signWithKmsKey = async ({ kmsId }: Pick<TSignWithKmsDTO, "kmsId">) => {
const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId);
if (!kmsDoc) {
throw new NotFoundError({ message: `KMS with ID '${kmsId}' not found` });
}
const encryptionAlgorithm = kmsDoc.internalKms?.encryptionAlgorithm as AsymmetricKeyAlgorithm;
verifyKeyTypeAndAlgorithm(kmsDoc.keyUsage as KmsKeyUsage, encryptionAlgorithm, {
forceType: KmsKeyUsage.SIGN_VERIFY
});
const keyCipher = symmetricCipherService(SymmetricKeyAlgorithm.AES_GCM_256);
const { sign } = signingService(encryptionAlgorithm);
return async ({
data,
signingAlgorithm,
isDigest
}: Pick<TSignWithKmsDTO, "data" | "signingAlgorithm" | "isDigest">) => {
const kmsKey = keyCipher.decrypt(kmsDoc.internalKms?.encryptedKey as Buffer, ROOT_ENCRYPTION_KEY);
const signature = await sign(data, kmsKey, signingAlgorithm, isDigest);
return Promise.resolve({ signature, algorithm: signingAlgorithm });
};
};
const verifyWithKmsKey = async ({
kmsId,
signingAlgorithm
}: Pick<TVerifyWithKmsDTO, "kmsId" | "signingAlgorithm">) => {
const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId);
if (!kmsDoc) {
throw new NotFoundError({ message: `KMS with ID '${kmsId}' not found` });
}
const encryptionAlgorithm = kmsDoc.internalKms?.encryptionAlgorithm as AsymmetricKeyAlgorithm;
verifyKeyTypeAndAlgorithm(kmsDoc.keyUsage as KmsKeyUsage, encryptionAlgorithm, {
forceType: KmsKeyUsage.SIGN_VERIFY
});
const keyCipher = symmetricCipherService(SymmetricKeyAlgorithm.AES_GCM_256);
const { verify, getPublicKeyFromPrivateKey } = signingService(encryptionAlgorithm);
return async ({ data, signature, isDigest }: Pick<TVerifyWithKmsDTO, "data" | "signature" | "isDigest">) => {
const kmsKey = keyCipher.decrypt(kmsDoc.internalKms?.encryptedKey as Buffer, ROOT_ENCRYPTION_KEY);
const publicKey = getPublicKeyFromPrivateKey(kmsKey);
const signatureValid = await verify(data, signature, publicKey, signingAlgorithm, isDigest);
return Promise.resolve({ signatureValid, algorithm: signingAlgorithm });
};
};
const encryptWithKmsKey = async ({ kmsId }: Omit<TEncryptWithKmsDTO, "plainText">, tx?: Knex) => {
const kmsDoc = await kmsDAL.findByIdWithAssociatedKms(kmsId, tx);
if (!kmsDoc) {
throw new NotFoundError({ message: `KMS with ID '${kmsId}' not found` });
}
const encryptionAlgorithm = kmsDoc.internalKms?.encryptionAlgorithm as SymmetricKeyAlgorithm;
verifyKeyTypeAndAlgorithm(kmsDoc.keyUsage as KmsKeyUsage, encryptionAlgorithm, {
forceType: KmsKeyUsage.ENCRYPT_DECRYPT
});
if (kmsDoc.externalKms) {
let externalKms: TExternalKmsProviderFns;
if (!kmsDoc.orgKms.id || !kmsDoc.orgKms.encryptedDataKey) {
@@ -454,8 +569,8 @@ export const kmsServiceFactory = ({
}
// internal KMS
const keyCipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
const dataCipher = symmetricCipherService(kmsDoc.internalKms?.encryptionAlgorithm as SymmetricEncryption);
const keyCipher = symmetricCipherService(SymmetricKeyAlgorithm.AES_GCM_256);
const dataCipher = symmetricCipherService(encryptionAlgorithm);
return ({ plainText }: Pick<TEncryptWithKmsDTO, "plainText">) => {
const kmsKey = keyCipher.decrypt(kmsDoc.internalKms?.encryptedKey as Buffer, ROOT_ENCRYPTION_KEY);
const encryptedPlainTextBlob = dataCipher.encrypt(plainText, kmsKey);
@@ -729,7 +844,7 @@ export const kmsServiceFactory = ({
// case 2: root key is encrypted with software encryption
if (kmsRootConfig.encryptionStrategy === RootKeyEncryptionStrategy.Software) {
const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
const cipher = symmetricCipherService(SymmetricKeyAlgorithm.AES_GCM_256);
const encryptionKeyBuffer = $getBasicEncryptionKey();
return cipher.decrypt(kmsRootConfig.encryptedRootKey, encryptionKeyBuffer);
@@ -749,7 +864,7 @@ export const kmsServiceFactory = ({
}
if (strategy === RootKeyEncryptionStrategy.Software) {
const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
const cipher = symmetricCipherService(SymmetricKeyAlgorithm.AES_GCM_256);
const encryptionKeyBuffer = $getBasicEncryptionKey();
return cipher.encrypt(plainKeyBuffer, encryptionKeyBuffer);
@@ -765,7 +880,7 @@ export const kmsServiceFactory = ({
const createCipherPairWithDataKey = async (encryptionContext: TEncryptWithKmsDataKeyDTO, trx?: Knex) => {
const dataKey = await $getDataKey(encryptionContext, trx);
const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
const cipher = symmetricCipherService(SymmetricKeyAlgorithm.AES_GCM_256);
return {
encryptor: ({ plainText }: Pick<TEncryptWithKmsDTO, "plainText">) => {
@@ -966,6 +1081,7 @@ export const kmsServiceFactory = ({
const decryptedRootKey = await $decryptRootKey(kmsRootConfig);
logger.info("KMS: Loading ROOT Key into Memory.");
ROOT_ENCRYPTION_KEY = decryptedRootKey;
};
@@ -1014,6 +1130,9 @@ export const kmsServiceFactory = ({
getKmsById,
createCipherPairWithDataKey,
getKeyMaterial,
importKeyMaterial
importKeyMaterial,
signWithKmsKey,
verifyWithKmsKey,
getPublicKey
};
};
+30 -3
View File
@@ -1,6 +1,7 @@
import { Knex } from "knex";
import { SymmetricEncryption } from "@app/lib/crypto/cipher";
import { SymmetricKeyAlgorithm } from "@app/lib/crypto/cipher";
import { AsymmetricKeyAlgorithm, SigningAlgorithm } from "@app/lib/crypto/sign/types";
export enum KmsDataKey {
Organization,
@@ -13,6 +14,11 @@ export enum KmsType {
Internal = "internal"
}
export enum KmsKeyUsage {
ENCRYPT_DECRYPT = "encrypt-decrypt",
SIGN_VERIFY = "sign-verify"
}
export type TEncryptWithKmsDataKeyDTO =
| { type: KmsDataKey.Organization; orgId: string }
| { type: KmsDataKey.SecretManager; projectId: string };
@@ -25,7 +31,8 @@ export type TEncryptWithKmsDataKeyDTO =
export type TGenerateKMSDTO = {
orgId: string;
projectId?: string;
encryptionAlgorithm?: SymmetricEncryption;
encryptionAlgorithm?: SymmetricKeyAlgorithm | AsymmetricKeyAlgorithm;
keyUsage?: KmsKeyUsage;
isReserved?: boolean;
name?: string;
description?: string;
@@ -37,6 +44,25 @@ export type TEncryptWithKmsDTO = {
plainText: Buffer;
};
export type TGetPublicKeyDTO = {
kmsId: string;
};
export type TSignWithKmsDTO = {
kmsId: string;
data: Buffer;
signingAlgorithm: SigningAlgorithm;
isDigest: boolean;
};
export type TVerifyWithKmsDTO = {
kmsId: string;
data: Buffer;
signature: Buffer;
signingAlgorithm: SigningAlgorithm;
isDigest: boolean;
};
export type TEncryptionWithKeyDTO = {
key: Buffer;
plainText: Buffer;
@@ -67,9 +93,10 @@ export type TGetKeyMaterialDTO = {
export type TImportKeyMaterialDTO = {
key: Buffer;
algorithm: SymmetricEncryption;
algorithm: SymmetricKeyAlgorithm;
name?: string;
isReserved: boolean;
projectId: string;
orgId: string;
keyUsage: KmsKeyUsage;
};
@@ -141,6 +141,7 @@ export const projectRoleServiceFactory = ({
validateHandlebarTemplate("Project Role Update", JSON.stringify(data.permissions || []), {
allowedExpressions: (val) => val.includes("identity.")
});
const updatedRole = await projectRoleDAL.updateById(projectRole.id, {
...data,
permissions: data.permissions ? data.permissions : undefined
+49 -1
View File
@@ -1,12 +1,15 @@
import crypto from "crypto";
import { ProjectVersion, TProjects } from "@app/db/schemas";
import { createSshCaHelper } from "@app/ee/services/ssh/ssh-certificate-authority-fns";
import { SshCaKeySource } from "@app/ee/services/ssh/ssh-certificate-authority-types";
import { SshCertKeyAlgorithm } from "@app/ee/services/ssh-certificate/ssh-certificate-types";
import { decryptAsymmetric, encryptAsymmetric } from "@app/lib/crypto";
import { NotFoundError } from "@app/lib/errors";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { AddUserToWsDTO } from "./project-types";
import { AddUserToWsDTO, TBootstrapSshProjectDTO } from "./project-types";
export const assignWorkspaceKeysToMembers = ({ members, decryptKey, userPrivateKey }: AddUserToWsDTO) => {
const plaintextProjectKey = decryptAsymmetric({
@@ -102,3 +105,48 @@ export const getProjectKmsCertificateKeyId = async ({
return keyId;
};
/**
* Bootstraps an SSH project.
* - Creates a user and host SSH CA
* - Creates a project SSH config with the user and host SSH CA as defaults
*/
export const bootstrapSshProject = async ({
projectId,
sshCertificateAuthorityDAL,
sshCertificateAuthoritySecretDAL,
kmsService,
projectSshConfigDAL,
tx
}: TBootstrapSshProjectDTO) => {
const userSshCa = await createSshCaHelper({
projectId,
friendlyName: "User CA",
keyAlgorithm: SshCertKeyAlgorithm.ED25519,
keySource: SshCaKeySource.INTERNAL,
sshCertificateAuthorityDAL,
sshCertificateAuthoritySecretDAL,
kmsService,
tx
});
const hostSshCa = await createSshCaHelper({
projectId,
friendlyName: "Host CA",
keyAlgorithm: SshCertKeyAlgorithm.ED25519,
keySource: SshCaKeySource.INTERNAL,
sshCertificateAuthorityDAL,
sshCertificateAuthoritySecretDAL,
kmsService,
tx
});
await projectSshConfigDAL.create(
{
projectId,
defaultHostSshCaId: hostSshCa.id,
defaultUserSshCaId: userSshCa.id
},
tx
);
};
@@ -1,4 +1,4 @@
import { ForbiddenError } from "@casl/ability";
import { ForbiddenError, subject } from "@casl/ability";
import slugify from "@sindresorhus/slugify";
import {
@@ -15,13 +15,16 @@ import { TPermissionServiceFactory } from "@app/ee/services/permission/permissio
import {
ProjectPermissionActions,
ProjectPermissionSecretActions,
ProjectPermissionSshHostActions,
ProjectPermissionSub
} from "@app/ee/services/permission/project-permission";
import { TProjectTemplateServiceFactory } from "@app/ee/services/project-template/project-template-service";
import { InfisicalProjectTemplate } from "@app/ee/services/project-template/project-template-types";
import { TSshCertificateAuthorityDALFactory } from "@app/ee/services/ssh/ssh-certificate-authority-dal";
import { TSshCertificateAuthoritySecretDALFactory } from "@app/ee/services/ssh/ssh-certificate-authority-secret-dal";
import { TSshCertificateDALFactory } from "@app/ee/services/ssh-certificate/ssh-certificate-dal";
import { TSshCertificateTemplateDALFactory } from "@app/ee/services/ssh-certificate-template/ssh-certificate-template-dal";
import { TSshHostDALFactory } from "@app/ee/services/ssh-host/ssh-host-dal";
import { TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption";
@@ -61,8 +64,9 @@ import { TSlackIntegrationDALFactory } from "../slack/slack-integration-dal";
import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service";
import { TUserDALFactory } from "../user/user-dal";
import { TProjectDALFactory } from "./project-dal";
import { assignWorkspaceKeysToMembers, createProjectKey } from "./project-fns";
import { assignWorkspaceKeysToMembers, bootstrapSshProject, createProjectKey } from "./project-fns";
import { TProjectQueueFactory } from "./project-queue";
import { TProjectSshConfigDALFactory } from "./project-ssh-config-dal";
import {
TCreateProjectDTO,
TDeleteProjectDTO,
@@ -77,6 +81,7 @@ import {
TListProjectSshCasDTO,
TListProjectSshCertificatesDTO,
TListProjectSshCertificateTemplatesDTO,
TListProjectSshHostsDTO,
TLoadProjectKmsBackupDTO,
TProjectAccessRequestDTO,
TSearchProjectsDTO,
@@ -97,8 +102,8 @@ export const DEFAULT_PROJECT_ENVS = [
];
type TProjectServiceFactoryDep = {
// TODO: Pick
projectDAL: TProjectDALFactory;
projectSshConfigDAL: Pick<TProjectSshConfigDALFactory, "create">;
projectQueue: TProjectQueueFactory;
userDAL: TUserDALFactory;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
@@ -123,9 +128,11 @@ type TProjectServiceFactoryDep = {
certificateTemplateDAL: Pick<TCertificateTemplateDALFactory, "getCertTemplatesByProjectId">;
pkiAlertDAL: Pick<TPkiAlertDALFactory, "find">;
pkiCollectionDAL: Pick<TPkiCollectionDALFactory, "find">;
sshCertificateAuthorityDAL: Pick<TSshCertificateAuthorityDALFactory, "find">;
sshCertificateAuthorityDAL: Pick<TSshCertificateAuthorityDALFactory, "find" | "create" | "transaction">;
sshCertificateAuthoritySecretDAL: Pick<TSshCertificateAuthoritySecretDALFactory, "create">;
sshCertificateDAL: Pick<TSshCertificateDALFactory, "find" | "countSshCertificatesInProject">;
sshCertificateTemplateDAL: Pick<TSshCertificateTemplateDALFactory, "find">;
sshHostDAL: Pick<TSshHostDALFactory, "find" | "findSshHostsWithLoginMappings">;
permissionService: TPermissionServiceFactory;
orgService: Pick<TOrgServiceFactory, "addGhostUser">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
@@ -144,6 +151,7 @@ type TProjectServiceFactoryDep = {
| "getKmsById"
| "getProjectSecretManagerKmsKeyId"
| "deleteInternalKms"
| "createCipherPairWithDataKey"
>;
projectTemplateService: TProjectTemplateServiceFactory;
};
@@ -152,6 +160,7 @@ export type TProjectServiceFactory = ReturnType<typeof projectServiceFactory>;
export const projectServiceFactory = ({
projectDAL,
projectSshConfigDAL,
secretDAL,
secretV2BridgeDAL,
projectQueue,
@@ -177,8 +186,10 @@ export const projectServiceFactory = ({
pkiCollectionDAL,
pkiAlertDAL,
sshCertificateAuthorityDAL,
sshCertificateAuthoritySecretDAL,
sshCertificateDAL,
sshCertificateTemplateDAL,
sshHostDAL,
keyStore,
kmsService,
projectBotDAL,
@@ -266,6 +277,17 @@ export const projectServiceFactory = ({
tx
);
if (type === ProjectType.SSH) {
await bootstrapSshProject({
projectId: project.id,
sshCertificateAuthorityDAL,
sshCertificateAuthoritySecretDAL,
kmsService,
projectSshConfigDAL,
tx
});
}
// set ghost user as admin of project
const projectMembership = await projectMembershipDAL.create(
{
@@ -1046,6 +1068,48 @@ export const projectServiceFactory = ({
return cas;
};
/**
* Return list of SSH hosts for project
*/
const listProjectSshHosts = async ({
actorId,
actorOrgId,
actorAuthMethod,
actor,
projectId
}: TListProjectSshHostsDTO) => {
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId,
actionProjectType: ActionProjectType.SSH
});
const allowedHosts = [];
// (dangtony98): room to optimize
const hosts = await sshHostDAL.findSshHostsWithLoginMappings(projectId);
for (const host of hosts) {
try {
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionSshHostActions.Read,
subject(ProjectPermissionSub.SshHosts, {
hostname: host.hostname
})
);
allowedHosts.push(host);
} catch {
// intentionally ignore projects where user lacks access
}
}
return allowedHosts;
};
/**
* Return list of SSH certificates for project
*/
@@ -1443,6 +1507,7 @@ export const projectServiceFactory = ({
listProjectPkiCollections,
listProjectCertificateTemplates,
listProjectSshCas,
listProjectSshHosts,
listProjectSshCertificates,
listProjectSshCertificateTemplates,
updateVersionLimit,
@@ -0,0 +1,11 @@
import { TDbClient } from "@app/db";
import { TableName } from "@app/db/schemas";
import { ormify } from "@app/lib/knex";
export type TProjectSshConfigDALFactory = ReturnType<typeof projectSshConfigDALFactory>;
export const projectSshConfigDALFactory = (db: TDbClient) => {
const projectSshConfigOrm = ormify(db, TableName.ProjectSshConfig);
return projectSshConfigOrm;
};
+15 -1
View File
@@ -1,6 +1,10 @@
import { Knex } from "knex";
import { ProjectType, SortDirection, TProjectKeys } from "@app/db/schemas";
import { ProjectType, TProjectKeys, SortDirection } from "@app/db/schemas";
import { TSshCertificateAuthorityDALFactory } from "@app/ee/services/ssh/ssh-certificate-authority-dal";
import { TSshCertificateAuthoritySecretDALFactory } from "@app/ee/services/ssh/ssh-certificate-authority-secret-dal";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { TProjectSshConfigDALFactory } from "@app/services/project/project-ssh-config-dal";
import { OrgServiceActor, TProjectPermission } from "@app/lib/types";
import { ActorAuthMethod, ActorType } from "../auth/auth-type";
@@ -143,6 +147,7 @@ export type TGetProjectKmsKey = TProjectPermission;
export type TListProjectCertificateTemplatesDTO = TProjectPermission;
export type TListProjectSshCasDTO = TProjectPermission;
export type TListProjectSshHostsDTO = TProjectPermission;
export type TListProjectSshCertificateTemplatesDTO = TProjectPermission;
export type TListProjectSshCertificatesDTO = {
offset: number;
@@ -159,6 +164,15 @@ export type TUpdateProjectSlackConfig = {
secretRequestChannels: string;
} & TProjectPermission;
export type TBootstrapSshProjectDTO = {
projectId: string;
sshCertificateAuthorityDAL: Pick<TSshCertificateAuthorityDALFactory, "transaction" | "create">;
sshCertificateAuthoritySecretDAL: Pick<TSshCertificateAuthoritySecretDALFactory, "create">;
projectSshConfigDAL: Pick<TProjectSshConfigDALFactory, "create">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
tx?: Knex;
};
export enum SearchProjectSortBy {
NAME = "name"
}
@@ -159,7 +159,8 @@ export const fnSecretsV2FromImports = async ({
decryptor,
expandSecretReferences,
hasSecretAccess,
viewSecretValue
viewSecretValue,
projectId
}: {
secretImports: (Omit<TSecretImports, "importEnv"> & {
importEnv: { id: string; slug: string; name: string };
@@ -176,6 +177,7 @@ export const fnSecretsV2FromImports = async ({
environment: string;
}) => Promise<string | undefined>;
hasSecretAccess: (environment: string, secretPath: string, secretName: string, secretTagSlugs: string[]) => boolean;
projectId: string;
}) => {
const cyclicDetector = new Set();
const stack: {
@@ -216,7 +218,8 @@ export const fnSecretsV2FromImports = async ({
type: SecretType.Shared
},
{
sort: [["id", "asc"]]
sort: [["id", "asc"]],
useCache: { projectId }
}
);
const importedSecretsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId);
@@ -44,7 +44,7 @@ type TSecretImportServiceFactoryDep = {
secretImportDAL: TSecretImportDALFactory;
folderDAL: TSecretFolderDALFactory;
secretDAL: Pick<TSecretDALFactory, "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find" | "invalidateSecretCacheByProjectId">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
projectEnvDAL: TProjectEnvDALFactory;
@@ -185,6 +185,7 @@ export const secretImportServiceFactory = ({
});
}
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
return { ...secImport, importEnv };
};
@@ -282,6 +283,8 @@ export const secretImportServiceFactory = ({
);
return doc;
});
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
return { ...updatedSecImport, importEnv: importedEnv };
};
@@ -356,6 +359,7 @@ export const secretImportServiceFactory = ({
actorId
});
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
return secImport;
};
@@ -694,6 +698,7 @@ export const secretImportServiceFactory = ({
projectId
});
const importedSecrets = await fnSecretsV2FromImports({
projectId,
secretImports,
folderDAL,
viewSecretValue: true,
@@ -0,0 +1,10 @@
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import { TSecretSyncListItem } from "@app/services/secret-sync/secret-sync-types";
export const CAMUNDA_SYNC_LIST_OPTION: TSecretSyncListItem = {
name: "Camunda",
destination: SecretSync.Camunda,
connection: AppConnection.Camunda,
canImportSecrets: true
};
@@ -0,0 +1,173 @@
import { request } from "@app/lib/config/request";
import { TAppConnectionDALFactory } from "@app/services/app-connection/app-connection-dal";
import { getCamundaConnectionAccessToken } from "@app/services/app-connection/camunda";
import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import {
TCamundaCreateSecret,
TCamundaDeleteSecret,
TCamundaListSecrets,
TCamundaListSecretsResponse,
TCamundaPutSecret,
TCamundaSyncWithCredentials
} from "@app/services/secret-sync/camunda/camunda-sync-types";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import { TSecretMap } from "../secret-sync-types";
type TCamundaSecretSyncFactoryDeps = {
appConnectionDAL: Pick<TAppConnectionDALFactory, "updateById">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
};
const getCamundaSecrets = async ({ accessToken, clusterUUID }: TCamundaListSecrets) => {
const { data } = await request.get<TCamundaListSecretsResponse>(
`${IntegrationUrls.CAMUNDA_API_URL}/clusters/${clusterUUID}/secrets`,
{
headers: {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json"
}
}
);
return data;
};
const createCamundaSecret = async ({ accessToken, clusterUUID, key, value }: TCamundaCreateSecret) =>
request.post(
`${IntegrationUrls.CAMUNDA_API_URL}/clusters/${clusterUUID}/secrets`,
{
secretName: key,
secretValue: value
},
{
headers: {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json"
}
}
);
const deleteCamundaSecret = async ({ accessToken, clusterUUID, key }: TCamundaDeleteSecret) =>
request.delete(`${IntegrationUrls.CAMUNDA_API_URL}/clusters/${clusterUUID}/secrets/${key}`, {
headers: {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json"
}
});
const updateCamundaSecret = async ({ accessToken, clusterUUID, key, value }: TCamundaPutSecret) =>
request.put(
`${IntegrationUrls.CAMUNDA_API_URL}/clusters/${clusterUUID}/secrets/${key}`,
{
secretValue: value
},
{
headers: {
Authorization: `Bearer ${accessToken}`,
"Accept-Encoding": "application/json"
}
}
);
export const camundaSyncFactory = ({ kmsService, appConnectionDAL }: TCamundaSecretSyncFactoryDeps) => {
const syncSecrets = async (secretSync: TCamundaSyncWithCredentials, secretMap: TSecretMap) => {
const {
destinationConfig: { clusterUUID },
connection
} = secretSync;
const accessToken = await getCamundaConnectionAccessToken(connection, appConnectionDAL, kmsService);
const camundaSecrets = await getCamundaSecrets({ accessToken, clusterUUID });
for await (const entry of Object.entries(secretMap)) {
const [key, { value }] = entry;
if (!value) {
// eslint-disable-next-line no-continue
continue;
}
try {
if (camundaSecrets[key] === undefined) {
await createCamundaSecret({
key,
value,
clusterUUID,
accessToken
});
} else if (camundaSecrets[key] !== value) {
await updateCamundaSecret({
key,
value,
clusterUUID,
accessToken
});
}
} catch (error) {
throw new SecretSyncError({
error,
secretKey: key
});
}
}
if (secretSync.syncOptions.disableSecretDeletion) return;
for await (const secret of Object.keys(camundaSecrets)) {
if (!(secret in secretMap) || !secretMap[secret].value) {
try {
await deleteCamundaSecret({
key: secret,
clusterUUID,
accessToken
});
} catch (error) {
throw new SecretSyncError({
error,
secretKey: secret
});
}
}
}
};
const removeSecrets = async (secretSync: TCamundaSyncWithCredentials, secretMap: TSecretMap) => {
const {
destinationConfig: { clusterUUID },
connection
} = secretSync;
const accessToken = await getCamundaConnectionAccessToken(connection, appConnectionDAL, kmsService);
const camundaSecrets = await getCamundaSecrets({ accessToken, clusterUUID });
for await (const secret of Object.keys(camundaSecrets)) {
if (!(secret in secretMap)) {
await deleteCamundaSecret({
key: secret,
clusterUUID,
accessToken
});
}
}
};
const getSecrets = async (secretSync: TCamundaSyncWithCredentials) => {
const {
destinationConfig: { clusterUUID },
connection
} = secretSync;
const accessToken = await getCamundaConnectionAccessToken(connection, appConnectionDAL, kmsService);
const camundaSecrets = await getCamundaSecrets({ accessToken, clusterUUID });
return Object.fromEntries(Object.entries(camundaSecrets).map(([key, value]) => [key, { value }]));
};
return {
syncSecrets,
removeSecrets,
getSecrets
};
};
@@ -0,0 +1,47 @@
import { z } from "zod";
import { SecretSyncs } from "@app/lib/api-docs";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import {
BaseSecretSyncSchema,
GenericCreateSecretSyncFieldsSchema,
GenericUpdateSecretSyncFieldsSchema
} from "@app/services/secret-sync/secret-sync-schemas";
import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types";
const CamundaSyncDestinationConfigSchema = z.object({
scope: z.string().trim().min(1, "Camunda scope required").describe(SecretSyncs.DESTINATION_CONFIG.CAMUNDA.scope),
clusterUUID: z
.string()
.min(1, "Camunda cluster UUID is required")
.describe(SecretSyncs.DESTINATION_CONFIG.CAMUNDA.clusterUUID)
});
const CamundaSyncOptionsConfig: TSyncOptionsConfig = { canImportSecrets: true };
export const CamundaSyncSchema = BaseSecretSyncSchema(SecretSync.Camunda, CamundaSyncOptionsConfig).extend({
destination: z.literal(SecretSync.Camunda),
destinationConfig: CamundaSyncDestinationConfigSchema
});
export const CreateCamundaSyncSchema = GenericCreateSecretSyncFieldsSchema(
SecretSync.Camunda,
CamundaSyncOptionsConfig
).extend({
destinationConfig: CamundaSyncDestinationConfigSchema
});
export const UpdateCamundaSyncSchema = GenericUpdateSecretSyncFieldsSchema(
SecretSync.Camunda,
CamundaSyncOptionsConfig
).extend({
destinationConfig: CamundaSyncDestinationConfigSchema.optional()
});
export const CamundaSyncListItemSchema = z.object({
name: z.literal("Camunda"),
connection: z.literal(AppConnection.Camunda),
destination: z.literal(SecretSync.Camunda),
canImportSecrets: z.literal(true)
});
@@ -0,0 +1,38 @@
import { z } from "zod";
import { TCamundaConnection } from "@app/services/app-connection/camunda";
import { CamundaSyncListItemSchema, CamundaSyncSchema, CreateCamundaSyncSchema } from "./camunda-sync-schemas";
export type TCamundaSync = z.infer<typeof CamundaSyncSchema>;
export type TCamundaSyncInput = z.infer<typeof CreateCamundaSyncSchema>;
export type TCamundaSyncListItem = z.infer<typeof CamundaSyncListItemSchema>;
export type TCamundaSyncWithCredentials = TCamundaSync & {
connection: TCamundaConnection;
};
export type TCamundaListSecretsResponse = { [key: string]: string };
type TBaseCamundaSecretRequest = {
accessToken: string;
clusterUUID: string;
};
export type TCamundaListSecrets = TBaseCamundaSecretRequest;
export type TCamundaCreateSecret = {
key: string;
value?: string;
} & TBaseCamundaSecretRequest;
export type TCamundaPutSecret = {
key: string;
value?: string;
} & TBaseCamundaSecretRequest;
export type TCamundaDeleteSecret = {
key: string;
} & TBaseCamundaSecretRequest;
@@ -0,0 +1,4 @@
export * from "./camunda-sync-constants";
export * from "./camunda-sync-fns";
export * from "./camunda-sync-schemas";
export * from "./camunda-sync-types";
@@ -6,7 +6,10 @@ export enum SecretSync {
AzureKeyVault = "azure-key-vault",
AzureAppConfiguration = "azure-app-configuration",
Databricks = "databricks",
Humanitec = "humanitec"
Humanitec = "humanitec",
TerraformCloud = "terraform-cloud",
Camunda = "camunda",
Vercel = "vercel"
}
export enum SecretSyncInitialSyncBehavior {
@@ -22,10 +22,13 @@ import { TAppConnectionDALFactory } from "../app-connection/app-connection-dal";
import { TKmsServiceFactory } from "../kms/kms-service";
import { AZURE_APP_CONFIGURATION_SYNC_LIST_OPTION, azureAppConfigurationSyncFactory } from "./azure-app-configuration";
import { AZURE_KEY_VAULT_SYNC_LIST_OPTION, azureKeyVaultSyncFactory } from "./azure-key-vault";
import { CAMUNDA_SYNC_LIST_OPTION, camundaSyncFactory } from "./camunda";
import { GCP_SYNC_LIST_OPTION } from "./gcp";
import { GcpSyncFns } from "./gcp/gcp-sync-fns";
import { HUMANITEC_SYNC_LIST_OPTION } from "./humanitec";
import { HumanitecSyncFns } from "./humanitec/humanitec-sync-fns";
import { TERRAFORM_CLOUD_SYNC_LIST_OPTION, TerraformCloudSyncFns } from "./terraform-cloud";
import { VERCEL_SYNC_LIST_OPTION, VercelSyncFns } from "./vercel";
const SECRET_SYNC_LIST_OPTIONS: Record<SecretSync, TSecretSyncListItem> = {
[SecretSync.AWSParameterStore]: AWS_PARAMETER_STORE_SYNC_LIST_OPTION,
@@ -35,7 +38,10 @@ const SECRET_SYNC_LIST_OPTIONS: Record<SecretSync, TSecretSyncListItem> = {
[SecretSync.AzureKeyVault]: AZURE_KEY_VAULT_SYNC_LIST_OPTION,
[SecretSync.AzureAppConfiguration]: AZURE_APP_CONFIGURATION_SYNC_LIST_OPTION,
[SecretSync.Databricks]: DATABRICKS_SYNC_LIST_OPTION,
[SecretSync.Humanitec]: HUMANITEC_SYNC_LIST_OPTION
[SecretSync.Humanitec]: HUMANITEC_SYNC_LIST_OPTION,
[SecretSync.TerraformCloud]: TERRAFORM_CLOUD_SYNC_LIST_OPTION,
[SecretSync.Camunda]: CAMUNDA_SYNC_LIST_OPTION,
[SecretSync.Vercel]: VERCEL_SYNC_LIST_OPTION
};
export const listSecretSyncOptions = () => {
@@ -121,6 +127,15 @@ export const SecretSyncFns = {
}).syncSecrets(secretSync, secretMap);
case SecretSync.Humanitec:
return HumanitecSyncFns.syncSecrets(secretSync, secretMap);
case SecretSync.TerraformCloud:
return TerraformCloudSyncFns.syncSecrets(secretSync, secretMap);
case SecretSync.Camunda:
return camundaSyncFactory({
appConnectionDAL,
kmsService
}).syncSecrets(secretSync, secretMap);
case SecretSync.Vercel:
return VercelSyncFns.syncSecrets(secretSync, secretMap);
default:
throw new Error(
`Unhandled sync destination for sync secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}`
@@ -165,6 +180,18 @@ export const SecretSyncFns = {
case SecretSync.Humanitec:
secretMap = await HumanitecSyncFns.getSecrets(secretSync);
break;
case SecretSync.TerraformCloud:
secretMap = await TerraformCloudSyncFns.getSecrets(secretSync);
break;
case SecretSync.Camunda:
secretMap = await camundaSyncFactory({
appConnectionDAL,
kmsService
}).getSecrets(secretSync);
break;
case SecretSync.Vercel:
secretMap = await VercelSyncFns.getSecrets(secretSync);
break;
default:
throw new Error(
`Unhandled sync destination for get secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}`
@@ -207,6 +234,15 @@ export const SecretSyncFns = {
}).removeSecrets(secretSync, secretMap);
case SecretSync.Humanitec:
return HumanitecSyncFns.removeSecrets(secretSync, secretMap);
case SecretSync.TerraformCloud:
return TerraformCloudSyncFns.removeSecrets(secretSync, secretMap);
case SecretSync.Camunda:
return camundaSyncFactory({
appConnectionDAL,
kmsService
}).removeSecrets(secretSync, secretMap);
case SecretSync.Vercel:
return VercelSyncFns.removeSecrets(secretSync, secretMap);
default:
throw new Error(
`Unhandled sync destination for remove secrets fns: ${(secretSync as TSecretSyncWithCredentials).destination}`
@@ -9,7 +9,10 @@ export const SECRET_SYNC_NAME_MAP: Record<SecretSync, string> = {
[SecretSync.AzureKeyVault]: "Azure Key Vault",
[SecretSync.AzureAppConfiguration]: "Azure App Configuration",
[SecretSync.Databricks]: "Databricks",
[SecretSync.Humanitec]: "Humanitec"
[SecretSync.Humanitec]: "Humanitec",
[SecretSync.TerraformCloud]: "Terraform Cloud",
[SecretSync.Camunda]: "Camunda",
[SecretSync.Vercel]: "Vercel"
};
export const SECRET_SYNC_CONNECTION_MAP: Record<SecretSync, AppConnection> = {
@@ -20,5 +23,8 @@ export const SECRET_SYNC_CONNECTION_MAP: Record<SecretSync, AppConnection> = {
[SecretSync.AzureKeyVault]: AppConnection.AzureKeyVault,
[SecretSync.AzureAppConfiguration]: AppConnection.AzureAppConfiguration,
[SecretSync.Databricks]: AppConnection.Databricks,
[SecretSync.Humanitec]: AppConnection.Humanitec
[SecretSync.Humanitec]: AppConnection.Humanitec,
[SecretSync.TerraformCloud]: AppConnection.TerraformCloud,
[SecretSync.Camunda]: AppConnection.Camunda,
[SecretSync.Vercel]: AppConnection.Vercel
};
@@ -213,7 +213,7 @@ export const secretSyncQueueFactory = ({
canExpandValue: () => true
});
const secrets = await secretV2BridgeDAL.findByFolderId(folderId);
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId, projectId });
await Promise.allSettled(
secrets.map(async (secret) => {
@@ -243,6 +243,7 @@ export const secretSyncQueueFactory = ({
if (secretImports.length) {
const importedSecrets = await fnSecretsV2FromImports({
projectId,
decryptor: decryptSecretValue,
folderDAL,
secretDAL: secretV2BridgeDAL,
@@ -9,6 +9,12 @@ import {
TAwsSecretsManagerSyncListItem,
TAwsSecretsManagerSyncWithCredentials
} from "@app/services/secret-sync/aws-secrets-manager";
import {
TCamundaSync,
TCamundaSyncInput,
TCamundaSyncListItem,
TCamundaSyncWithCredentials
} from "@app/services/secret-sync/camunda";
import {
TDatabricksSync,
TDatabricksSyncInput,
@@ -49,6 +55,13 @@ import {
THumanitecSyncListItem,
THumanitecSyncWithCredentials
} from "./humanitec";
import {
TTerraformCloudSync,
TTerraformCloudSyncInput,
TTerraformCloudSyncListItem,
TTerraformCloudSyncWithCredentials
} from "./terraform-cloud";
import { TVercelSync, TVercelSyncInput, TVercelSyncListItem, TVercelSyncWithCredentials } from "./vercel";
export type TSecretSync =
| TAwsParameterStoreSync
@@ -58,7 +71,10 @@ export type TSecretSync =
| TAzureKeyVaultSync
| TAzureAppConfigurationSync
| TDatabricksSync
| THumanitecSync;
| THumanitecSync
| TTerraformCloudSync
| TCamundaSync
| TVercelSync;
export type TSecretSyncWithCredentials =
| TAwsParameterStoreSyncWithCredentials
@@ -68,7 +84,10 @@ export type TSecretSyncWithCredentials =
| TAzureKeyVaultSyncWithCredentials
| TAzureAppConfigurationSyncWithCredentials
| TDatabricksSyncWithCredentials
| THumanitecSyncWithCredentials;
| THumanitecSyncWithCredentials
| TTerraformCloudSyncWithCredentials
| TCamundaSyncWithCredentials
| TVercelSyncWithCredentials;
export type TSecretSyncInput =
| TAwsParameterStoreSyncInput
@@ -78,7 +97,10 @@ export type TSecretSyncInput =
| TAzureKeyVaultSyncInput
| TAzureAppConfigurationSyncInput
| TDatabricksSyncInput
| THumanitecSyncInput;
| THumanitecSyncInput
| TTerraformCloudSyncInput
| TCamundaSyncInput
| TVercelSyncInput;
export type TSecretSyncListItem =
| TAwsParameterStoreSyncListItem
@@ -88,7 +110,10 @@ export type TSecretSyncListItem =
| TAzureKeyVaultSyncListItem
| TAzureAppConfigurationSyncListItem
| TDatabricksSyncListItem
| THumanitecSyncListItem;
| THumanitecSyncListItem
| TTerraformCloudSyncListItem
| TCamundaSyncListItem
| TVercelSyncListItem;
export type TSyncOptionsConfig = {
canImportSecrets: boolean;
@@ -0,0 +1,5 @@
export * from "./terraform-cloud-sync-constants";
export * from "./terraform-cloud-sync-enums";
export * from "./terraform-cloud-sync-fns";
export * from "./terraform-cloud-sync-schemas";
export * from "./terraform-cloud-sync-types";
@@ -0,0 +1,10 @@
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import { TSecretSyncListItem } from "@app/services/secret-sync/secret-sync-types";
export const TERRAFORM_CLOUD_SYNC_LIST_OPTION: TSecretSyncListItem = {
name: "Terraform Cloud",
destination: SecretSync.TerraformCloud,
connection: AppConnection.TerraformCloud,
canImportSecrets: false
};
@@ -0,0 +1,9 @@
export enum TerraformCloudSyncScope {
VariableSet = "variable-set",
Workspace = "workspace"
}
export enum TerraformCloudSyncCategory {
Environment = "env",
Terraform = "terraform"
}
@@ -0,0 +1,253 @@
/* eslint-disable no-await-in-loop */
import { AxiosResponse } from "axios";
import { request } from "@app/lib/config/request";
import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
import { SECRET_SYNC_NAME_MAP } from "../secret-sync-maps";
import { TerraformCloudSyncScope } from "./terraform-cloud-sync-enums";
import {
TerraformCloudApiResponse,
TerraformCloudApiVariable,
TerraformCloudVariable,
TTerraformCloudSyncWithCredentials
} from "./terraform-cloud-sync-types";
const getTerraformCloudVariables = async (
secretSync: TTerraformCloudSyncWithCredentials
): Promise<TerraformCloudVariable[]> => {
const {
destinationConfig,
connection: {
credentials: { apiToken }
}
} = secretSync;
let url: string;
let source: TerraformCloudVariable["source"];
if (destinationConfig.scope === TerraformCloudSyncScope.VariableSet) {
url = `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/varsets/${destinationConfig.variableSetId}/relationships/vars`;
source = "varset";
} else {
url = `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/workspaces/${destinationConfig.workspaceId}/vars`;
source = "workspace";
}
const headers = {
Authorization: `Bearer ${apiToken}`,
"Content-Type": "application/vnd.api+json"
};
const fetchAllPages = async (): Promise<TerraformCloudApiVariable[]> => {
let results: TerraformCloudApiVariable[] = [];
let nextUrl: string | null = url;
while (nextUrl) {
const res: AxiosResponse<TerraformCloudApiResponse<TerraformCloudApiVariable[]>> = await request.get<
TerraformCloudApiResponse<TerraformCloudApiVariable[]>
>(nextUrl, {
headers
});
if (res.data?.data) {
results = results.concat(res.data.data);
}
nextUrl = res.data?.links?.next ?? null;
}
return results;
};
const allVariableData = await fetchAllPages();
const variables: TerraformCloudVariable[] = allVariableData.map((variable) => ({
id: variable.id,
key: variable.attributes.key,
value: variable.attributes.value || "",
sensitive: variable.attributes.sensitive,
description: variable.attributes.description || "",
category: variable.attributes.category,
source
}));
return variables;
};
const deleteVariable = async (
secretSync: TTerraformCloudSyncWithCredentials,
variable: TerraformCloudVariable
): Promise<void> => {
const {
destinationConfig,
connection: {
credentials: { apiToken }
}
} = secretSync;
try {
let url;
if (destinationConfig.scope === TerraformCloudSyncScope.VariableSet) {
url = `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/varsets/${destinationConfig.variableSetId}/relationships/vars/${variable.id}`;
} else {
url = `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/workspaces/${destinationConfig.workspaceId}/vars/${variable.id}`;
}
await request.delete(url, {
headers: {
Authorization: `Bearer ${apiToken}`,
"Content-Type": "application/vnd.api+json"
}
});
} catch (error) {
throw new SecretSyncError({
error,
secretKey: variable.key
});
}
};
const createVariable = async (
secretSync: TTerraformCloudSyncWithCredentials,
secretMap: TSecretMap,
key: string
): Promise<void> => {
try {
const {
destinationConfig,
connection: {
credentials: { apiToken }
}
} = secretSync;
let url;
if (destinationConfig.scope === TerraformCloudSyncScope.VariableSet) {
url = `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/varsets/${destinationConfig.variableSetId}/relationships/vars`;
} else {
url = `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/workspaces/${destinationConfig.workspaceId}/vars`;
}
await request.post(
url,
{
data: {
type: "vars",
attributes: {
key,
value: secretMap[key].value,
description: secretMap[key].comment || "",
category: secretSync.destinationConfig.category,
sensitive: true
}
}
},
{
headers: {
Authorization: `Bearer ${apiToken}`,
"Content-Type": "application/vnd.api+json"
}
}
);
} catch (error) {
throw new SecretSyncError({
error,
secretKey: key
});
}
};
const updateVariable = async (
secretSync: TTerraformCloudSyncWithCredentials,
secretMap: TSecretMap,
variable: TerraformCloudVariable
): Promise<void> => {
try {
const {
destinationConfig,
connection: {
credentials: { apiToken }
}
} = secretSync;
let url;
if (destinationConfig.scope === TerraformCloudSyncScope.VariableSet) {
url = `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/varsets/${destinationConfig.variableSetId}/relationships/vars/${variable.id}`;
} else {
url = `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/workspaces/${destinationConfig.workspaceId}/vars/${variable.id}`;
}
await request.patch(
url,
{
data: {
type: "vars",
id: variable.id,
attributes: {
value: secretMap[variable.key].value,
description: secretMap[variable.key].comment || "",
category: secretSync.destinationConfig.category
}
}
},
{
headers: {
Authorization: `Bearer ${apiToken}`,
"Content-Type": "application/vnd.api+json"
}
}
);
} catch (error) {
throw new SecretSyncError({
error,
secretKey: variable.key
});
}
};
export const TerraformCloudSyncFns = {
syncSecrets: async (secretSync: TTerraformCloudSyncWithCredentials, secretMap: TSecretMap): Promise<void> => {
const terraformCloudVariables = await getTerraformCloudVariables(secretSync);
const terraformCloudVariablesMap = new Map<string, TerraformCloudVariable>(
terraformCloudVariables.map((v) => [v.key, v])
);
const secretKeys = Object.keys(secretMap);
for (const key of secretKeys) {
const existingVariable = terraformCloudVariablesMap.get(key);
if (!existingVariable) {
await createVariable(secretSync, secretMap, key);
} else {
await updateVariable(secretSync, secretMap, existingVariable);
}
}
if (secretSync.syncOptions.disableSecretDeletion) return;
for (const terraformCloudVariable of terraformCloudVariables) {
if (!Object.prototype.hasOwnProperty.call(secretMap, terraformCloudVariable.key)) {
await deleteVariable(secretSync, terraformCloudVariable);
}
}
},
getSecrets: async (secretSync: TTerraformCloudSyncWithCredentials): Promise<TSecretMap> => {
throw new Error(`${SECRET_SYNC_NAME_MAP[secretSync.destination]} does not support importing secrets.`);
},
removeSecrets: async (secretSync: TTerraformCloudSyncWithCredentials, secretMap: TSecretMap): Promise<void> => {
const terraformCloudVariables = await getTerraformCloudVariables(secretSync);
for (const variable of terraformCloudVariables) {
if (Object.prototype.hasOwnProperty.call(secretMap, variable.key)) {
await deleteVariable(secretSync, variable);
}
}
}
};
@@ -0,0 +1,77 @@
import { z } from "zod";
import { SecretSyncs } from "@app/lib/api-docs";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import {
BaseSecretSyncSchema,
GenericCreateSecretSyncFieldsSchema,
GenericUpdateSecretSyncFieldsSchema
} from "@app/services/secret-sync/secret-sync-schemas";
import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types";
import {
TerraformCloudSyncCategory,
TerraformCloudSyncScope
} from "@app/services/secret-sync/terraform-cloud/terraform-cloud-sync-enums";
const TerraformCloudSyncDestinationConfigSchema = z.discriminatedUnion("scope", [
z.object({
scope: z
.literal(TerraformCloudSyncScope.VariableSet)
.describe(SecretSyncs.DESTINATION_CONFIG.TERRAFORM_CLOUD.scope),
org: z.string().min(1, "Org ID is required").describe(SecretSyncs.DESTINATION_CONFIG.TERRAFORM_CLOUD.org),
variableSetName: z
.string()
.min(1, "Variable set name is required")
.describe(SecretSyncs.DESTINATION_CONFIG.TERRAFORM_CLOUD.variableSetName),
variableSetId: z
.string()
.min(1, "Variable set ID is required")
.describe(SecretSyncs.DESTINATION_CONFIG.TERRAFORM_CLOUD.variableSetId),
category: z.nativeEnum(TerraformCloudSyncCategory).describe(SecretSyncs.DESTINATION_CONFIG.TERRAFORM_CLOUD.category)
}),
z.object({
scope: z.literal(TerraformCloudSyncScope.Workspace).describe(SecretSyncs.DESTINATION_CONFIG.TERRAFORM_CLOUD.scope),
org: z.string().min(1, "Org ID is required").describe(SecretSyncs.DESTINATION_CONFIG.TERRAFORM_CLOUD.org),
workspaceName: z
.string()
.min(1, "Workspace name is required")
.describe(SecretSyncs.DESTINATION_CONFIG.TERRAFORM_CLOUD.workspaceName),
workspaceId: z
.string()
.min(1, "Workspace ID is required")
.describe(SecretSyncs.DESTINATION_CONFIG.TERRAFORM_CLOUD.workspaceId),
category: z.nativeEnum(TerraformCloudSyncCategory).describe(SecretSyncs.DESTINATION_CONFIG.TERRAFORM_CLOUD.category)
})
]);
const TerraformCloudSyncOptionsConfig: TSyncOptionsConfig = { canImportSecrets: false };
export const TerraformCloudSyncSchema = BaseSecretSyncSchema(
SecretSync.TerraformCloud,
TerraformCloudSyncOptionsConfig
).extend({
destination: z.literal(SecretSync.TerraformCloud),
destinationConfig: TerraformCloudSyncDestinationConfigSchema
});
export const CreateTerraformCloudSyncSchema = GenericCreateSecretSyncFieldsSchema(
SecretSync.TerraformCloud,
TerraformCloudSyncOptionsConfig
).extend({
destinationConfig: TerraformCloudSyncDestinationConfigSchema
});
export const UpdateTerraformCloudSyncSchema = GenericUpdateSecretSyncFieldsSchema(
SecretSync.TerraformCloud,
TerraformCloudSyncOptionsConfig
).extend({
destinationConfig: TerraformCloudSyncDestinationConfigSchema.optional()
});
export const TerraformCloudSyncListItemSchema = z.object({
name: z.literal("Terraform Cloud"),
connection: z.literal(AppConnection.TerraformCloud),
destination: z.literal(SecretSync.TerraformCloud),
canImportSecrets: z.literal(false)
});
@@ -0,0 +1,77 @@
import z from "zod";
import { TTerraformCloudConnection } from "@app/services/app-connection/terraform-cloud";
import {
CreateTerraformCloudSyncSchema,
TerraformCloudSyncListItemSchema,
TerraformCloudSyncSchema
} from "./terraform-cloud-sync-schemas";
export type TTerraformCloudSyncListItem = z.infer<typeof TerraformCloudSyncListItemSchema>;
export type TTerraformCloudSync = z.infer<typeof TerraformCloudSyncSchema>;
export type TTerraformCloudSyncInput = z.infer<typeof CreateTerraformCloudSyncSchema>;
export type TTerraformCloudSyncWithCredentials = TTerraformCloudSync & {
connection: TTerraformCloudConnection;
};
export type TerraformCloudApiVariable = {
id: string;
type: string;
attributes: {
key: string;
value: string | null;
sensitive: boolean;
category: "terraform" | "env";
hcl: boolean;
description: string | null;
};
relationships: {
workspace?: {
data: {
id: string;
type: string;
};
};
project?: {
data: {
id: string;
type: string;
};
};
};
};
export type TerraformCloudVariable = {
id: string;
key: string;
value: string;
sensitive: boolean;
description: string;
category: "terraform" | "env";
source: "varset" | "workspace";
};
export type TerraformCloudApiResponse<T> = {
data: T;
included?: unknown[];
links?: {
self?: string;
first?: string;
prev?: string;
next?: string;
last?: string;
};
meta?: {
pagination?: {
current_page: number;
prev_page: number | null;
next_page: number | null;
total_pages: number;
total_count: number;
};
};
};
@@ -0,0 +1,5 @@
export * from "./vercel-sync-constants";
export * from "./vercel-sync-enums";
export * from "./vercel-sync-fns";
export * from "./vercel-sync-schemas";
export * from "./vercel-sync-types";
@@ -0,0 +1,10 @@
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import { TSecretSyncListItem } from "@app/services/secret-sync/secret-sync-types";
export const VERCEL_SYNC_LIST_OPTION: TSecretSyncListItem = {
name: "Vercel",
destination: SecretSync.Vercel,
connection: AppConnection.Vercel,
canImportSecrets: true
};
@@ -0,0 +1,12 @@
export enum VercelSyncScope {
Application = "application",
Environment = "environment"
}
export const VercelEnvironmentType = {
Development: "development",
Preview: "preview",
Production: "production"
} as const;
export type VercelEnvironment = (typeof VercelEnvironmentType)[keyof typeof VercelEnvironmentType];
@@ -0,0 +1,313 @@
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import { request } from "@app/lib/config/request";
import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
import { VercelEnvironmentType } from "./vercel-sync-enums";
import { DefaultVercelEnvType, TVercelSyncWithCredentials, VercelApiSecret } from "./vercel-sync-types";
function isVercelDefaultEnvType(value: string): value is DefaultVercelEnvType {
return Object.values(VercelEnvironmentType).map(String).includes(value);
}
const MAX_RETRIES = 5;
const sleep = async () =>
new Promise((resolve) => {
setTimeout(resolve, 60000);
});
const getVercelSecretsWithRetries = async (
secretSync: TVercelSyncWithCredentials,
attempt = 0
): Promise<VercelApiSecret[]> => {
const {
destinationConfig,
connection: {
credentials: { apiToken }
}
} = secretSync;
const params: { [key: string]: string } = {
decrypt: "true",
...(destinationConfig.branch ? { gitBranch: destinationConfig.branch } : {})
};
try {
const { data } = await request.get<{ envs: VercelApiSecret[] }>(
`${IntegrationUrls.VERCEL_API_URL}/v9/projects/${destinationConfig.app}/env?teamId=${destinationConfig.teamId}`,
{
params,
headers: {
Authorization: `Bearer ${apiToken}`,
"Accept-Encoding": "application/json"
}
}
);
return data.envs;
} catch (error) {
if ((error as { response: { status: number } }).response.status === 429 && attempt < MAX_RETRIES) {
await sleep();
return await getVercelSecretsWithRetries(secretSync, attempt + 1);
}
throw error;
}
};
const getDecryptedVercelSecret = async (
secretSync: TVercelSyncWithCredentials,
secret: VercelApiSecret,
attempt = 0
): Promise<VercelApiSecret> => {
const {
destinationConfig,
connection: {
credentials: { apiToken }
}
} = secretSync;
const params: { [key: string]: string } = {
decrypt: "true",
...(destinationConfig.branch ? { gitBranch: destinationConfig.branch } : {})
};
try {
const { data: decryptedSecret } = await request.get(
`${IntegrationUrls.VERCEL_API_URL}/v9/projects/${destinationConfig.app}/env/${secret.id}?teamId=${destinationConfig.teamId}`,
{
params,
headers: {
Authorization: `Bearer ${apiToken}`,
"Accept-Encoding": "application/json"
}
}
);
return decryptedSecret as VercelApiSecret;
} catch (error) {
if ((error as { response: { status: number } }).response.status === 429 && attempt < MAX_RETRIES) {
await sleep();
return await getDecryptedVercelSecret(secretSync, secret, attempt + 1);
}
throw error;
}
};
const getVercelSecrets = async (secretSync: TVercelSyncWithCredentials): Promise<VercelApiSecret[]> => {
const { destinationConfig } = secretSync;
const secrets = await getVercelSecretsWithRetries(secretSync);
const filteredSecrets = secrets.filter((secret) => {
if (!isVercelDefaultEnvType(destinationConfig.env)) {
if (secret.customEnvironmentIds?.includes(destinationConfig.env)) {
return true;
}
return false;
}
if (secret.target.includes(destinationConfig.env)) {
// If it's preview environment with a branch specified
if (
destinationConfig.env === VercelEnvironmentType.Preview &&
destinationConfig.branch &&
secret.gitBranch &&
secret.gitBranch !== destinationConfig.branch
) {
return false;
}
return true;
}
return false;
});
// For secrets of type "encrypted", we need to get their decrypted value
const secretsWithValues = await Promise.all(
filteredSecrets.map(async (secret) => {
if (secret.type === "encrypted") {
const decryptedSecret = await getDecryptedVercelSecret(secretSync, secret);
return decryptedSecret;
}
return secret;
})
);
return secretsWithValues;
};
const deleteSecret = async (
secretSync: TVercelSyncWithCredentials,
vercelSecret: VercelApiSecret,
attempt = 0
): Promise<void> => {
const {
destinationConfig,
connection: {
credentials: { apiToken }
}
} = secretSync;
try {
await request.delete(
`${IntegrationUrls.VERCEL_API_URL}/v9/projects/${destinationConfig.app}/env/${vercelSecret.id}?teamId=${destinationConfig.teamId}`,
{
headers: {
Authorization: `Bearer ${apiToken}`,
"Accept-Encoding": "application/json"
}
}
);
} catch (error) {
if ((error as { response: { status: number } }).response.status === 429 && attempt < MAX_RETRIES) {
await sleep();
return await deleteSecret(secretSync, vercelSecret, attempt + 1);
}
throw new SecretSyncError({
error,
secretKey: vercelSecret.key
});
}
};
const createSecret = async (
secretSync: TVercelSyncWithCredentials,
secretMap: TSecretMap,
key: string,
attempt = 0
): Promise<void> => {
try {
const {
destinationConfig,
connection: {
credentials: { apiToken }
}
} = secretSync;
await request.post(
`${IntegrationUrls.VERCEL_API_URL}/v10/projects/${destinationConfig.app}/env?teamId=${destinationConfig.teamId}`,
{
key,
value: secretMap[key].value,
type: "encrypted",
target: isVercelDefaultEnvType(destinationConfig.env) ? [destinationConfig.env] : [],
customEnvironmentIds: !isVercelDefaultEnvType(destinationConfig.env) ? [destinationConfig.env] : [],
...(destinationConfig.env === VercelEnvironmentType.Preview && destinationConfig.branch
? { gitBranch: destinationConfig.branch }
: {})
},
{
headers: {
Authorization: `Bearer ${apiToken}`,
"Accept-Encoding": "application/json"
}
}
);
} catch (error) {
if ((error as { response: { status: number } }).response.status === 429 && attempt < MAX_RETRIES) {
await sleep();
return await createSecret(secretSync, secretMap, key, attempt + 1);
}
throw new SecretSyncError({
error,
secretKey: key
});
}
};
const updateSecret = async (
secretSync: TVercelSyncWithCredentials,
secretMap: TSecretMap,
vercelSecret: VercelApiSecret,
attempt = 0
): Promise<void> => {
try {
const {
destinationConfig,
connection: {
credentials: { apiToken }
}
} = secretSync;
let target = [...vercelSecret.target];
if (isVercelDefaultEnvType(destinationConfig.env) && !vercelSecret.target.includes(destinationConfig.env)) {
target = [...target, destinationConfig.env];
}
let customEnvironmentIds = [...(vercelSecret.customEnvironmentIds || [])];
if (
!isVercelDefaultEnvType(destinationConfig.env) &&
!vercelSecret.customEnvironmentIds?.includes(destinationConfig.env)
) {
customEnvironmentIds = [...customEnvironmentIds, destinationConfig.env];
}
await request.patch(
`${IntegrationUrls.VERCEL_API_URL}/v9/projects/${destinationConfig.app}/env/${vercelSecret.id}?teamId=${destinationConfig.teamId}`,
{
...(vercelSecret.type !== "sensitive" && { key: vercelSecret.key }),
value: secretMap[vercelSecret.key].value,
type: vercelSecret.type,
target,
customEnvironmentIds,
...(destinationConfig.env === VercelEnvironmentType.Preview && destinationConfig.branch
? { gitBranch: destinationConfig.branch }
: {})
},
{
headers: {
Authorization: `Bearer ${apiToken}`,
"Accept-Encoding": "application/json"
}
}
);
} catch (error) {
if ((error as { response: { status: number } }).response.status === 429 && attempt < MAX_RETRIES) {
await sleep();
return await updateSecret(secretSync, secretMap, vercelSecret, attempt + 1);
}
throw new SecretSyncError({
error,
secretKey: vercelSecret.key
});
}
};
export const VercelSyncFns = {
syncSecrets: async (secretSync: TVercelSyncWithCredentials, secretMap: TSecretMap) => {
const vercelSecrets = await getVercelSecrets(secretSync);
const vercelSecretsMap = new Map(vercelSecrets.map((s) => [s.key, s]));
// Create or update secrets
for await (const key of Object.keys(secretMap)) {
const existingSecret = vercelSecretsMap.get(key);
if (!existingSecret) {
await createSecret(secretSync, secretMap, key);
} else if (existingSecret.value !== secretMap[key].value) {
await updateSecret(secretSync, secretMap, existingSecret);
}
}
// Delete secrets if disableSecretDeletion is not set
if (secretSync.syncOptions.disableSecretDeletion) return;
for await (const vercelSecret of vercelSecrets) {
if (!secretMap[vercelSecret.key]) {
await deleteSecret(secretSync, vercelSecret);
}
}
},
getSecrets: async (secretSync: TVercelSyncWithCredentials): Promise<TSecretMap> => {
const vercelSecrets = await getVercelSecrets(secretSync);
return Object.fromEntries(vercelSecrets.map((s) => [s.key, { value: s.value ?? "" }]));
},
removeSecrets: async (secretSync: TVercelSyncWithCredentials, secretMap: TSecretMap) => {
const vercelSecrets = await getVercelSecrets(secretSync);
for await (const vercelSecret of vercelSecrets) {
if (vercelSecret.key in secretMap) {
await deleteSecret(secretSync, vercelSecret);
}
}
}
};
@@ -0,0 +1,49 @@
import { z } from "zod";
import { SecretSyncs } from "@app/lib/api-docs";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
import {
BaseSecretSyncSchema,
GenericCreateSecretSyncFieldsSchema,
GenericUpdateSecretSyncFieldsSchema
} from "@app/services/secret-sync/secret-sync-schemas";
import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types";
import { VercelEnvironmentType } from "./vercel-sync-enums";
const VercelSyncDestinationConfigSchema = z.object({
app: z.string().min(1, "App ID is required").describe(SecretSyncs.DESTINATION_CONFIG.VERCEL.app),
appName: z.string().min(1, "App Name is required").describe(SecretSyncs.DESTINATION_CONFIG.VERCEL.appName),
env: z.nativeEnum(VercelEnvironmentType).or(z.string()).describe(SecretSyncs.DESTINATION_CONFIG.VERCEL.env),
branch: z.string().optional().describe(SecretSyncs.DESTINATION_CONFIG.VERCEL.branch),
teamId: z.string().describe(SecretSyncs.DESTINATION_CONFIG.VERCEL.teamId)
});
const VercelSyncOptionsConfig: TSyncOptionsConfig = { canImportSecrets: true };
export const VercelSyncSchema = BaseSecretSyncSchema(SecretSync.Vercel, VercelSyncOptionsConfig).extend({
destination: z.literal(SecretSync.Vercel),
destinationConfig: VercelSyncDestinationConfigSchema
});
export const CreateVercelSyncSchema = GenericCreateSecretSyncFieldsSchema(
SecretSync.Vercel,
VercelSyncOptionsConfig
).extend({
destinationConfig: VercelSyncDestinationConfigSchema
});
export const UpdateVercelSyncSchema = GenericUpdateSecretSyncFieldsSchema(
SecretSync.Vercel,
VercelSyncOptionsConfig
).extend({
destinationConfig: VercelSyncDestinationConfigSchema.optional()
});
export const VercelSyncListItemSchema = z.object({
name: z.literal("Vercel"),
connection: z.literal(AppConnection.Vercel),
destination: z.literal(SecretSync.Vercel),
canImportSecrets: z.literal(true)
});
@@ -0,0 +1,40 @@
import z from "zod";
import { TVercelConnection } from "@app/services/app-connection/vercel";
import { VercelEnvironmentType } from "./vercel-sync-enums";
import { CreateVercelSyncSchema, VercelSyncListItemSchema, VercelSyncSchema } from "./vercel-sync-schemas";
export type TVercelSyncListItem = z.infer<typeof VercelSyncListItemSchema>;
export type TVercelSync = z.infer<typeof VercelSyncSchema>;
export type TVercelSyncInput = z.infer<typeof CreateVercelSyncSchema>;
export type TVercelSyncWithCredentials = TVercelSync & {
connection: TVercelConnection;
};
export type VercelSecret = {
description: string;
is_secret: boolean;
key: string;
source: "app" | "env";
value: string;
};
export interface VercelApiSecret {
id: string;
key: string;
value: string;
type: string;
target: string[];
customEnvironmentIds?: string[];
gitBranch?: string;
createdAt?: number;
updatedAt?: number;
configurationId?: string;
system?: boolean;
}
export type DefaultVercelEnvType = (typeof VercelEnvironmentType)[keyof typeof VercelEnvironmentType];
@@ -2,7 +2,10 @@ import { Knex } from "knex";
import { validate as uuidValidate } from "uuid";
import { TDbClient } from "@app/db";
import { SecretsV2Schema, SecretType, TableName, TSecretsV2, TSecretsV2Update } from "@app/db/schemas";
import { ProjectType, SecretsV2Schema, SecretType, TableName, TSecretsV2, TSecretsV2Update } from "@app/db/schemas";
import { TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { generateCacheKeyFromData } from "@app/lib/crypto/cache";
import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
import {
buildFindFilter,
@@ -12,15 +15,67 @@ import {
TFindFilter,
TFindOpt
} from "@app/lib/knex";
import { OrderByDirection } from "@app/lib/types";
import { BufferKeysToString, OrderByDirection } from "@app/lib/types";
import { SecretsOrderBy } from "@app/services/secret/secret-types";
import { TFindSecretsByFolderIdsFilter } from "@app/services/secret-v2-bridge/secret-v2-bridge-types";
import type { TFindSecretsByFolderIdsFilter } from "@app/services/secret-v2-bridge/secret-v2-bridge-types";
export const SecretDalCacheKeys = {
get productKey() {
const { INFISICAL_PLATFORM_VERSION } = getConfig();
return `${ProjectType.SecretManager}:${INFISICAL_PLATFORM_VERSION || 0}`;
},
getSecretDalVersion: (projectId: string) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${TableName.SecretV2}-dal-version`;
},
findByFolderIds: (
projectId: string,
version: number,
{ useCache, tx, ...cacheKey }: Parameters<TSecretV2BridgeDALFactory["findByFolderIds"]>[0]
) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${
TableName.SecretV2
}-dal:v${version}:find-by-folder-ids:${generateCacheKeyFromData(cacheKey)}`;
},
findByFolderId: (
projectId: string,
version: number,
{ useCache, tx, ...cacheKey }: Parameters<TSecretV2BridgeDALFactory["findByFolderId"]>[0]
) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${
TableName.SecretV2
}-dal:v${version}:find-by-folder-id:${generateCacheKeyFromData(cacheKey)}`;
},
find: (projectId: string, version: number, ...args: Parameters<TSecretV2BridgeDALFactory["find"]>) => {
const [filter, opts] = args;
delete opts?.tx;
delete opts?.useCache;
return `${SecretDalCacheKeys.productKey}:${projectId}:${
TableName.SecretV2
}-dal:v${version}:find:${generateCacheKeyFromData({
filter,
opts
})}`;
}
};
export type TSecretV2BridgeDALFactory = ReturnType<typeof secretV2BridgeDALFactory>;
interface TSecretV2DalArg {
db: TDbClient;
keyStore: TKeyStoreFactory;
}
export const secretV2BridgeDALFactory = (db: TDbClient) => {
const SECRET_DAL_TTL = 5 * 60;
const SECRET_DAL_VERSION_TTL = 15 * 60;
const MAX_SECRET_CACHE_BYTES = 25 * 1024 * 1024;
export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
const secretOrm = ormify(db, TableName.SecretV2);
const invalidateSecretCacheByProjectId = async (projectId: string) => {
const secretDalVersionKey = SecretDalCacheKeys.getSecretDalVersion(projectId);
await keyStore.incrementBy(secretDalVersionKey, 1);
await keyStore.setExpiry(secretDalVersionKey, SECRET_DAL_VERSION_TTL);
};
const findOne = async (filter: Partial<TSecretsV2>, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.SecretV2)
@@ -73,8 +128,35 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
};
const find = async (filter: TFindFilter<TSecretsV2>, { offset, limit, sort, tx }: TFindOpt<TSecretsV2> = {}) => {
const find = async (
filter: TFindFilter<TSecretsV2>,
opts: TFindOpt<TSecretsV2> & { useCache?: { projectId: string } } = {}
) => {
const { offset, limit, sort, tx, useCache } = opts;
try {
let secretDalVersion = 0;
if (useCache) {
const cachedSecretDalVersion = await keyStore.getItem(
SecretDalCacheKeys.getSecretDalVersion(useCache.projectId)
);
secretDalVersion = Number(cachedSecretDalVersion || 0);
const cacheKey = SecretDalCacheKeys.find(useCache.projectId, secretDalVersion, filter, opts);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const query = (tx || db)(TableName.SecretV2)
// eslint-disable-next-line @typescript-eslint/no-misused-promises
.where(buildFindFilter(filter))
@@ -142,6 +224,23 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
]
});
if (useCache) {
const cachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
const cache = JSON.stringify(cachedSecrets);
if (Buffer.byteLength(cache, "utf8") < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.find(useCache.projectId, secretDalVersion, filter, opts),
SECRET_DAL_TTL,
cache
);
}
}
return data;
} catch (error) {
throw new DatabaseError({ error, name: `${TableName.SecretV2}: Find` });
@@ -246,14 +345,43 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
};
const findByFolderId = async (folderId: string, userId?: string, tx?: Knex) => {
const findByFolderId = async (dto: {
folderId: string;
userId?: string;
tx?: Knex;
projectId: string;
useCache?: boolean;
}) => {
try {
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo)
const { folderId, tx, projectId } = dto;
let { userId } = dto;
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo
if (userId && !uuidValidate(userId)) {
// eslint-disable-next-line
userId = undefined;
}
const cachedSecretDalVersion = await keyStore.getItem(SecretDalCacheKeys.getSecretDalVersion(projectId));
const secretDalVersion = Number(cachedSecretDalVersion || 0);
if (dto.useCache) {
const cacheKey = SecretDalCacheKeys.findByFolderId(projectId, secretDalVersion, dto);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const secs = await (tx || db.replicaNode())(TableName.SecretV2)
.where({ folderId })
.where((bd) => {
@@ -309,6 +437,22 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
]
});
if (dto.useCache) {
const newCachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
const cache = JSON.stringify(newCachedSecrets);
if (Buffer.byteLength(cache, "utf8") < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.findByFolderId(projectId, secretDalVersion, dto),
SECRET_DAL_TTL,
cache
);
}
}
return data;
} catch (error) {
throw new DatabaseError({ error, name: "get all secret" });
@@ -394,12 +538,16 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
};
const findByFolderIds = async (
folderIds: string[],
userId?: string,
tx?: Knex,
filters?: TFindSecretsByFolderIdsFilter
) => {
const findByFolderIds = async (dto: {
folderIds: string[];
userId?: string;
tx?: Knex;
projectId: string;
filters?: TFindSecretsByFolderIdsFilter;
useCache?: boolean;
}) => {
const { folderIds, tx, filters, useCache, projectId } = dto;
let { userId } = dto;
try {
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo)
if (userId && !uuidValidate(userId)) {
@@ -407,6 +555,26 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
userId = undefined;
}
const cachedSecretDalVersion = await keyStore.getItem(SecretDalCacheKeys.getSecretDalVersion(projectId));
const secretDalVersion = Number(cachedSecretDalVersion || 0);
if (useCache) {
const cacheKey = SecretDalCacheKeys.findByFolderIds(projectId, secretDalVersion, dto);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const query = (tx || db.replicaNode())(TableName.SecretV2)
.whereIn(`${TableName.SecretV2}.folderId`, folderIds)
.where((bd) => {
@@ -532,6 +700,22 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
]
});
if (useCache) {
const cachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
const cache = JSON.stringify(cachedSecrets);
if (Buffer.byteLength(cache, "utf8") < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.findByFolderIds(projectId, secretDalVersion, dto),
SECRET_DAL_TTL,
cache
);
}
}
return data;
} catch (error) {
@@ -724,6 +908,7 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
findAllProjectSecretValues,
countByFolderIds,
findOne,
find
find,
invalidateSecretCacheByProjectId
};
};
@@ -501,7 +501,7 @@ export const expandSecretReferencesFactory = ({
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) return { value: "", tags: [] };
const secrets = await secretDAL.findByFolderId(folder.id);
const secrets = await secretDAL.findByFolderId({ folderId: folder.id, projectId, useCache: true });
const decryptedSecret = secrets.reduce<Record<string, { value: string; tags: string[] }>>((prev, secret) => {
// eslint-disable-next-line no-param-reassign
@@ -331,6 +331,7 @@ export const secretV2BridgeServiceFactory = ({
return createdSecret;
});
await secretDAL.invalidateSecretCacheByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -539,6 +540,7 @@ export const secretV2BridgeServiceFactory = ({
projectId
});
await secretDAL.invalidateSecretCacheByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -647,6 +649,7 @@ export const secretV2BridgeServiceFactory = ({
})
);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -796,12 +799,14 @@ export const secretV2BridgeServiceFactory = ({
) => {
const groupedFolderMappings = groupBy(folderMappings, (folderMapping) => folderMapping.folderId);
const secrets = await secretDAL.findByFolderIds(
folderMappings.map((folderMapping) => folderMapping.folderId),
const secrets = await secretDAL.findByFolderIds({
projectId,
folderIds: folderMappings.map((folderMapping) => folderMapping.folderId),
userId,
undefined,
filters
);
tx: undefined,
filters,
useCache: true
});
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
@@ -952,12 +957,14 @@ export const secretV2BridgeServiceFactory = ({
const groupedPaths = groupBy(paths, (p) => p.folderId);
const secrets = await secretDAL.findByFolderIds(
paths.map((p) => p.folderId),
actorId,
undefined,
params
);
const secrets = await secretDAL.findByFolderIds({
projectId,
folderIds: paths.map((p) => p.folderId),
userId: actorId,
tx: undefined,
filters: params,
useCache: true
});
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
@@ -1087,6 +1094,7 @@ export const secretV2BridgeServiceFactory = ({
const secretImports = await secretImportDAL.findByFolderIds(paths.map((p) => p.folderId));
const allowedImports = secretImports.filter(({ isReplication }) => !isReplication);
const importedSecrets = await fnSecretsV2FromImports({
projectId,
viewSecretValue,
secretImports: allowedImports,
secretDAL,
@@ -1304,6 +1312,7 @@ export const secretV2BridgeServiceFactory = ({
if (!secret && includeImports) {
const secretImports = await secretImportDAL.find({ folderId, isReplication: false });
const importedSecrets = await fnSecretsV2FromImports({
projectId,
secretImports,
viewSecretValue,
secretDAL,
@@ -1543,7 +1552,7 @@ export const secretV2BridgeServiceFactory = ({
tx
})
);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
actor,
@@ -1883,6 +1892,7 @@ export const secretV2BridgeServiceFactory = ({
}
});
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await Promise.allSettled(folders.map((el) => (el?.id ? snapshotService.performSnapshot(el.id) : undefined)));
await Promise.allSettled(
folders.map((el) =>
@@ -2014,6 +2024,7 @@ export const secretV2BridgeServiceFactory = ({
})
);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
actor,
@@ -2537,6 +2548,9 @@ export const secretV2BridgeServiceFactory = ({
}
});
if (isDestinationUpdated || isSourceUpdated) {
await secretDAL.invalidateSecretCacheByProjectId(projectId);
}
if (isDestinationUpdated) {
await snapshotService.performSnapshot(destinationFolder.id);
await secretQueueService.syncSecrets({
@@ -2715,7 +2729,7 @@ export const secretV2BridgeServiceFactory = ({
generatePaths(folderMap).map(({ folderId, path }) => [folderId, path === "/" ? path : path.substring(1)])
);
const secrets = await secretDAL.findByFolderIds(folders.map((f) => f.id));
const secrets = await secretDAL.findByFolderIds({ folderIds: folders.map((f) => f.id), projectId, useCache: true });
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
@@ -279,6 +279,13 @@ export type TUpdateManySecretsFnFactory = {
folderDAL: TSecretFolderDALFactory;
};
export type TFindByFolderIdDALDTO = {
folderId: string;
userId?: string;
tx?: Knex;
projectId: string;
};
export type TUpdateManySecretsFn = {
projectId: string;
environment: string;
+2 -1
View File
@@ -367,7 +367,7 @@ export const secretQueueFactory = ({
canExpandValue: () => true
});
// process secrets in current folder
const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId);
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId: dto.folderId, projectId: dto.projectId });
await Promise.allSettled(
secrets.map(async (secret) => {
@@ -397,6 +397,7 @@ export const secretQueueFactory = ({
// if no imports then return secrets in the current folder
if (!secretImports.length) return content;
const importedSecrets = await fnSecretsV2FromImports({
projectId: dto.projectId,
decryptor: dto.decryptor,
folderDAL,
secretDAL: secretV2BridgeDAL,
+6 -1
View File
@@ -87,7 +87,12 @@ View the complete details <${appCfg.SITE_URL}/secret-manager/${payload.projectId
The following permissions are requested: ${payload.permissions.join(", ")}
View the request and approve or deny it <${payload.approvalUrl}|here>.`;
View the request and approve or deny it <${payload.approvalUrl}|here>.${
payload.note
? `
User Note: ${payload.note}`
: ""
}`;
const payloadBlocks = [
{
@@ -76,5 +76,6 @@ export type TSlackNotification =
projectName: string;
permissions: string[];
approvalUrl: string;
note?: string;
};
};
@@ -40,6 +40,9 @@
{{/each}}
</ul>
</p>
{{#if note}}
<p>User Note: "{{note}}"</p>
{{/if}}
<p>
View the request and approve or deny it
@@ -18,6 +18,8 @@ export enum PostHogEventTypes {
SecretRequestDeleted = "Secret Request Deleted",
SignSshKey = "Sign SSH Key",
IssueSshCreds = "Issue SSH Credentials",
IssueSshHostUserCert = "Issue SSH Host User Certificate",
IssueSshHostHostCert = "Issue SSH Host Host Certificate",
SignCert = "Sign PKI Certificate",
IssueCert = "Issue PKI Certificate"
}
@@ -161,6 +163,26 @@ export type TIssueSshCredsEvent = {
};
};
export type TIssueSshHostUserCertEvent = {
event: PostHogEventTypes.IssueSshHostUserCert;
properties: {
sshHostId: string;
hostname: string;
principals: string[];
userAgent?: string;
};
};
export type TIssueSshHostHostCertEvent = {
event: PostHogEventTypes.IssueSshHostHostCert;
properties: {
sshHostId: string;
hostname: string;
principals: string[];
userAgent?: string;
};
};
export type TSignCertificateEvent = {
event: PostHogEventTypes.SignCert;
properties: {
@@ -195,6 +217,8 @@ export type TPostHogEvent = { distinctId: string } & (
| TSecretRequestDeletedEvent
| TSignSshKeyEvent
| TIssueSshCredsEvent
| TIssueSshHostUserCertEvent
| TIssueSshHostHostCertEvent
| TSignCertificateEvent
| TIssueCertificateEvent
);