feat(KMS): New external KMS support for Google GCP KMS (#2825)

* feat(KMS): New external KMS support for Google GCP KMS
This commit is contained in:
McPizza
2024-12-03 18:14:42 +01:00
committed by GitHub
parent 7728a4793b
commit 5ceb30f43f
30 changed files with 1095 additions and 49 deletions
@@ -36,7 +36,7 @@ export const MultiValueRemove = (props: MultiValueRemoveProps) => {
export const Option = <T,>({ isSelected, children, ...props }: OptionProps<T>) => {
return (
<components.Option isSelected={isSelected} {...props}>
<div className="flex items-center">
<div className="flex flex-row items-center justify-between">
<p className="truncate">{children}</p>
{isSelected && (
<FontAwesomeIcon className="ml-2 text-primary" icon={faCheckCircle} size="sm" />
+1
View File
@@ -1,5 +1,6 @@
export {
useAddExternalKms,
useExternalKmsFetchGcpKeys,
useLoadProjectKmsBackup,
useRemoveExternalKms,
useUpdateExternalKms,
+49 -2
View File
@@ -3,7 +3,13 @@ import { useMutation, useQueryClient } from "@tanstack/react-query";
import { apiRequest } from "@app/config/request";
import { kmsKeys } from "./queries";
import { AddExternalKmsType, KmsType } from "./types";
import {
AddExternalKmsType,
ExternalKmsGcpSchemaType,
KmsGcpKeyFetchAuthType,
KmsType,
UpdateExternalKmsType
} from "./types";
export const useAddExternalKms = (orgId: string) => {
const queryClient = useQueryClient();
@@ -33,7 +39,7 @@ export const useUpdateExternalKms = (orgId: string) => {
provider
}: {
kmsId: string;
} & AddExternalKmsType) => {
} & UpdateExternalKmsType) => {
const { data } = await apiRequest.patch(`/api/v1/external-kms/${kmsId}`, {
name,
description,
@@ -96,3 +102,44 @@ export const useLoadProjectKmsBackup = (projectId: string) => {
}
});
};
export const useExternalKmsFetchGcpKeys = (orgId: string) => {
const queryClient = useQueryClient();
return useMutation({
mutationFn: async ({
gcpRegion,
...rest
}: Pick<ExternalKmsGcpSchemaType, "gcpRegion"> &
(
| (Pick<ExternalKmsGcpSchemaType, KmsGcpKeyFetchAuthType.Credential> & {
[KmsGcpKeyFetchAuthType.Kms]?: never;
})
| {
[KmsGcpKeyFetchAuthType.Kms]: string;
[KmsGcpKeyFetchAuthType.Credential]?: never;
}
)): Promise<{ keys: string[] }> => {
const {
[KmsGcpKeyFetchAuthType.Credential]: credential,
[KmsGcpKeyFetchAuthType.Kms]: kmsId
} = rest;
if ((credential && kmsId) || (!credential && !kmsId)) {
throw new Error(
`Either '${KmsGcpKeyFetchAuthType.Credential}' or '${KmsGcpKeyFetchAuthType.Kms}' must be provided, but not both.`
);
}
const { data } = await apiRequest.post("/api/v1/external-kms/gcp/keys", {
authMethod: credential ? KmsGcpKeyFetchAuthType.Credential : KmsGcpKeyFetchAuthType.Kms,
region: gcpRegion,
...rest
});
return data;
},
onSuccess: () => {
queryClient.invalidateQueries(kmsKeys.getExternalKmsList(orgId));
}
});
};
+101 -2
View File
@@ -35,7 +35,8 @@ export enum KmsType {
}
export enum ExternalKmsProvider {
AWS = "aws"
Aws = "aws",
Gcp = "gcp"
}
export const INTERNAL_KMS_KEY_ID = "internal";
@@ -44,6 +45,10 @@ export enum KmsAwsCredentialType {
AssumeRole = "assume-role",
AccessKey = "access-key"
}
// Google uses snake_case for their enum values and we need to match that
export enum KmsGcpCredentialType {
ServiceAccount = "service_account"
}
export const ExternalKmsAwsSchema = z.object({
credential: z
@@ -83,8 +88,34 @@ export const ExternalKmsAwsSchema = z.object({
)
});
export const ExternalKmsGcpCredentialSchema = z.object({
type: z.literal(KmsGcpCredentialType.ServiceAccount),
project_id: z.string().min(1),
private_key_id: z.string().min(1),
private_key: z.string().min(1),
client_email: z.string().min(1),
client_id: z.string().min(1),
auth_uri: z.string().min(1),
token_uri: z.string().min(1),
auth_provider_x509_cert_url: z.string().min(1),
client_x509_cert_url: z.string().min(1),
universe_domain: z.string().min(1)
});
export type ExternalKmsGcpCredentialSchemaType = z.infer<typeof ExternalKmsGcpCredentialSchema>;
export const ExternalKmsGcpSchema = z.object({
credential: ExternalKmsGcpCredentialSchema.describe(
"GCP Service Account JSON credential to connect"
),
gcpRegion: z.string().min(1).trim().describe("GCP region where the KMS key is located"),
keyName: z.string().min(1).trim().describe("GCP key name")
});
export type ExternalKmsGcpSchemaType = z.infer<typeof ExternalKmsGcpSchema>;
export const ExternalKmsInputSchema = z.discriminatedUnion("type", [
z.object({ type: z.literal(ExternalKmsProvider.AWS), inputs: ExternalKmsAwsSchema })
z.object({ type: z.literal(ExternalKmsProvider.Aws), inputs: ExternalKmsAwsSchema }),
z.object({ type: z.literal(ExternalKmsProvider.Gcp), inputs: ExternalKmsGcpSchema })
]);
export const AddExternalKmsSchema = z.object({
@@ -100,3 +131,71 @@ export const AddExternalKmsSchema = z.object({
});
export type AddExternalKmsType = z.infer<typeof AddExternalKmsSchema>;
// we need separate schema for update because the credential field is not required on GCP
export const ExternalKmsUpdateInputSchema = z.discriminatedUnion("type", [
z.object({ type: z.literal(ExternalKmsProvider.Aws), inputs: ExternalKmsAwsSchema }),
z.object({
type: z.literal(ExternalKmsProvider.Gcp),
inputs: ExternalKmsGcpSchema.pick({ gcpRegion: true, keyName: true })
})
]);
export const UpdateExternalKmsSchema = z.object({
name: z
.string()
.trim()
.min(1)
.refine((v) => slugify(v) === v, {
message: "Alias must be a valid slug"
}),
description: z.string().trim().optional(),
provider: ExternalKmsUpdateInputSchema
});
export type UpdateExternalKmsType = z.infer<typeof UpdateExternalKmsSchema>;
const GCP_CREDENTIAL_MAX_FILE_SIZE = 8 * 1024; // 8KB
const GCP_CREDENTIAL_ACCEPTED_FILE_TYPES = ["application/json"];
const AddExternalKmsGcpFormSchemaStandardInputs = z.object({
keyObject: z
.object({ label: z.string().trim(), value: z.string().trim() })
.describe("GCP key name"),
gcpRegion: z.object({ label: z.string().trim(), value: z.string().trim() }).describe("GCP Region")
});
export const AddExternalKmsGcpFormSchema = z.discriminatedUnion("formType", [
z
.object({
formType: z.literal("newGcpKms"),
// `FileList` is a browser-only (window-specific) type, so we need to handle it differently on the server to avoid SSR errors
credentialFile:
typeof window === "undefined"
? z.any()
: z
.instanceof(FileList)
.refine((files) => files?.length === 1, "Image is required.")
.refine(
(files) => files?.[0]?.size <= GCP_CREDENTIAL_MAX_FILE_SIZE,
"Max file size is 8KB."
)
.refine(
(files) => GCP_CREDENTIAL_ACCEPTED_FILE_TYPES.includes(files?.[0]?.type),
"Only .json files are accepted."
)
})
.merge(AddExternalKmsGcpFormSchemaStandardInputs)
.merge(AddExternalKmsSchema.pick({ name: true, description: true })),
z
.object({ formType: z.literal("updateGcpKms") })
.merge(AddExternalKmsGcpFormSchemaStandardInputs)
.merge(AddExternalKmsSchema.pick({ name: true, description: true }))
]);
export type AddExternalKmsGcpFormSchemaType = z.infer<typeof AddExternalKmsGcpFormSchema>;
export enum KmsGcpKeyFetchAuthType {
Credential = "credential",
Kms = "kmsId"
}
@@ -1,5 +1,5 @@
import { useState } from "react";
import { faAws } from "@fortawesome/free-brands-svg-icons";
import { faAws, faGoogle } from "@fortawesome/free-brands-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { AnimatePresence, motion } from "framer-motion";
@@ -7,6 +7,7 @@ import { Modal, ModalContent } from "@app/components/v2";
import { ExternalKmsProvider } from "@app/hooks/api/kms/types";
import { AwsKmsForm } from "./AwsKmsForm";
import { GcpKmsForm } from "./GcpKmsForm";
type Props = {
isOpen?: boolean;
@@ -21,8 +22,13 @@ enum WizardSteps {
const EXTERNAL_KMS_LIST = [
{
icon: faAws,
provider: ExternalKmsProvider.AWS,
provider: ExternalKmsProvider.Aws,
title: "AWS KMS"
},
{
icon: faGoogle,
provider: ExternalKmsProvider.Gcp,
title: "GCP KMS"
}
];
@@ -42,6 +48,7 @@ export const AddExternalKmsForm = ({ isOpen, onToggle }: Props) => {
title="Add a Key Management System"
subTitle="Configure an external key management system (KMS)"
className="my-4"
bodyClassName="overflow-visible"
>
<AnimatePresence exitBeforeEnter>
{wizardStep === WizardSteps.SelectProvider && (
@@ -79,7 +86,7 @@ export const AddExternalKmsForm = ({ isOpen, onToggle }: Props) => {
</motion.div>
)}
{wizardStep === WizardSteps.ProviderInputs &&
selectedProvider === ExternalKmsProvider.AWS && (
selectedProvider === ExternalKmsProvider.Aws && (
<motion.div
key="kms-aws"
transition={{ duration: 0.1 }}
@@ -90,6 +97,18 @@ export const AddExternalKmsForm = ({ isOpen, onToggle }: Props) => {
<AwsKmsForm onCancel={() => onToggle(false)} onCompleted={() => onToggle(false)} />
</motion.div>
)}
{wizardStep === WizardSteps.ProviderInputs &&
selectedProvider === ExternalKmsProvider.Gcp && (
<motion.div
key="kms-gcp"
transition={{ duration: 0.1 }}
initial={{ opacity: 0, translateX: 30 }}
animate={{ opacity: 1, translateX: 0 }}
exit={{ opacity: 0, translateX: -30 }}
>
<GcpKmsForm onCancel={() => onToggle(false)} onCompleted={() => onToggle(false)} />
</motion.div>
)}
</AnimatePresence>
</ModalContent>
</Modal>
@@ -64,7 +64,7 @@ export const AwsKmsForm = ({ onCompleted, onCancel, kms }: Props) => {
name: kms?.name,
description: kms?.description ?? "",
provider: {
type: ExternalKmsProvider.AWS,
type: ExternalKmsProvider.Aws,
inputs: {
credential: {
type: kms?.external?.providerInput?.credential?.type,
@@ -88,7 +88,7 @@ export const AwsKmsForm = ({ onCompleted, onCancel, kms }: Props) => {
const selectedAwsAuthType = watch("provider.inputs.credential.type");
const handleAddAwsKms = async (data: AddExternalKmsType) => {
const handleAwsKmsFormSubmit = async (data: AddExternalKmsType) => {
const { name, description, provider } = data;
try {
if (kms) {
@@ -123,7 +123,7 @@ export const AwsKmsForm = ({ onCompleted, onCancel, kms }: Props) => {
};
return (
<form onSubmit={handleSubmit(handleAddAwsKms)} autoComplete="off">
<form onSubmit={handleSubmit(handleAwsKmsFormSubmit)} autoComplete="off">
<Controller
control={control}
name="name"
@@ -0,0 +1,360 @@
import { useEffect, useState } from "react";
import { Controller, useForm } from "react-hook-form";
import { zodResolver } from "@hookform/resolvers/zod";
import { createNotification } from "@app/components/notifications";
import { Badge, Button, FilterableSelect, FormControl, Input } from "@app/components/v2";
import { useOrganization } from "@app/context";
import {
useAddExternalKms,
useExternalKmsFetchGcpKeys,
useUpdateExternalKms
} from "@app/hooks/api";
import {
AddExternalKmsGcpFormSchema,
AddExternalKmsGcpFormSchemaType,
ExternalKmsGcpCredentialSchema,
ExternalKmsGcpCredentialSchemaType,
ExternalKmsProvider,
Kms
} from "@app/hooks/api/kms/types";
type Props = {
onCompleted: () => void;
onCancel: () => void;
kms?: Kms;
};
const GCP_REGIONS = [
{ label: "Johannesburg", value: "africa-south1" },
{ label: "Taiwan", value: "asia-east1" },
{ label: "Hong Kong", value: "asia-east2" },
{ label: "Tokyo", value: "asia-northeast1" },
{ label: "Osaka", value: "asia-northeast2" },
{ label: "Seoul", value: "asia-northeast3" },
{ label: "Mumbai", value: "asia-south1" },
{ label: "Delhi", value: "asia-south2" },
{ label: "Singapore", value: "asia-southeast1" },
{ label: "Jakarta", value: "asia-southeast2" },
{ label: "Sydney", value: "australia-southeast1" },
{ label: "Melbourne", value: "australia-southeast2" },
{ label: "Warsaw", value: "europe-central2" },
{ label: "Finland", value: "europe-north1" },
{ label: "Belgium", value: "europe-west1" },
{ label: "London", value: "europe-west2" },
{ label: "Frankfurt", value: "europe-west3" },
{ label: "Netherlands", value: "europe-west4" },
{ label: "Zurich", value: "europe-west6" },
{ label: "Milan", value: "europe-west8" },
{ label: "Paris", value: "europe-west9" },
{ label: "Berlin", value: "europe-west10" },
{ label: "Turin", value: "europe-west12" },
{ label: "Madrid", value: "europe-southwest1" },
{ label: "Doha", value: "me-central1" },
{ label: "Dammam", value: "me-central2" },
{ label: "Tel Aviv", value: "me-west1" },
{ label: "Montréal", value: "northamerica-northeast1" },
{ label: "Toronto", value: "northamerica-northeast2" },
{ label: "São Paulo", value: "southamerica-east1" },
{ label: "Santiago", value: "southamerica-west1" },
{ label: "Iowa", value: "us-central1" },
{ label: "South Carolina", value: "us-east1" },
{ label: "North Virginia", value: "us-east4" },
{ label: "Columbus", value: "us-east5" },
{ label: "Dallas", value: "us-south1" },
{ label: "Oregon", value: "us-west1" },
{ label: "Los Angeles", value: "us-west2" },
{ label: "Salt Lake City", value: "us-west3" },
{ label: "Las Vegas", value: "us-west4" }
];
const formatOptionLabel = ({ value, label }: { value: string; label: string }) => (
<div className="flex w-full flex-row items-center justify-between">
<span>{label}</span>
<Badge variant="success">{value}</Badge>
</div>
);
export const GcpKmsForm = ({ onCompleted, onCancel, kms }: Props) => {
const [isCredentialValid, setIsCredentialValid] = useState<boolean>(false);
const [keys, setKeys] = useState<{ value: string; label: string }[]>([]);
const {
control,
handleSubmit,
setError,
clearErrors,
getValues,
resetField,
setValue,
formState: { isSubmitting }
} = useForm<AddExternalKmsGcpFormSchemaType>({
resolver: zodResolver(AddExternalKmsGcpFormSchema),
defaultValues: {
formType: kms ? "updateGcpKms" : "newGcpKms",
name: kms?.name ?? "",
description: kms?.description ?? "",
gcpRegion: kms
? {
label:
GCP_REGIONS.find((r) => r.value === kms.external.providerInput.gcpRegion)?.label ??
"",
value: kms.external.providerInput.gcpRegion
}
: undefined,
keyObject: undefined
}
});
const { currentOrg } = useOrganization();
const { mutateAsync: addGcpExternalKms } = useAddExternalKms(currentOrg?.id!);
const { mutateAsync: updateGcpExternalKms } = useUpdateExternalKms(currentOrg?.id!);
const { mutateAsync: fetchGcpKeys, isLoading: isFetchGcpKeysLoading } =
useExternalKmsFetchGcpKeys(currentOrg?.id!);
// transforms the credential file into a JSON object
async function getCredentialFileJson(): Promise<ExternalKmsGcpCredentialSchemaType | null> {
const files = getValues("credentialFile");
if (!files || !files.length) {
return null;
}
const file = files[0];
if (file.type !== "application/json") {
setError("credentialFile", {
message: "Only .json files are accepted."
});
return null;
}
const jsonContents = await file.text();
const parsedJson = ExternalKmsGcpCredentialSchema.safeParse(JSON.parse(jsonContents));
if (!parsedJson.success) {
setError("credentialFile", {
message: "Invalid Service Account credential JSON."
});
return null;
}
clearErrors("credentialFile");
return parsedJson.data;
}
// handles the form submission
const handleGcpKmsFormSubmit = async (data: AddExternalKmsGcpFormSchemaType) => {
const { name, description, gcpRegion: gcpRegionObject, keyObject } = data;
const gcpRegion = gcpRegionObject.value;
if (!keys.find((k) => k.value === keyObject?.value)) {
setError("keyObject", {
message: "Please select a valid key."
});
resetField("keyObject");
return;
}
try {
if (kms) {
await updateGcpExternalKms({
kmsId: kms.id,
name,
description,
provider: {
type: ExternalKmsProvider.Gcp,
inputs: {
gcpRegion,
keyName: keyObject?.value
}
}
});
createNotification({
text: "Successfully updated GCP External KMS",
type: "success"
});
} else {
const credentialJson = await getCredentialFileJson();
if (!credentialJson) {
return;
}
await addGcpExternalKms({
name,
description,
provider: {
type: ExternalKmsProvider.Gcp,
inputs: {
gcpRegion,
keyName: keyObject?.value,
credential: credentialJson
}
}
});
createNotification({
text: "Successfully added GCP External KMS",
type: "success"
});
}
onCompleted();
} catch (err) {
console.error(err);
}
};
const fetchGCPKeys = async () => {
// @ts-expect-error - issue with the way react-select renders the placeholder. We need to set the value to null explicitly otherwise it will not re-render
setValue("keyObject", null);
setKeys([]);
const credentialJson = kms ? undefined : await getCredentialFileJson();
if (!kms && !credentialJson) {
return;
}
const gcpRegion = getValues("gcpRegion").value;
if (!gcpRegion.length) {
setError("gcpRegion", {
message: "Please select a GCP region to fetch GCP Keys."
});
return;
}
const res = await fetchGcpKeys({
gcpRegion,
...(kms ? { kmsId: kms.id } : { credential: credentialJson! })
});
setIsCredentialValid(!!res.keys);
const returnedKeys = res.keys.map((key) => {
const parts = key.split("/");
const keyLabel = `${parts[5]}/${parts[7]}`;
return {
value: key,
label: keyLabel
};
});
setKeys(returnedKeys);
if (kms) {
const existingKey = returnedKeys.find((k) => k.value === kms.external.providerInput.keyName);
if (existingKey) {
setValue("keyObject", existingKey);
}
}
};
const getPlaceholderText = () => {
if (isFetchGcpKeysLoading) {
return "Loading keys in this region...";
}
if (!isCredentialValid) {
return "Upload a valid credential file";
}
if (keys.length) {
return "Select a key";
}
return "No valid keys found in this region";
};
useEffect(() => {
if (kms && !isCredentialValid) {
fetchGCPKeys();
}
}, [kms]);
return (
<form onSubmit={handleSubmit(handleGcpKmsFormSubmit)} autoComplete="off">
<Controller
control={control}
name="name"
render={({ field, fieldState: { error } }) => (
<FormControl label="Alias" errorText={error?.message} isError={Boolean(error)}>
<Input placeholder="" {...field} />
</FormControl>
)}
/>
<Controller
control={control}
name="description"
render={({ field, fieldState: { error } }) => (
<FormControl label="Description" errorText={error?.message} isError={Boolean(error)}>
<Input placeholder="" {...field} />
</FormControl>
)}
/>
<Controller
control={control}
name="gcpRegion"
render={({ field, fieldState: { error } }) => (
<FormControl label="GCP Region" errorText={error?.message} isError={Boolean(error)}>
<FilterableSelect
className="w-full"
placeholder="Select a GCP region"
name="gcpRegion"
options={GCP_REGIONS}
value={field.value}
onChange={(e) => {
resetField("keyObject");
field.onChange(e);
fetchGCPKeys();
}}
formatOptionLabel={formatOptionLabel}
/>
</FormControl>
)}
/>
{!kms && (
<Controller
control={control}
name="credentialFile"
render={({ field: { value, onChange, ref, ...rest }, fieldState: { error } }) => (
<FormControl
label="Service Account Credential JSON"
errorText={error?.message}
isError={Boolean(error)}
>
<Input
{...rest}
ref={ref}
type="file"
accept=".json"
placeholder=""
value={value?.filename}
onChange={(e) => {
onChange(e.target.files);
fetchGCPKeys();
}}
/>
</FormControl>
)}
/>
)}
<Controller
control={control}
name="keyObject"
render={({ field, fieldState: { error } }) => (
<FormControl label="GCP Key Name" errorText={error?.message} isError={Boolean(error)}>
<FilterableSelect
className="w-full"
placeholder={getPlaceholderText()}
isDisabled={!isCredentialValid || !keys.length}
name="key"
options={keys}
value={field.value}
onChange={field.onChange}
/>
</FormControl>
)}
/>
{kms && (
<span className="text-xs text-mineshaft-300">
To change your GCP credentials, create a new external KMS and assign it to project you
want to use it with.
</span>
)}
<div className="mt-6 flex items-center space-x-4">
<Button type="submit" isLoading={isSubmitting}>
Save
</Button>
<Button variant="outline_bg" onClick={onCancel}>
Cancel
</Button>
</div>
</form>
);
};
@@ -1,4 +1,4 @@
import { faAws } from "@fortawesome/free-brands-svg-icons";
import { faAws, faGoogle } from "@fortawesome/free-brands-svg-icons";
import { faEllipsis, faLock, faPlus } from "@fortawesome/free-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { twMerge } from "tailwind-merge";
@@ -117,9 +117,12 @@ export const OrgEncryptionTab = withPermission(
externalKmsList?.map((kms) => (
<Tr key={kms.id}>
<Td className="flex max-w-xs items-center overflow-hidden text-ellipsis hover:overflow-auto hover:break-all">
{kms.externalKms.provider === ExternalKmsProvider.AWS && (
{kms.externalKms.provider === ExternalKmsProvider.Aws && (
<FontAwesomeIcon icon={faAws} />
)}
{kms.externalKms.provider === ExternalKmsProvider.Gcp && (
<FontAwesomeIcon icon={faGoogle} />
)}
<div className="ml-2">{kms.externalKms.provider.toUpperCase()}</div>
</Td>
<Td>{kms.name}</Td>
@@ -3,6 +3,7 @@ import { useGetExternalKmsById } from "@app/hooks/api";
import { ExternalKmsProvider } from "@app/hooks/api/kms/types";
import { AwsKmsForm } from "./AwsKmsForm";
import { GcpKmsForm } from "./GcpKmsForm";
type Props = {
isOpen: boolean;
@@ -14,15 +15,22 @@ export const UpdateExternalKmsForm = ({ isOpen, kmsId, onOpenChange }: Props) =>
const { data: externalKms, isLoading } = useGetExternalKmsById(kmsId);
return (
<Modal isOpen={isOpen} onOpenChange={onOpenChange}>
<ModalContent title="Edit configuration">
<ModalContent title="Edit configuration" bodyClassName="overflow-visible">
{isLoading && <ContentLoader />}
{externalKms?.external?.provider === ExternalKmsProvider.AWS && (
{externalKms?.external?.provider === ExternalKmsProvider.Aws && (
<AwsKmsForm
kms={externalKms}
onCancel={() => onOpenChange(false)}
onCompleted={() => onOpenChange(false)}
/>
)}
{externalKms?.external?.provider === ExternalKmsProvider.Gcp && (
<GcpKmsForm
kms={externalKms}
onCancel={() => onOpenChange(false)}
onCompleted={() => onOpenChange(false)}
/>
)}
</ModalContent>
</Modal>
);