awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-14 17:40:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(KMS): New external KMS support for Google GCP KMS (#2825)

Browse Source 
* feat(KMS): New external KMS support for Google GCP KMS
This commit is contained in:
McPizza
2024-12-03 18:14:42 +01:00
committed by GitHub
parent 7728a4793b
commit 5ceb30f43f
30 changed files with 1095 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/package-lock.json
Generated
+63
View File
@@ -28,6 +28,7 @@
"@fastify/session": "^10.7.0",
"@fastify/swagger": "^8.14.0",
"@fastify/swagger-ui": "^2.1.0",
"@google-cloud/kms": "^4.5.0",
"@node-saml/passport-saml": "^4.0.4",
"@octokit/auth-app": "^7.1.1",
"@octokit/plugin-retry": "^5.0.5",
@@ -5598,6 +5599,18 @@
"yaml": "^2.2.2"
}
},
"node_modules/@google-cloud/kms": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/@google-cloud/kms/-/kms-4.5.0.tgz",
"integrity": "sha512-i2vC0DI7bdfEhQszqASTw0KVvbB7HsO2CwTBod423NawAu7FWi+gVVa7NLfXVNGJaZZayFfci2Hu+om/HmyEjQ==",
"license": "Apache-2.0",
"dependencies": {
"google-gax": "^4.0.3"
},
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/@google-cloud/paginator": {
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.2.tgz",
@@ -15086,6 +15099,44 @@
"safe-buffer": "^5.0.1"
}
},
"node_modules/google-gax": {
"version": "4.4.1",
"resolved": "https://registry.npmjs.org/google-gax/-/google-gax-4.4.1.tgz",
"integrity": "sha512-Phyp9fMfA00J3sZbJxbbB4jC55b7DBjE3F6poyL3wKMEBVKA79q6BGuHcTiM28yOzVql0NDbRL8MLLh8Iwk9Dg==",
"license": "Apache-2.0",
"dependencies": {
"@grpc/grpc-js": "^1.10.9",
"@grpc/proto-loader": "^0.7.13",
"@types/long": "^4.0.0",
"abort-controller": "^3.0.0",
"duplexify": "^4.0.0",
"google-auth-library": "^9.3.0",
"node-fetch": "^2.7.0",
"object-hash": "^3.0.0",
"proto3-json-serializer": "^2.0.2",
"protobufjs": "^7.3.2",
"retry-request": "^7.0.0",
"uuid": "^9.0.1"
},
"engines": {
"node": ">=14"
}
},
"node_modules/google-gax/node_modules/@types/long": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz",
"integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==",
"license": "MIT"
},
"node_modules/google-gax/node_modules/object-hash": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz",
"integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==",
"license": "MIT",
"engines": {
"node": ">= 6"
}
},
"node_modules/googleapis": {
"version": "137.1.0",
"resolved": "https://registry.npmjs.org/googleapis/-/googleapis-137.1.0.tgz",
@@ -19223,6 +19274,18 @@
"node": ">=6"
}
},
"node_modules/proto3-json-serializer": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-2.0.2.tgz",
"integrity": "sha512-SAzp/O4Yh02jGdRc+uIrGoe87dkN/XtwxfZ4ZyafJHymd79ozp5VG5nyZ7ygqPM5+cpLDjjGnYFUkngonyDPOQ==",
"license": "Apache-2.0",
"dependencies": {
"protobufjs": "^7.2.5"
},
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/protobufjs": {
"version": "7.4.0",
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz",
backend/package.json
+1
View File
@@ -136,6 +136,7 @@
"@fastify/session": "^10.7.0",
"@fastify/swagger": "^8.14.0",
"@fastify/swagger-ui": "^2.1.0",
"@google-cloud/kms": "^4.5.0",
"@node-saml/passport-saml": "^4.0.4",
"@octokit/auth-app": "^7.1.1",
"@octokit/plugin-retry": "^5.0.5",
backend/src/ee/routes/v1/external-kms-router.ts
+72 -2
View File
@@ -4,9 +4,15 @@ import { ExternalKmsSchema, KmsKeysSchema } from "@app/db/schemas";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import {
ExternalKmsAwsSchema,
ExternalKmsGcpCredentialSchema,
ExternalKmsGcpSchema,
ExternalKmsInputSchema,
ExternalKmsInputUpdateSchema
ExternalKmsInputUpdateSchema,
KmsGcpKeyFetchAuthType,
KmsProviders,
TExternalKmsGcpCredentialSchema
} from "@app/ee/services/external-kms/providers/model";
import { NotFoundError } from "@app/lib/errors";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { AuthMode } from "@app/services/auth/auth-type";
@@ -44,7 +50,8 @@ const sanitizedExternalSchemaForGetById = KmsKeysSchema.extend({
statusDetails: true,
provider: true
}).extend({
providerInput: ExternalKmsAwsSchema
// for GCP, we don't return the credential object as it is sensitive data that should not be exposed
providerInput: z.union([ExternalKmsAwsSchema, ExternalKmsGcpSchema.pick({ gcpRegion: true, keyName: true })])
})
});
@@ -286,4 +293,67 @@ export const registerExternalKmsRouter = async (server: FastifyZodProvider) => {
return { externalKms };
}
});
server.route({
method: "POST",
url: "/gcp/keys",
config: {
rateLimit: writeLimit
},
schema: {
body: z.discriminatedUnion("authMethod", [
z.object({
authMethod: z.literal(KmsGcpKeyFetchAuthType.Credential),
region: z.string().trim().min(1),
credential: ExternalKmsGcpCredentialSchema
}),
z.object({
authMethod: z.literal(KmsGcpKeyFetchAuthType.Kms),
region: z.string().trim().min(1),
kmsId: z.string().trim().min(1)
})
]),
response: {
200: z.object({
keys: z.string().array()
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
handler: async (req) => {
const { region, authMethod } = req.body;
let credentialJson: TExternalKmsGcpCredentialSchema | undefined;
if (authMethod === KmsGcpKeyFetchAuthType.Credential) {
credentialJson = req.body.credential;
} else if (authMethod === KmsGcpKeyFetchAuthType.Kms) {
const externalKms = await server.services.externalKms.findById({
actor: req.permission.type,
actorId: req.permission.id,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
id: req.body.kmsId
});
if (!externalKms || externalKms.external.provider !== KmsProviders.Gcp) {
throw new NotFoundError({ message: "KMS not found or not of type GCP" });
}
credentialJson = externalKms.external.providerInput.credential as TExternalKmsGcpCredentialSchema;
}
if (!credentialJson) {
throw new NotFoundError({
message: "Something went wrong while fetching the GCP credential, please check inputs and try again"
});
}
const results = await server.services.externalKms.fetchGcpKeys({
credential: credentialJson,
gcpRegion: region
});
return results;
}
});
};
backend/src/ee/services/external-kms/external-kms-service.ts
+46 -7
View File
@@ -20,7 +20,8 @@ import {
TUpdateExternalKmsDTO
} from "./external-kms-types";
import { AwsKmsProviderFactory } from "./providers/aws-kms";
import { ExternalKmsAwsSchema, KmsProviders } from "./providers/model";
import { GcpKmsProviderFactory } from "./providers/gcp-kms";
import { ExternalKmsAwsSchema, ExternalKmsGcpSchema, KmsProviders, TExternalKmsGcpSchema } from "./providers/model";
type TExternalKmsServiceFactoryDep = {
externalKmsDAL: TExternalKmsDALFactory;
@@ -78,6 +79,13 @@ export const externalKmsServiceFactory = ({
await externalKms.validateConnection();
}
break;
case KmsProviders.Gcp:
{
const externalKms = await GcpKmsProviderFactory({ inputs: provider.inputs });
await externalKms.validateConnection();
sanitizedProviderInput = JSON.stringify(provider.inputs);
}
break;
default:
throw new BadRequestError({ message: "external kms provided is invalid" });
}
@@ -88,7 +96,7 @@ export const externalKmsServiceFactory = ({
});
const { cipherTextBlob: encryptedProviderInputs } = orgDataKeyEncryptor({
plainText: Buffer.from(sanitizedProviderInput, "utf8")
plainText: Buffer.from(sanitizedProviderInput)
});
const externalKms = await externalKmsDAL.transaction(async (tx) => {
@@ -162,7 +170,7 @@ export const externalKmsServiceFactory = ({
case KmsProviders.Aws:
{
const decryptedProviderInput = await ExternalKmsAwsSchema.parseAsync(
JSON.parse(decryptedProviderInputBlob.toString("utf8"))
JSON.parse(decryptedProviderInputBlob.toString())
);
const updatedProviderInput = { ...decryptedProviderInput, ...provider.inputs };
const externalKms = await AwsKmsProviderFactory({ inputs: updatedProviderInput });
@@ -170,6 +178,17 @@ export const externalKmsServiceFactory = ({
sanitizedProviderInput = JSON.stringify(updatedProviderInput);
}
break;
case KmsProviders.Gcp:
{
const decryptedProviderInput = await ExternalKmsGcpSchema.parseAsync(
JSON.parse(decryptedProviderInputBlob.toString())
);
const updatedProviderInput = { ...decryptedProviderInput, ...provider.inputs };
const externalKms = await GcpKmsProviderFactory({ inputs: updatedProviderInput });
await externalKms.validateConnection();
sanitizedProviderInput = JSON.stringify(updatedProviderInput);
}
break;
default:
throw new BadRequestError({ message: "external kms provided is invalid" });
}
@@ -178,7 +197,7 @@ export const externalKmsServiceFactory = ({
let encryptedProviderInputs: Buffer | undefined;
if (sanitizedProviderInput) {
const { cipherTextBlob } = orgDataKeyEncryptor({
plainText: Buffer.from(sanitizedProviderInput, "utf8")
plainText: Buffer.from(sanitizedProviderInput)
});
encryptedProviderInputs = cipherTextBlob;
}
@@ -271,10 +290,17 @@ export const externalKmsServiceFactory = ({
switch (externalKmsDoc.provider) {
case KmsProviders.Aws: {
const decryptedProviderInput = await ExternalKmsAwsSchema.parseAsync(
JSON.parse(decryptedProviderInputBlob.toString("utf8"))
JSON.parse(decryptedProviderInputBlob.toString())
);
return { ...kmsDoc, external: { ...externalKmsDoc, providerInput: decryptedProviderInput } };
}
case KmsProviders.Gcp: {
const decryptedProviderInput = await ExternalKmsGcpSchema.parseAsync(
JSON.parse(decryptedProviderInputBlob.toString())
);
return { ...kmsDoc, external: { ...externalKmsDoc, providerInput: decryptedProviderInput } };
}
default:
throw new BadRequestError({ message: "external kms provided is invalid" });
}
@@ -312,21 +338,34 @@ export const externalKmsServiceFactory = ({
switch (externalKmsDoc.provider) {
case KmsProviders.Aws: {
const decryptedProviderInput = await ExternalKmsAwsSchema.parseAsync(
JSON.parse(decryptedProviderInputBlob.toString("utf8"))
JSON.parse(decryptedProviderInputBlob.toString())
);
return { ...kmsDoc, external: { ...externalKmsDoc, providerInput: decryptedProviderInput } };
}
case KmsProviders.Gcp: {
const decryptedProviderInput = await ExternalKmsGcpSchema.parseAsync(
JSON.parse(decryptedProviderInputBlob.toString())
);
return { ...kmsDoc, external: { ...externalKmsDoc, providerInput: decryptedProviderInput } };
}
default:
throw new BadRequestError({ message: "external kms provided is invalid" });
}
};
const fetchGcpKeys = async ({ credential, gcpRegion }: Pick<TExternalKmsGcpSchema, "credential" | "gcpRegion">) => {
const externalKms = await GcpKmsProviderFactory({ inputs: { credential, gcpRegion, keyName: "" } });
return externalKms.getKeysList();
};
return {
create,
updateById,
deleteById,
list,
findById,
findByName
findByName,
fetchGcpKeys
};
};
backend/src/ee/services/external-kms/providers/gcp-kms.ts
+113
View File
@@ -0,0 +1,113 @@
import { KeyManagementServiceClient } from "@google-cloud/kms";
import { BadRequestError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
import { ExternalKmsGcpSchema, TExternalKmsGcpClientSchema, TExternalKmsProviderFns } from "./model";
const getGcpKmsClient = async ({ credential, gcpRegion }: TExternalKmsGcpClientSchema) => {
const gcpKmsClient = new KeyManagementServiceClient({
credentials: credential
});
const projectId = credential.project_id;
const locationName = gcpKmsClient.locationPath(projectId, gcpRegion);
return {
gcpKmsClient,
locationName
};
};
type GcpKmsProviderArgs = {
inputs: unknown;
};
type TGcpKmsProviderFactoryReturn = TExternalKmsProviderFns & {
getKeysList: () => Promise<{ keys: string[] }>;
};
export const GcpKmsProviderFactory = async ({ inputs }: GcpKmsProviderArgs): Promise<TGcpKmsProviderFactoryReturn> => {
const { credential, gcpRegion, keyName } = await ExternalKmsGcpSchema.parseAsync(inputs);
const { gcpKmsClient, locationName } = await getGcpKmsClient({
credential,
gcpRegion
});
const validateConnection = async () => {
try {
await gcpKmsClient.listKeyRings({
parent: locationName
});
return true;
} catch (error) {
throw new BadRequestError({
message: "Cannot connect to GCP KMS"
});
}
};
// Used when adding the KMS to fetch the list of keys in specified region
const getKeysList = async () => {
try {
const [keyRings] = await gcpKmsClient.listKeyRings({
parent: locationName
});
const validKeyRings = keyRings
.filter(
(keyRing): keyRing is { name: string } =>
keyRing !== null && typeof keyRing === "object" && "name" in keyRing && typeof keyRing.name === "string"
)
.map((keyRing) => keyRing.name);
const keyList: string[] = [];
const keyListPromises = validKeyRings.map((keyRingName) =>
gcpKmsClient
.listCryptoKeys({
parent: keyRingName
})
.then(([cryptoKeys]) =>
cryptoKeys
.filter(
(key): key is { name: string } =>
key !== null && typeof key === "object" && "name" in key && typeof key.name === "string"
)
.map((key) => key.name)
)
);
const cryptoKeyLists = await Promise.all(keyListPromises);
keyList.push(...cryptoKeyLists.flat());
return { keys: keyList };
} catch (error) {
logger.error(error, "Could not validate GCP KMS connection and credentials");
throw new BadRequestError({
message: "Could not validate GCP KMS connection and credentials",
error
});
}
};
const encrypt = async (data: Buffer) => {
const encryptedText = await gcpKmsClient.encrypt({
name: keyName,
plaintext: data
});
if (!encryptedText[0].ciphertext) throw new Error("encryption failed");
return { encryptedBlob: Buffer.from(encryptedText[0].ciphertext) };
};
const decrypt = async (encryptedBlob: Buffer) => {
const decryptedText = await gcpKmsClient.decrypt({
name: keyName,
ciphertext: encryptedBlob
});
if (!decryptedText[0].plaintext) throw new Error("decryption failed");
return { data: Buffer.from(decryptedText[0].plaintext) };
};
return {
validateConnection,
getKeysList,
encrypt,
decrypt
};
};
backend/src/ee/services/external-kms/providers/model.ts
+43 -3
View File
@@ -1,13 +1,23 @@
import { z } from "zod";
export enum KmsProviders {
Aws = "aws"
Aws = "aws",
Gcp = "gcp"
}
export enum KmsAwsCredentialType {
AssumeRole = "assume-role",
AccessKey = "access-key"
}
// Google uses snake_case for their enum values and we need to match that
export enum KmsGcpCredentialType {
ServiceAccount = "service_account"
}
export enum KmsGcpKeyFetchAuthType {
Credential = "credential",
Kms = "kmsId"
}
export const ExternalKmsAwsSchema = z.object({
credential: z
@@ -42,14 +52,44 @@ export const ExternalKmsAwsSchema = z.object({
});
export type TExternalKmsAwsSchema = z.infer<typeof ExternalKmsAwsSchema>;
export const ExternalKmsGcpCredentialSchema = z.object({
type: z.literal(KmsGcpCredentialType.ServiceAccount),
project_id: z.string().min(1),
private_key_id: z.string().min(1),
private_key: z.string().min(1),
client_email: z.string().min(1),
client_id: z.string().min(1),
auth_uri: z.string().min(1),
token_uri: z.string().min(1),
auth_provider_x509_cert_url: z.string().min(1),
client_x509_cert_url: z.string().min(1),
universe_domain: z.string().min(1)
});
export type TExternalKmsGcpCredentialSchema = z.infer<typeof ExternalKmsGcpCredentialSchema>;
export const ExternalKmsGcpSchema = z.object({
credential: ExternalKmsGcpCredentialSchema.describe("GCP Service Account JSON credential to connect"),
gcpRegion: z.string().trim().describe("GCP region where the KMS key is located"),
keyName: z.string().trim().describe("GCP key name")
});
export type TExternalKmsGcpSchema = z.infer<typeof ExternalKmsGcpSchema>;
const ExternalKmsGcpClientSchema = ExternalKmsGcpSchema.pick({ gcpRegion: true }).extend({
credential: ExternalKmsGcpCredentialSchema
});
export type TExternalKmsGcpClientSchema = z.infer<typeof ExternalKmsGcpClientSchema>;
// The root schema of the JSON
export const ExternalKmsInputSchema = z.discriminatedUnion("type", [
z.object({ type: z.literal(KmsProviders.Aws), inputs: ExternalKmsAwsSchema })
z.object({ type: z.literal(KmsProviders.Aws), inputs: ExternalKmsAwsSchema }),
z.object({ type: z.literal(KmsProviders.Gcp), inputs: ExternalKmsGcpSchema })
]);
export type TExternalKmsInputSchema = z.infer<typeof ExternalKmsInputSchema>;
export const ExternalKmsInputUpdateSchema = z.discriminatedUnion("type", [
z.object({ type: z.literal(KmsProviders.Aws), inputs: ExternalKmsAwsSchema.partial() })
z.object({ type: z.literal(KmsProviders.Aws), inputs: ExternalKmsAwsSchema.partial() }),
z.object({ type: z.literal(KmsProviders.Gcp), inputs: ExternalKmsGcpSchema.partial() })
]);
export type TExternalKmsInputUpdateSchema = z.infer<typeof ExternalKmsInputUpdateSchema>;
backend/src/services/kms/kms-service.ts
+22
View File
@@ -4,8 +4,10 @@ import { z } from "zod";
import { KmsKeysSchema, TKmsRootConfig } from "@app/db/schemas";
import { AwsKmsProviderFactory } from "@app/ee/services/external-kms/providers/aws-kms";
import { GcpKmsProviderFactory } from "@app/ee/services/external-kms/providers/gcp-kms";
import {
ExternalKmsAwsSchema,
ExternalKmsGcpSchema,
KmsProviders,
TExternalKmsProviderFns
} from "@app/ee/services/external-kms/providers/model";
@@ -291,6 +293,16 @@ export const kmsServiceFactory = ({
});
break;
}
case KmsProviders.Gcp: {
const decryptedProviderInput = await ExternalKmsGcpSchema.parseAsync(
JSON.parse(decryptedProviderInputBlob.toString("utf8"))
);
externalKms = await GcpKmsProviderFactory({
inputs: decryptedProviderInput
});
break;
}
default:
throw new Error("Invalid KMS provider.");
}
@@ -353,6 +365,16 @@ export const kmsServiceFactory = ({
});
break;
}
case KmsProviders.Gcp: {
const decryptedProviderInput = await ExternalKmsGcpSchema.parseAsync(
JSON.parse(decryptedProviderInputBlob.toString("utf8"))
);
externalKms = await GcpKmsProviderFactory({
inputs: decryptedProviderInput
});
break;
}
default:
throw new Error("Invalid KMS provider.");
}