feat: implmented first internal functions for KMS service

This commit is contained in:
=
2024-06-04 02:10:12 +05:30
parent a0653883b6
commit 1acf25dd53
20 changed files with 408 additions and 5 deletions

View File

@@ -28,6 +28,7 @@ export const mockKeyStore = (): TKeyStoreFactory => {
},
acquireLock: () => {
throw new Error("Not implemented");
}
},
waitTillReady: async () => {}
};
};

View File

@@ -98,6 +98,15 @@ import {
TIntegrations,
TIntegrationsInsert,
TIntegrationsUpdate,
TKmsKeys,
TKmsKeysInsert,
TKmsKeysUpdate,
TKmsKeyVersions,
TKmsKeyVersionsInsert,
TKmsKeyVersionsUpdate,
TKmsRootConfig,
TKmsRootConfigInsert,
TKmsRootConfigUpdate,
TLdapConfigs,
TLdapConfigsInsert,
TLdapConfigsUpdate,
@@ -176,6 +185,9 @@ import {
TSecretImports,
TSecretImportsInsert,
TSecretImportsUpdate,
TSecretReferences,
TSecretReferencesInsert,
TSecretReferencesUpdate,
TSecretRotationOutputs,
TSecretRotationOutputsInsert,
TSecretRotationOutputsUpdate,
@@ -240,7 +252,6 @@ import {
TWebhooksInsert,
TWebhooksUpdate
} from "@app/db/schemas";
import { TSecretReferences, TSecretReferencesInsert, TSecretReferencesUpdate } from "@app/db/schemas/secret-references";
declare module "knex/types/tables" {
interface Tables {
@@ -514,5 +525,13 @@ declare module "knex/types/tables" {
TSecretVersionTagJunctionInsert,
TSecretVersionTagJunctionUpdate
>;
// KMS service
[TableName.KmsServerRootConfig]: Knex.CompositeTableType<
TKmsRootConfig,
TKmsRootConfigInsert,
TKmsRootConfigUpdate
>;
[TableName.KmsKey]: Knex.CompositeTableType<TKmsKeys, TKmsKeysInsert, TKmsKeysUpdate>;
[TableName.KmsKeyVersion]: Knex.CompositeTableType<TKmsKeyVersions, TKmsKeyVersionsInsert, TKmsKeyVersionsUpdate>;
}
}

View File

@@ -0,0 +1,51 @@
import { Knex } from "knex";
import { TableName } from "../schemas";
import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
export async function up(knex: Knex): Promise<void> {
if (!(await knex.schema.hasTable(TableName.KmsServerRootConfig))) {
await knex.schema.createTable(TableName.KmsServerRootConfig, (t) => {
t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
t.binary("encryptedRootKey").notNullable();
});
}
await createOnUpdateTrigger(knex, TableName.KmsServerRootConfig);
if (!(await knex.schema.hasTable(TableName.KmsKey))) {
await knex.schema.createTable(TableName.KmsKey, (t) => {
t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
t.binary("encryptedKey").notNullable();
t.string("encryptionAlgorithm").notNullable();
t.integer("version").defaultTo(1).notNullable();
t.string("description");
t.boolean("isDisabled").defaultTo(false);
});
}
await createOnUpdateTrigger(knex, TableName.KmsKey);
if (!(await knex.schema.hasTable(TableName.KmsKeyVersion))) {
await knex.schema.createTable(TableName.KmsKeyVersion, (t) => {
t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
t.binary("encryptedKey").notNullable();
t.integer("version").notNullable();
t.uuid("kmsKeyId").notNullable();
t.foreign("kmsKeyId").references("id").inTable(TableName.KmsKey).onDelete("CASCADE");
});
}
await createOnUpdateTrigger(knex, TableName.KmsKeyVersion);
}
export async function down(knex: Knex): Promise<void> {
await knex.schema.dropTableIfExists(TableName.KmsServerRootConfig);
await dropOnUpdateTrigger(knex, TableName.KmsServerRootConfig);
await knex.schema.dropTableIfExists(TableName.KmsKeyVersion);
await dropOnUpdateTrigger(knex, TableName.KmsKeyVersion);
await knex.schema.dropTableIfExists(TableName.KmsKey);
await dropOnUpdateTrigger(knex, TableName.KmsKey);
}

View File

@@ -30,6 +30,9 @@ export * from "./identity-universal-auths";
export * from "./incident-contacts";
export * from "./integration-auths";
export * from "./integrations";
export * from "./kms-key-versions";
export * from "./kms-keys";
export * from "./kms-root-config";
export * from "./ldap-configs";
export * from "./ldap-group-maps";
export * from "./models";
@@ -57,6 +60,7 @@ export * from "./secret-blind-indexes";
export * from "./secret-folder-versions";
export * from "./secret-folders";
export * from "./secret-imports";
export * from "./secret-references";
export * from "./secret-rotation-outputs";
export * from "./secret-rotations";
export * from "./secret-scanning-git-risks";

View File

@@ -0,0 +1,21 @@
// Code generated by automation script, DO NOT EDIT.
// Automated by pulling database and generating zod schema
// To update. Just run npm run generate:schema
// Written by akhilmhdh.
import { z } from "zod";
import { zodBuffer } from "@app/lib/zod";
import { TImmutableDBKeys } from "./models";
export const KmsKeyVersionsSchema = z.object({
id: z.string().uuid(),
encryptedKey: zodBuffer,
version: z.number(),
kmsKeyId: z.string().uuid()
});
export type TKmsKeyVersions = z.infer<typeof KmsKeyVersionsSchema>;
export type TKmsKeyVersionsInsert = Omit<z.input<typeof KmsKeyVersionsSchema>, TImmutableDBKeys>;
export type TKmsKeyVersionsUpdate = Partial<Omit<z.input<typeof KmsKeyVersionsSchema>, TImmutableDBKeys>>;

View File

@@ -0,0 +1,23 @@
// Code generated by automation script, DO NOT EDIT.
// Automated by pulling database and generating zod schema
// To update. Just run npm run generate:schema
// Written by akhilmhdh.
import { z } from "zod";
import { zodBuffer } from "@app/lib/zod";
import { TImmutableDBKeys } from "./models";
export const KmsKeysSchema = z.object({
id: z.string().uuid(),
encryptedKey: zodBuffer,
encryptionAlgorithm: z.string(),
version: z.number().default(1),
description: z.string().nullable().optional(),
isDisabled: z.boolean().default(false).nullable().optional()
});
export type TKmsKeys = z.infer<typeof KmsKeysSchema>;
export type TKmsKeysInsert = Omit<z.input<typeof KmsKeysSchema>, TImmutableDBKeys>;
export type TKmsKeysUpdate = Partial<Omit<z.input<typeof KmsKeysSchema>, TImmutableDBKeys>>;

View File

@@ -0,0 +1,19 @@
// Code generated by automation script, DO NOT EDIT.
// Automated by pulling database and generating zod schema
// To update. Just run npm run generate:schema
// Written by akhilmhdh.
import { z } from "zod";
import { zodBuffer } from "@app/lib/zod";
import { TImmutableDBKeys } from "./models";
export const KmsRootConfigSchema = z.object({
id: z.string().uuid(),
encryptedRootKey: zodBuffer
});
export type TKmsRootConfig = z.infer<typeof KmsRootConfigSchema>;
export type TKmsRootConfigInsert = Omit<z.input<typeof KmsRootConfigSchema>, TImmutableDBKeys>;
export type TKmsRootConfigUpdate = Partial<Omit<z.input<typeof KmsRootConfigSchema>, TImmutableDBKeys>>;

View File

@@ -81,7 +81,11 @@ export enum TableName {
DynamicSecretLease = "dynamic_secret_leases",
// junction tables with tags
JnSecretTag = "secret_tag_junction",
SecretVersionTag = "secret_version_tag_junction"
SecretVersionTag = "secret_version_tag_junction",
// KMS Service
KmsServerRootConfig = "kms_root_config",
KmsKey = "kms_keys",
KmsKeyVersion = "kms_key_versions"
}
export type TImmutableDBKeys = "id" | "createdAt" | "updatedAt";

View File

@@ -9,6 +9,15 @@ export enum KeyStorePrefixes {
SecretReplication = "secret-replication-import-lock"
}
type TWaitTillReady = {
key: string;
waitingCb?: () => void;
keyCheckCb: (val: string | null) => boolean;
waitIteration?: number;
delay?: number;
jitter?: number;
};
export const keyStoreFactory = (redisUrl: string) => {
const redis = new Redis(redisUrl);
const redisLock = new Redlock([redis], { retryCount: 2, retryDelay: 200 });
@@ -29,6 +38,29 @@ export const keyStoreFactory = (redisUrl: string) => {
const incrementBy = async (key: string, value: number) => redis.incrby(key, value);
const waitTillReady = async ({
key,
waitingCb,
keyCheckCb,
waitIteration = 10,
delay = 1000,
jitter = 200
}: TWaitTillReady) => {
let attempts = 0;
let isReady = keyCheckCb(await getItem(key));
while (!isReady) {
if (attempts > waitIteration) return;
// eslint-disable-next-line
await new Promise((resolve) => {
waitingCb?.();
setTimeout(resolve, Math.max(0, delay + Math.floor((Math.random() * 2 - 1) * jitter)));
});
attempts += 1;
// eslint-disable-next-line
isReady = keyCheckCb(await getItem(key, "wait_till_ready"));
}
};
return {
setItem,
getItem,
@@ -37,6 +69,7 @@ export const keyStoreFactory = (redisUrl: string) => {
incrementBy,
acquireLock(resources: string[], duration: number, settings?: Partial<Settings>) {
return redisLock.acquire(resources, duration, settings);
}
},
waitTillReady
};
};

View File

@@ -0,0 +1,49 @@
import crypto from "crypto";
import { SymmetricEncryption, TSymmetricEncryptionFns } from "./types";
const getIvLength = () => {
return 12;
};
const getTagLength = () => {
return 16;
};
export const symmetricCipherService = (type: SymmetricEncryption): TSymmetricEncryptionFns => {
const IV_LENGTH = getIvLength();
const TAG_LENGTH = getTagLength();
const encrypt = (text: Buffer, key: Buffer) => {
const iv = crypto.randomBytes(IV_LENGTH);
const cipher = crypto.createCipheriv(type, key, iv);
let encrypted = cipher.update(text);
encrypted = Buffer.concat([encrypted, cipher.final()]);
// Get the authentication tag
const tag = cipher.getAuthTag();
// Concatenate IV, encrypted text, and tag into a single buffer
const ciphertextBlob = Buffer.concat([iv, encrypted, tag]);
return ciphertextBlob;
};
const decrypt = (ciphertextBlob: Buffer, key: Buffer) => {
// Extract the IV, encrypted text, and tag from the buffer
const iv = ciphertextBlob.subarray(0, IV_LENGTH);
const tag = ciphertextBlob.subarray(-TAG_LENGTH);
const encrypted = ciphertextBlob.subarray(IV_LENGTH, -TAG_LENGTH);
const decipher = crypto.createDecipheriv(type, key, iv);
decipher.setAuthTag(tag);
const decrypted = Buffer.concat([decipher.update(encrypted), decipher.final()]);
return decrypted;
};
return {
encrypt,
decrypt
};
};

View File

@@ -0,0 +1,2 @@
export { symmetricCipherService } from "./cipher";
export { SymmetricEncryption } from "./types";

View File

@@ -0,0 +1,9 @@
export enum SymmetricEncryption {
AES_GCM_256 = "aes-256-gcm",
AES_GCM_128 = "aes-128-gcm"
}
export type TSymmetricEncryptionFns = {
encrypt: (text: Buffer, key: Buffer) => Buffer;
decrypt: (blob: Buffer, key: Buffer) => Buffer;
};

View File

@@ -11,6 +11,8 @@ import { getConfig } from "../config/env";
export const decodeBase64 = (s: string) => naclUtils.decodeBase64(s);
export const encodeBase64 = (u: Uint8Array) => naclUtils.encodeBase64(u);
export const randomSecureBytes = (length = 32) => crypto.randomBytes(length);
export type TDecryptSymmetricInput = {
ciphertext: string;
iv: string;

View File

@@ -9,7 +9,8 @@ export {
encryptAsymmetric,
encryptSymmetric,
encryptSymmetric128BitHexKeyUTF8,
generateAsymmetricKeyPair
generateAsymmetricKeyPair,
randomSecureBytes
} from "./encryption";
export {
decryptIntegrationAuths,

View File

@@ -7,3 +7,7 @@ export const zpStr = <T extends ZodTypeAny>(schema: T, opt: { stripNull: boolean
if (typeof val !== "string") return val;
return val.trim() || undefined;
}, schema);
export const zodBuffer = z.custom<Buffer>((data) => Buffer.isBuffer(data) || data instanceof Uint8Array, {
message: "Expected binary data (Buffer Or Uint8Array)"
});

View File

@@ -97,6 +97,9 @@ import { integrationDALFactory } from "@app/services/integration/integration-dal
import { integrationServiceFactory } from "@app/services/integration/integration-service";
import { integrationAuthDALFactory } from "@app/services/integration-auth/integration-auth-dal";
import { integrationAuthServiceFactory } from "@app/services/integration-auth/integration-auth-service";
import { kmsDALFactory } from "@app/services/kms/kms-dal";
import { kmsRootConfigDALFactory } from "@app/services/kms/kms-root-config-dal";
import { kmsServiceFactory } from "@app/services/kms/kms-service";
import { incidentContactDALFactory } from "@app/services/org/incident-contacts-dal";
import { orgBotDALFactory } from "@app/services/org/org-bot-dal";
import { orgDALFactory } from "@app/services/org/org-dal";
@@ -261,6 +264,9 @@ export const registerRoutes = async (
const dynamicSecretDAL = dynamicSecretDALFactory(db);
const dynamicSecretLeaseDAL = dynamicSecretLeaseDALFactory(db);
const kmsDAL = kmsDALFactory(db);
const kmsRootConfigDAL = kmsRootConfigDALFactory(db);
const permissionService = permissionServiceFactory({
permissionDAL,
orgRoleDAL,
@@ -269,6 +275,12 @@ export const registerRoutes = async (
projectDAL
});
const licenseService = licenseServiceFactory({ permissionService, orgDAL, licenseDAL, keyStore });
const kmsService = kmsServiceFactory({
kmsRootConfigDAL,
keyStore,
kmsDAL
});
const trustedIpService = trustedIpServiceFactory({
licenseService,
projectDAL,
@@ -823,6 +835,7 @@ export const registerRoutes = async (
await telemetryQueue.startTelemetryCheck();
await dailyResourceCleanUp.startCleanUp();
await kmsService.startService();
// inject all services
server.decorate<FastifyZodProvider["services"]>("services", {

View File

@@ -0,0 +1,10 @@
import { TDbClient } from "@app/db";
import { TableName } from "@app/db/schemas";
import { ormify } from "@app/lib/knex";
export type TKmsDALFactory = ReturnType<typeof kmsDALFactory>;
export const kmsDALFactory = (db: TDbClient) => {
const kmsOrm = ormify(db, TableName.KmsKey);
return kmsOrm;
};

View File

@@ -0,0 +1,10 @@
import { TDbClient } from "@app/db";
import { TableName } from "@app/db/schemas";
import { ormify } from "@app/lib/knex";
export type TKmsRootConfigDALFactory = ReturnType<typeof kmsRootConfigDALFactory>;
export const kmsRootConfigDALFactory = (db: TDbClient) => {
const kmsOrm = ormify(db, TableName.KmsServerRootConfig);
return kmsOrm;
};

View File

@@ -0,0 +1,118 @@
import { TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { randomSecureBytes } from "@app/lib/crypto";
import { symmetricCipherService, SymmetricEncryption } from "@app/lib/crypto/cipher";
import { BadRequestError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
import { TKmsDALFactory } from "./kms-dal";
import { TKmsRootConfigDALFactory } from "./kms-root-config-dal";
import { TDecryptWithKmsDTO, TEncryptWithKmsDTO } from "./kms-types";
type TKmsServiceFactoryDep = {
kmsDAL: TKmsDALFactory;
kmsRootConfigDAL: Pick<TKmsRootConfigDALFactory, "findById" | "create">;
keyStore: Pick<TKeyStoreFactory, "acquireLock" | "waitTillReady" | "setItemWithExpiry">;
};
export type TKmsServiceFactory = ReturnType<typeof kmsServiceFactory>;
const KMS_ROOT_CONFIG_UUID = "00000000-0000-0000-0000-000000000000";
const KMS_ROOT_CREATION_WAIT_KEY = "wait_till_ready_kms_root_key";
const KMS_ROOT_CREATION_WAIT_TIME = 10;
// akhilmhdh: Don't edit this value. This is measured for blob concatination in kms
const KMS_VERSION = "v01";
const KMS_VERSION_BLOB_LENGTH = 3;
export const kmsServiceFactory = ({ kmsDAL, kmsRootConfigDAL, keyStore }: TKmsServiceFactoryDep) => {
let ROOT_ENCRYPTION_KEY = Buffer.alloc(0);
// this is used symmetric encryption
const generateKmsKey = async () => {
const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
const kmsKeyMaterial = randomSecureBytes(32);
const encryptedKeyMaterial = cipher.encrypt(kmsKeyMaterial, ROOT_ENCRYPTION_KEY);
const { encryptedKey, ...doc } = await kmsDAL.create({
version: 1,
encryptedKey: encryptedKeyMaterial,
encryptionAlgorithm: SymmetricEncryption.AES_GCM_256
});
return doc;
};
const encrypt = async ({ kmsId, plainText }: TEncryptWithKmsDTO) => {
const kmsDoc = await kmsDAL.findById(kmsId);
if (!kmsDoc) throw new BadRequestError({ message: "KMS ID not found" });
// akhilmhdh: as more encryption are added do a check here on kmsDoc.encryptionAlgorithm
const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
const kmsKey = cipher.decrypt(kmsDoc.encryptedKey, ROOT_ENCRYPTION_KEY);
const encryptedPlainTextBlob = cipher.encrypt(Buffer.from(plainText, "utf8"), kmsKey);
// Buffer#1 encrypted text + Buffer#2 version number
const versionBlob = Buffer.from(KMS_VERSION, "utf8"); // length is 3
const cipherTextBlob = Buffer.concat([encryptedPlainTextBlob, versionBlob]);
return { cipherTextBlob };
};
const decrypt = async ({ cipherTextBlob: versionedCipherTextBlob, kmsId }: TDecryptWithKmsDTO) => {
const kmsDoc = await kmsDAL.findById(kmsId);
if (!kmsDoc) throw new BadRequestError({ message: "KMS ID not found" });
// akhilmhdh: as more encryption are added do a check here on kmsDoc.encryptionAlgorithm
const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
const kmsKey = cipher.decrypt(kmsDoc.encryptedKey, ROOT_ENCRYPTION_KEY);
const cipherTextBlob = versionedCipherTextBlob.subarray(0, -KMS_VERSION_BLOB_LENGTH);
const decryptedKmsDataString = cipher.decrypt(cipherTextBlob, kmsKey);
return decryptedKmsDataString;
};
const startService = async () => {
const appCfg = getConfig();
// This will switch to a seal process and HMS flow in future
const encryptionKey = appCfg.ENCRYPTION_KEY || appCfg.ROOT_ENCRYPTION_KEY;
if (!encryptionKey) throw new Error("Root encryption key not found for KMS service.");
const lock = await keyStore.acquireLock([`KMS_ROOT_CFG_LOCK`], 3000, { retryCount: 3 }).catch(() => null);
if (!lock) {
await keyStore.waitTillReady({
key: KMS_ROOT_CREATION_WAIT_KEY,
keyCheckCb: (val) => val === "true",
waitingCb: () => logger.info("KMS. Waiting for leader to finish creation of KMS Root Key")
});
}
// check if KMS root key was already generated and saved in DB
const kmsRootConfig = await kmsRootConfigDAL.findById(KMS_ROOT_CONFIG_UUID);
const cipher = symmetricCipherService(SymmetricEncryption.AES_GCM_256);
if (kmsRootConfig) {
if (lock) await lock.release();
logger.info("KMS: Encrypted ROOT Key found from DB. Decrypting.");
const decryptedRootKey = cipher.decrypt(kmsRootConfig.encryptedRootKey, Buffer.from(encryptionKey, "utf8"));
await keyStore.setItemWithExpiry(KMS_ROOT_CREATION_WAIT_KEY, KMS_ROOT_CREATION_WAIT_TIME, "true");
logger.info("KMS: Loading ROOT Key into Memory.");
ROOT_ENCRYPTION_KEY = decryptedRootKey;
return;
}
logger.info("KMS: Generating ROOT Key");
const newRootKey = randomSecureBytes(32);
const encryptedRootKey = cipher.encrypt(newRootKey, Buffer.from(encryptionKey, "utf8"));
// @ts-expect-error id is kept as fixed for idempotence and to avoid race condition
await kmsRootConfigDAL.create({ encryptedRootKey, id: KMS_ROOT_CONFIG_UUID });
await keyStore.setItemWithExpiry(KMS_ROOT_CREATION_WAIT_KEY, KMS_ROOT_CREATION_WAIT_TIME, "true");
logger.info("KMS: Saved and loaded ROOT Key into memory");
if (lock) await lock.release();
ROOT_ENCRYPTION_KEY = newRootKey;
};
return {
startService,
generateKmsKey,
encrypt,
decrypt
};
};

View File

@@ -0,0 +1,10 @@
export type TEncryptWithKmsDTO = {
kmsId: string;
// utf8 encoded
plainText: string;
};
export type TDecryptWithKmsDTO = {
kmsId: string;
cipherTextBlob: Buffer;
};