Checkpoint finish preliminary support for ROOT_ENCRYPTION_KEY

This commit is contained in:
Tuan Dang
2023-05-06 22:07:59 +03:00
parent 3d3d7c9821
commit 87e047a152
54 changed files with 1876 additions and 1568 deletions
+1 -72
View File
@@ -7,19 +7,14 @@ import {
IEncryptAsymmetricOutput,
IDecryptAsymmetricInput,
IEncryptSymmetricInput,
IEncryptSymmetricOutput,
IDecryptSymmetricInput
} from '../../interfaces/utils';
import {
BadRequestError,
InternalServerError
} from '../errors';
import { BadRequestError } from '../errors';
import {
ALGORITHM_AES_256_GCM,
NONCE_BYTES_SIZE,
BLOCK_SIZE_BYTES_16
} from '../../variables';
import { validateEncryptionKey } from '../../validation';
/**
* Return new base64, NaCl, public-private key pair.
@@ -96,70 +91,6 @@ const decryptAsymmetric = ({
return util.encodeUTF8(plaintext);
};
/**
* Return symmetrically encrypted [plaintext] using [key].
* @param {Object} obj
* @param {String} obj.plaintext - (utf8) plaintext to encrypt
* @param {String} obj.key - (base64) 256-bit key
* @returns {Object} obj
* @returns {String} obj.ciphertext (base64) ciphertext
* @returns {String} obj.iv (base64) iv
* @returns {String} obj.tag (base64) tag
*/
const encryptSymmetric = ({
plaintext,
key
}: IEncryptSymmetricInput): IEncryptSymmetricOutput => {
validateEncryptionKey(key);
const iv = crypto.randomBytes(NONCE_BYTES_SIZE);
const secretKey = crypto.createSecretKey(key, 'base64');
const cipher = crypto.createCipheriv(ALGORITHM_AES_256_GCM, secretKey, iv);
let ciphertext = cipher.update(plaintext, 'utf8', 'base64');
ciphertext += cipher.final('base64');
return {
ciphertext,
iv: iv.toString('base64'),
tag: cipher.getAuthTag().toString('base64')
};
};
/**
* Return symmetrically decrypted [ciphertext] using [iv], [tag],
* and [key].
* @param {Object} obj
* @param {String} obj.ciphertext - ciphertext to decrypt
* @param {String} obj.iv - (base64) 256-bit iv
* @param {String} obj.tag - (base64) tag
* @param {String} obj.key - (base64) 256-bit key
* @returns {String} cleartext - the deciphered ciphertext
*/
const decryptSymmetric = ({
ciphertext,
iv,
tag,
key
}: IDecryptSymmetricInput): string => {
validateEncryptionKey(key);
const secretKey = crypto.createSecretKey(key, 'base64');
const decipher = crypto.createDecipheriv(
ALGORITHM_AES_256_GCM,
secretKey,
Buffer.from(iv, 'base64')
);
decipher.setAuthTag(Buffer.from(tag, 'base64'));
let cleartext = decipher.update(ciphertext, 'base64', 'utf8');
cleartext += decipher.final('utf8');
return cleartext;
};
/**
* Return symmetrically encrypted [plaintext] using [key].
*
@@ -230,8 +161,6 @@ export {
generateKeyPair,
encryptAsymmetric,
decryptAsymmetric,
encryptSymmetric,
decryptSymmetric,
encryptSymmetric128BitHexKeyUTF8,
decryptSymmetric128BitHexKeyUTF8
};
-215
View File
@@ -1,215 +0,0 @@
import crypto from 'crypto';
import { encryptSymmetric128BitHexKeyUTF8 } from '../crypto';
import { EESecretService } from '../../ee/services';
import { SecretVersion } from '../../ee/models';
import {
Secret,
ISecret,
SecretBlindIndexData,
Workspace,
Bot,
BackupPrivateKey,
IntegrationAuth
} from '../../models';
import { getEncryptionKey, getRootEncryptionKey } from '../../config';
import {
ALGORITHM_AES_256_GCM,
ENCODING_SCHEME_UTF8
} from '../../variables';
/**
*
*/
export const backfillSecretVersions = async () => {
await Secret.updateMany(
{ version: { $exists: false } },
{ $set: { version: 1 } }
);
const unversionedSecrets: ISecret[] = await Secret.aggregate([
{
$lookup: {
from: "secretversions",
localField: "_id",
foreignField: "secret",
as: "versions",
},
},
{
$match: {
versions: { $size: 0 },
},
},
]);
if (unversionedSecrets.length > 0) {
await EESecretService.addSecretVersions({
secretVersions: unversionedSecrets.map(
(s, idx) =>
new SecretVersion({
...s,
secret: s._id,
version: s.version ? s.version : 1,
isDeleted: false,
workspace: s.workspace,
environment: s.environment,
})
),
});
}
}
export const backfillSecretBlindIndexData = async () => {
const workspaceIdsBlindIndexed = await SecretBlindIndexData.distinct('workspace');
const workspaceIdsToBlindIndex = await Workspace.distinct('_id', {
_id: {
$nin: workspaceIdsBlindIndexed
}
});
const secretBlindIndexDataToInsert = await Promise.all(
workspaceIdsToBlindIndex.map(async (workspaceToBlindIndex) => {
const salt = crypto.randomBytes(16).toString('base64');
const {
ciphertext: encryptedSaltCiphertext,
iv: saltIV,
tag: saltTag
} = encryptSymmetric128BitHexKeyUTF8({
plaintext: salt,
key: await getEncryptionKey()
});
const secretBlindIndexData = new SecretBlindIndexData({
workspace: workspaceToBlindIndex,
encryptedSaltCiphertext,
saltIV,
saltTag
})
return secretBlindIndexData;
})
);
if (secretBlindIndexDataToInsert.length > 0) {
await SecretBlindIndexData.insertMany(secretBlindIndexDataToInsert);
}
}
export const backfillEncryptionMetadata = async () => {
// backfill bot encryption metadata
await Bot.updateMany(
{
algorithm: {
$exists: false
},
keySize: {
$exists: false
},
keyEncoding: {
$exists: false
}
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}
}
);
// backfill secret blind index encryption metadata
await SecretBlindIndexData.updateMany(
{
algorithm: {
$exists: false
},
keySize: {
$exists: false
},
keyEncoding: {
$exists: false
}
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}
}
);
// backfill backup private key encryption metadata
await BackupPrivateKey.updateMany(
{
algorithm: {
$exists: false
},
keySize: {
$exists: false
},
keyEncoding: {
$exists: false
}
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}
}
);
// backfill integration auth encryption metadata
await IntegrationAuth.updateMany(
{
algorithm: {
$exists: false
},
keySize: {
$exists: false
},
keyEncoding: {
$exists: false
}
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}
}
);
// TODO: blind indices
// TODO: secret versions and snapshots etc.
// TODO: re-encrypt keys logic
// TODO: how do you handle different parts of the software
// encrypting under different schemes?
// const encryptionKey = await getEncryptionKey();
// const rootEncryptionKey = await getRootEncryptionKey();
// console.log('rootEncryptionKey: ', rootEncryptionKey);
// if (encryptionKey && rootEncryptionKey) {
// // case: both the old encryption key and new encryption key are present
// // -> perform migration if needed
// console.log('rootEncryptionKey is defined');
// const bots = await Bot.find({
// algorithm: ALGORITHM_AES_256_GCM,
// keySize: 256,
// keyEncoding: ENCODING_SCHEME_UTF8
// }, 'encryptedPrivateKey iv tag');
// if (bots.length > 0) {
// // TODO: unencrypt and re-encrypt
// // TODO: unencrypt and re-encrypt blind-indices
// // probably then need to move this function
// console.log('bots: ', bots);
// }
// }
}
+324
View File
@@ -0,0 +1,324 @@
import crypto from 'crypto';
import { encryptSymmetric128BitHexKeyUTF8 } from '../crypto';
import { EESecretService } from '../../ee/services';
import { SecretVersion } from '../../ee/models';
import {
Secret,
ISecret,
SecretBlindIndexData,
Workspace,
Bot,
BackupPrivateKey,
IntegrationAuth,
} from '../../models';
import {
generateKeyPair
} from '../../utils/crypto';
import {
client,
getEncryptionKey,
getRootEncryptionKey
} from '../../config';
import {
ALGORITHM_AES_256_GCM,
ENCODING_SCHEME_UTF8,
ENCODING_SCHEME_BASE64
} from '../../variables';
import { InternalServerError } from '../errors';
/**
* Backfill secrets to ensure that they're all versioned and have
* corresponding secret versions
*/
export const backfillSecretVersions = async () => {
await Secret.updateMany(
{ version: { $exists: false } },
{ $set: { version: 1 } }
);
const unversionedSecrets: ISecret[] = await Secret.aggregate([
{
$lookup: {
from: "secretversions",
localField: "_id",
foreignField: "secret",
as: "versions",
},
},
{
$match: {
versions: { $size: 0 },
},
},
]);
if (unversionedSecrets.length > 0) {
await EESecretService.addSecretVersions({
secretVersions: unversionedSecrets.map(
(s, idx) =>
new SecretVersion({
...s,
secret: s._id,
version: s.version ? s.version : 1,
isDeleted: false,
workspace: s.workspace,
environment: s.environment,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
})
),
});
}
}
/**
* Backfill workspace bots to ensure that every workspace has a bot
*/
export const backfillBots = async () => {
const encryptionKey = await getEncryptionKey();
const rootEncryptionKey = await getRootEncryptionKey();
const workspaceIdsWithBot = await Bot.distinct('workspace');
const workspaceIdsToAddBot = await Workspace.distinct('_id', {
_id: {
$nin: workspaceIdsWithBot
}
});
if (workspaceIdsToAddBot.length === 0) return;
const botsToInsert = await Promise.all(
workspaceIdsToAddBot.map(async (workspaceToAddBot) => {
const { publicKey, privateKey } = generateKeyPair();
if (rootEncryptionKey) {
const {
ciphertext: encryptedPrivateKey,
iv,
tag
} = client.encryptSymmetric(privateKey, rootEncryptionKey);
return new Bot({
name: 'Infisical Bot',
workspace: workspaceToAddBot,
isActive: false,
publicKey,
encryptedPrivateKey,
iv,
tag,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_BASE64
});
} else if (encryptionKey) {
const {
ciphertext: encryptedPrivateKey,
iv,
tag
} = encryptSymmetric128BitHexKeyUTF8({
plaintext: privateKey,
key: encryptionKey
});
return new Bot({
name: 'Infisical Bot',
workspace: workspaceToAddBot,
isActive: false,
publicKey,
encryptedPrivateKey,
iv,
tag,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
});
}
throw InternalServerError({
message: 'Failed to backfill workspace bots due to missing encryption key'
});
})
);
await Bot.insertMany(botsToInsert);
}
/**
* Backfill secret blind index data to ensure that every workspace
* has a secret blind index data
*/
export const backfillSecretBlindIndexData = async () => {
const encryptionKey = await getEncryptionKey();
const rootEncryptionKey = await getRootEncryptionKey();
const workspaceIdsBlindIndexed = await SecretBlindIndexData.distinct('workspace');
const workspaceIdsToBlindIndex = await Workspace.distinct('_id', {
_id: {
$nin: workspaceIdsBlindIndexed
}
});
if (workspaceIdsToBlindIndex.length === 0) return;
const secretBlindIndexDataToInsert = await Promise.all(
workspaceIdsToBlindIndex.map(async (workspaceToBlindIndex) => {
const salt = crypto.randomBytes(16).toString('base64');
if (rootEncryptionKey) {
const {
ciphertext: encryptedSaltCiphertext,
iv: saltIV,
tag: saltTag
} = client.encryptSymmetric(salt, rootEncryptionKey)
return new SecretBlindIndexData({
workspace: workspaceToBlindIndex,
encryptedSaltCiphertext,
saltIV,
saltTag,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_BASE64
});
} else if (encryptionKey) {
const {
ciphertext: encryptedSaltCiphertext,
iv: saltIV,
tag: saltTag
} = encryptSymmetric128BitHexKeyUTF8({
plaintext: salt,
key: encryptionKey
});
return new SecretBlindIndexData({
workspace: workspaceToBlindIndex,
encryptedSaltCiphertext,
saltIV,
saltTag,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
});
}
throw InternalServerError({
message: 'Failed to backfill secret blind index data due to missing encryption key'
});
})
);
SecretBlindIndexData.insertMany(secretBlindIndexDataToInsert);
}
/**
* Backfill Secret, SecretVersion, SecretBlindIndexData, Bot,
* BackupPrivateKey, IntegrationAuth collections to ensure that
* they all have encryption metadata documented
*/
export const backfillEncryptionMetadata = async () => {
// backfill secret encryption metadata
await Secret.updateMany(
{
algorithm: {
$exists: false
},
keyEncoding: {
$exists: false
}
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}
}
);
// backfill secret version encryption metadata
await SecretVersion.updateMany(
{
algorithm: {
$exists: false
},
keyEncoding: {
$exists: false
}
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}
}
);
// backfill secret blind index encryption metadata
await SecretBlindIndexData.updateMany(
{
algorithm: {
$exists: false
},
keyEncoding: {
$exists: false
}
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}
}
);
// backfill bot encryption metadata
await Bot.updateMany(
{
algorithm: {
$exists: false
},
keyEncoding: {
$exists: false
}
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}
}
);
// backfill backup private key encryption metadata
await BackupPrivateKey.updateMany(
{
algorithm: {
$exists: false
},
keyEncoding: {
$exists: false
}
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}
}
);
// backfill integration auth encryption metadata
await IntegrationAuth.updateMany(
{
algorithm: {
$exists: false
},
keyEncoding: {
$exists: false
}
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}
}
);
}
+34 -12
View File
@@ -1,15 +1,23 @@
import * as Sentry from '@sentry/node';
import { DatabaseService } from '../../services';
import { DatabaseService, TelemetryService } from '../../services';
import { setTransporter } from '../../helpers/nodemailer';
import { initSmtp } from '../../services/smtp';
import { createTestUserForDevelopment } from '../addDevelopmentUser'
// eslint-disable-next-line @typescript-eslint/no-var-requires
const { patchRouterParam } = require('../patchAsyncRoutes');
import {
validateEncryptionKeysConfig
} from './validateConfig';
import {
backfillSecretVersions,
backfillBots,
backfillSecretBlindIndexData,
backfillEncryptionMetadata
} from './backfill';
} from './backfillData';
import {
reencryptBotPrivateKeys,
reencryptSecretBlindIndexDataSalts
} from './reencryptData';
import {
getNodeEnv,
getMongoURL,
@@ -18,34 +26,48 @@ import {
/**
* Prepare Infisical upon startup. This includes tasks like:
* - Log initial telemetry message
* - Initializing SMTP configuration
* - Initializing the database connection
* - Initializing Sentry
* - Backfilling data
* - Re-encrypting data
*/
export const setup = async () => {
patchRouterParam();
await validateEncryptionKeysConfig();
await TelemetryService.logTelemetryMessage();
// initializing SMTP configuration
setTransporter(await initSmtp());
// initializing the database connection
await DatabaseService.initDatabase(await getMongoURL());
/**
* NOTE: the order in this setup function is critical.
* It is important to backfill data before performing any re-encryption functionality.
*/
// backfilling data
// backfilling data to catch up with new collections and updated fields
await backfillSecretVersions();
await backfillBots();
await backfillSecretBlindIndexData();
await backfillEncryptionMetadata();
// re-encrypt any data previously encrypted under server hex 128-bit ENCRYPTION_KEY
// to base64 256-bit ROOT_ENCRYPTION_KEY
await reencryptBotPrivateKeys();
await reencryptSecretBlindIndexDataSalts();
// initializing Sentry
if ((await getNodeEnv()) !== 'development') {
Sentry.init({
dsn: await getSentryDSN(),
tracesSampleRate: 1.0,
debug: await getNodeEnv() === 'production' ? false : true,
environment: await getNodeEnv()
});
}
Sentry.init({
dsn: await getSentryDSN(),
tracesSampleRate: 1.0,
debug: (await getNodeEnv()) === 'production' ? false : true,
environment: (await getNodeEnv())
});
patchRouterParam();
await createTestUserForDevelopment();
}
+126
View File
@@ -0,0 +1,126 @@
import {
Bot,
IBot,
ISecretBlindIndexData,
SecretBlindIndexData
} from '../../models';
import { decryptSymmetric128BitHexKeyUTF8 } from '../../utils/crypto';
import {
client,
getEncryptionKey,
getRootEncryptionKey
} from '../../config';
import {
ALGORITHM_AES_256_GCM,
ENCODING_SCHEME_UTF8,
ENCODING_SCHEME_BASE64
} from '../../variables';
/**
* Re-encrypt bot private keys from hex 128-bit ENCRYPTION_KEY
* to base64 256-bit ROOT_ENCRYPTION_KEY
*/
export const reencryptBotPrivateKeys = async () => {
const encryptionKey = await getEncryptionKey();
const rootEncryptionKey = await getRootEncryptionKey();
if (encryptionKey && rootEncryptionKey) {
// 1: re-encrypt bot private keys under ROOT_ENCRYPTION_KEY
const bots = await Bot.find({
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}).select('+encryptedPrivateKey iv tag algorithm keyEncoding');
if (bots.length === 0) return;
const operationsBot = await Promise.all(
bots.map(async (bot: IBot) => {
const privateKey = decryptSymmetric128BitHexKeyUTF8({
ciphertext: bot.encryptedPrivateKey,
iv: bot.iv,
tag: bot.tag,
key: encryptionKey
});
const {
ciphertext: encryptedPrivateKey,
iv,
tag
} = client.encryptSymmetric(privateKey, rootEncryptionKey);
return ({
updateOne: {
filter: {
_id: bot._id
},
update: {
encryptedPrivateKey,
iv,
tag,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_BASE64
}
}
})
})
);
await Bot.bulkWrite(operationsBot);
}
}
/**
* Re-encrypt secret blind index data salts from hex 128-bit ENCRYPTION_KEY
* to base64 256-bit ROOT_ENCRYPTION_KEY
*/
export const reencryptSecretBlindIndexDataSalts = async () => {
const encryptionKey = await getEncryptionKey();
const rootEncryptionKey = await getRootEncryptionKey();
// 2. re-encrypt secret blind index salts under ROOT_ENCRYPTION_KEY
if (encryptionKey && rootEncryptionKey) {
const secretBlindIndexData = await SecretBlindIndexData.find({
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_UTF8
}).select('+encryptedSaltCiphertext +saltIV +saltTag +algorithm +keyEncoding');
if (secretBlindIndexData.length == 0) return;
const operationsSecretBlindIndexData = await Promise.all(
secretBlindIndexData.map(async (secretBlindIndexDatum: ISecretBlindIndexData) => {
const salt = decryptSymmetric128BitHexKeyUTF8({
ciphertext: secretBlindIndexDatum.encryptedSaltCiphertext,
iv: secretBlindIndexDatum.saltIV,
tag: secretBlindIndexDatum.saltTag,
key: encryptionKey
});
const {
ciphertext: encryptedSaltCiphertext,
iv: saltIV,
tag: saltTag
} = client.encryptSymmetric(salt, rootEncryptionKey);
return ({
updateOne: {
filter: {
_id: secretBlindIndexDatum._id
},
update: {
encryptedSaltCiphertext,
saltIV,
saltTag,
algorithm: ALGORITHM_AES_256_GCM,
keyEncoding: ENCODING_SCHEME_BASE64
}
}
})
})
);
await SecretBlindIndexData.bulkWrite(operationsSecretBlindIndexData);
}
}
+69
View File
@@ -0,0 +1,69 @@
import {
getEncryptionKey,
getRootEncryptionKey
} from '../../config';
import {
InternalServerError
} from '../../utils/errors';
/**
* Validate ENCRYPTION_KEY and ROOT_ENCRYPTION_KEY. Specifically:
* - ENCRYPTION_KEY is a hex, 128-bit string
* - ROOT_ENCRYPTION_KEY is a base64, 128-bit string
* - Either ENCRYPTION_KEY or ROOT_ENCRYPTION_KEY are present
*
* - Encrypted data is consistent with the passed in encryption keys
*
* NOTE 1: ENCRYPTION_KEY is being transitioned to ROOT_ENCRYPTION_KEY
* NOTE 2: In the future, we will have a superior validation function
* built into the SDK.
*/
export const validateEncryptionKeysConfig = async () => {
const encryptionKey = await getEncryptionKey();
const rootEncryptionKey = await getRootEncryptionKey();
// TODO: handle case where either of keys can be empty strings
// and it would actually count as being defined for encryption
// within the application
// console.log('validateEncryptionKeysConfig');
// console.log('encryptionKey: ', encryptionKey);
// console.log('rootEncryptionKey: ', rootEncryptionKey);
if (
(encryptionKey === undefined || encryptionKey === "") &&
(rootEncryptionKey === undefined || rootEncryptionKey === "")
) throw InternalServerError({
message: "Failed to find required root encryption key environment variable. Please make sure that you're passing in a ROOT_ENCRYPTION_KEY environment variable."
});
if (encryptionKey && encryptionKey !== '') {
// validate [encryptionKey]
const keyBuffer = Buffer.from(encryptionKey, 'hex');
const decoded = keyBuffer.toString('hex');
if (decoded !== encryptionKey) throw InternalServerError({
message: 'Failed to validate that the encryption key is correctly encoded in hex.'
});
if (keyBuffer.length !== 16) throw InternalServerError({
message: 'Failed to validate that the encryption key is a 128-bit hex string.'
});
}
if (rootEncryptionKey && rootEncryptionKey !== '') {
// validate [rootEncryptionKey]
const keyBuffer = Buffer.from(rootEncryptionKey, 'base64')
const decoded = keyBuffer.toString('base64');
if (decoded !== rootEncryptionKey) throw InternalServerError({
message: 'Failed to validate that the root encryption key is correctly encoded in base64'
});
if (keyBuffer.length !== 32) throw InternalServerError({
message: 'Failed to validate that the encryption key is a 256-bit base64 string'
});
}
}