Begin refactoring backfilling and preparation operations into setup and start adding encryption metadata to models

This commit is contained in:
Tuan Dang
2023-05-03 14:21:42 +03:00
parent b462ca3e89
commit 099cee7f39
37 changed files with 797 additions and 619 deletions
-139
View File
@@ -1,139 +0,0 @@
import nacl from 'tweetnacl';
import util from 'tweetnacl-util';
import AesGCM from './aes-gcm';
/**
* Return new base64, NaCl, public-private key pair.
* @returns {Object} obj
* @returns {String} obj.publicKey - base64, NaCl, public key
* @returns {String} obj.privateKey - base64, NaCl, private key
*/
const generateKeyPair = () => {
const pair = nacl.box.keyPair();
return ({
publicKey: util.encodeBase64(pair.publicKey),
privateKey: util.encodeBase64(pair.secretKey)
});
}
/**
* Return assymmetrically encrypted [plaintext] using [publicKey] where
* [publicKey] likely belongs to the recipient.
* @param {Object} obj
* @param {String} obj.plaintext - plaintext to encrypt
* @param {String} obj.publicKey - public key of the recipient
* @param {String} obj.privateKey - private key of the sender (current user)
* @returns {Object} obj
* @returns {String} ciphertext - base64-encoded ciphertext
* @returns {String} nonce - base64-encoded nonce
*/
const encryptAsymmetric = ({
plaintext,
publicKey,
privateKey
}: {
plaintext: string;
publicKey: string;
privateKey: string;
}) => {
const nonce = nacl.randomBytes(24);
const ciphertext = nacl.box(
util.decodeUTF8(plaintext),
nonce,
util.decodeBase64(publicKey),
util.decodeBase64(privateKey)
);
return {
ciphertext: util.encodeBase64(ciphertext),
nonce: util.encodeBase64(nonce)
};
};
/**
* Return assymmetrically decrypted [ciphertext] using [privateKey] where
* [privateKey] likely belongs to the recipient.
* @param {Object} obj
* @param {String} obj.ciphertext - ciphertext to decrypt
* @param {String} obj.nonce - nonce
* @param {String} obj.publicKey - public key of the sender
* @param {String} obj.privateKey - private key of the receiver (current user)
* @param {String} plaintext - UTF8 plaintext
*/
const decryptAsymmetric = ({
ciphertext,
nonce,
publicKey,
privateKey
}: {
ciphertext: string;
nonce: string;
publicKey: string;
privateKey: string;
}): string => {
const plaintext: any = nacl.box.open(
util.decodeBase64(ciphertext),
util.decodeBase64(nonce),
util.decodeBase64(publicKey),
util.decodeBase64(privateKey)
);
return util.encodeUTF8(plaintext);
};
/**
* Return symmetrically encrypted [plaintext] using [key].
* @param {Object} obj
* @param {String} obj.plaintext - plaintext to encrypt
* @param {String} obj.key - hex key
*/
const encryptSymmetric = ({
plaintext,
key
}: {
plaintext: string;
key: string;
}) => {
const obj = AesGCM.encrypt(plaintext, key);
const { ciphertext, iv, tag } = obj;
return {
ciphertext,
iv,
tag
};
};
/**
* Return symmetrically decypted [ciphertext] using [iv], [tag],
* and [key].
* @param {Object} obj
* @param {String} obj.ciphertext - ciphertext to decrypt
* @param {String} obj.iv - iv
* @param {String} obj.tag - tag
* @param {String} obj.key - hex key
*
*/
const decryptSymmetric = ({
ciphertext,
iv,
tag,
key
}: {
ciphertext: string;
iv: string;
tag: string;
key: string;
}): string => {
const plaintext = AesGCM.decrypt(ciphertext, iv, tag, key);
return plaintext;
};
export {
generateKeyPair,
encryptAsymmetric,
decryptAsymmetric,
encryptSymmetric,
decryptSymmetric
};
+237
View File
@@ -0,0 +1,237 @@
import crypto from 'crypto';
import nacl from 'tweetnacl';
import util from 'tweetnacl-util';
import {
IGenerateKeyPairOutput,
IEncryptAsymmetricInput,
IEncryptAsymmetricOutput,
IDecryptAsymmetricInput,
IEncryptSymmetricInput,
IEncryptSymmetricOutput,
IDecryptSymmetricInput
} from '../../interfaces/utils';
import {
BadRequestError,
InternalServerError
} from '../errors';
import {
ALGORITHM_AES_256_GCM,
BLOCK_SIZE_BYTES_32,
BLOCK_SIZE_BYTES_16
} from '../../variables';
import { validateEncryptionKey } from '../../validation';
/**
* Return new base64, NaCl, public-private key pair.
* @returns {Object} obj
* @returns {String} obj.publicKey - (base64) NaCl, public key
* @returns {String} obj.privateKey - (base64), NaCl, private key
*/
const generateKeyPair = (): IGenerateKeyPairOutput => {
const pair = nacl.box.keyPair();
return ({
publicKey: util.encodeBase64(pair.publicKey),
privateKey: util.encodeBase64(pair.secretKey)
});
}
/**
* Return assymmetrically encrypted [plaintext] using [publicKey] where
* [publicKey] likely belongs to the recipient.
* @param {Object} obj
* @param {String} obj.plaintext - plaintext to encrypt
* @param {String} obj.publicKey - (base64) Nacl public key of the recipient
* @param {String} obj.privateKey - (base64) Nacl private key of the sender (current user)
* @returns {Object} obj
* @returns {String} obj.ciphertext - (base64) ciphertext
* @returns {String} obj.nonce - (base64) nonce
*/
const encryptAsymmetric = ({
plaintext,
publicKey,
privateKey
}: IEncryptAsymmetricInput): IEncryptAsymmetricOutput => {
const nonce = nacl.randomBytes(24);
const ciphertext = nacl.box(
util.decodeUTF8(plaintext),
nonce,
util.decodeBase64(publicKey),
util.decodeBase64(privateKey)
);
return {
ciphertext: util.encodeBase64(ciphertext),
nonce: util.encodeBase64(nonce)
};
};
/**
* Return assymmetrically decrypted [ciphertext] using [privateKey] where
* [privateKey] likely belongs to the recipient.
* @param {Object} obj
* @param {String} obj.ciphertext - ciphertext to decrypt
* @param {String} obj.nonce - (base64) nonce
* @param {String} obj.publicKey - (base64) public key of the sender
* @param {String} obj.privateKey - (base64) private key of the receiver (current user)
* @returns {String} plaintext - (utf8) plaintext
*/
const decryptAsymmetric = ({
ciphertext,
nonce,
publicKey,
privateKey
}: IDecryptAsymmetricInput): string => {
const plaintext: Uint8Array | null = nacl.box.open(
util.decodeBase64(ciphertext),
util.decodeBase64(nonce),
util.decodeBase64(publicKey),
util.decodeBase64(privateKey)
);
if (plaintext == null) throw BadRequestError({
message: 'Invalid ciphertext or keys'
});
return util.encodeUTF8(plaintext);
};
/**
* Return symmetrically encrypted [plaintext] using [key].
* @param {Object} obj
* @param {String} obj.plaintext - (utf8) plaintext to encrypt
* @param {String} obj.key - (base64) 256-bit key
* @returns {Object} obj
* @returns {String} obj.ciphertext (base64) ciphertext
* @returns {String} obj.iv (base64) iv
* @returns {String} obj.tag (base64) tag
*/
const encryptSymmetric = ({
plaintext,
key
}: IEncryptSymmetricInput): IEncryptSymmetricOutput => {
validateEncryptionKey(key);
const iv = crypto.randomBytes(BLOCK_SIZE_BYTES_32);
const secretKey = crypto.createSecretKey(key, 'base64');
const cipher = crypto.createCipheriv(ALGORITHM_AES_256_GCM, secretKey, iv);
let ciphertext = cipher.update(plaintext, 'utf8', 'base64');
ciphertext += cipher.final('base64');
return {
ciphertext,
iv: iv.toString('base64'),
tag: cipher.getAuthTag().toString('base64')
};
};
/**
* Return symmetrically decrypted [ciphertext] using [iv], [tag],
* and [key].
* @param {Object} obj
* @param {String} obj.ciphertext - ciphertext to decrypt
* @param {String} obj.iv - (base64) 256-bit iv
* @param {String} obj.tag - (base64) tag
* @param {String} obj.key - (base64) 256-bit key
* @returns {String} cleartext - the deciphered ciphertext
*/
const decryptSymmetric = ({
ciphertext,
iv,
tag,
key
}: IDecryptSymmetricInput): string => {
validateEncryptionKey(key);
const secretKey = crypto.createSecretKey(key, 'base64');
const decipher = crypto.createDecipheriv(
ALGORITHM_AES_256_GCM,
secretKey,
Buffer.from(iv, 'base64')
);
decipher.setAuthTag(Buffer.from(tag, 'base64'));
let cleartext = decipher.update(ciphertext, 'base64', 'utf8');
cleartext += decipher.final('utf8');
return cleartext;
};
/**
* Return symmetrically encrypted [plaintext] using [key].
*
* NOTE: THIS FUNCTION SHOULD NOT BE USED FOR ALL FUTURE
* ENCRYPTION OPERATIONS UNLESS IT TOUCHES OLD FUNCTIONALITY
* THAT USES IT. USE encryptSymmetric() instead
*
* @param {Object} obj
* @param {String} obj.plaintext - (utf8) plaintext to encrypt
* @param {String} obj.key - (base64) 256-bit key
* @returns {Object} obj
* @returns {String} obj.ciphertext (base64) ciphertext
* @returns {String} obj.iv (base64) iv
* @returns {String} obj.tag (base64) tag
*/
const encryptSymmetric128BitHexKeyUTF8 = ({
plaintext,
key
}: IEncryptSymmetricInput) => {
const iv = crypto.randomBytes(BLOCK_SIZE_BYTES_16);
const cipher = crypto.createCipheriv(ALGORITHM_AES_256_GCM, key, iv);
let ciphertext = cipher.update(plaintext, 'utf8', 'base64');
ciphertext += cipher.final('base64');
return {
ciphertext,
iv: iv.toString('base64'),
tag: cipher.getAuthTag().toString('base64')
};
}
/**
* Return symmetrically decrypted [ciphertext] using [iv], [tag],
* and [key].
*
* NOTE: THIS FUNCTION SHOULD NOT BE USED FOR ALL FUTURE
* DECRYPTION OPERATIONS UNLESS IT TOUCHES OLD FUNCTIONALITY
* THAT USES IT. USE decryptSymmetric() instead
*
* @param {Object} obj
* @param {String} obj.ciphertext - ciphertext to decrypt
* @param {String} obj.iv - (base64) 256-bit iv
* @param {String} obj.tag - (base64) tag
* @param {String} obj.key - (hex) 128-bit key
* @returns {String} cleartext - the deciphered ciphertext
*/
const decryptSymmetric128BitHexKeyUTF8 = ({
ciphertext,
iv,
tag,
key
}: IDecryptSymmetricInput) => {
const decipher = crypto.createDecipheriv(
ALGORITHM_AES_256_GCM,
key,
Buffer.from(iv, 'base64')
);
decipher.setAuthTag(Buffer.from(tag, 'base64'));
let cleartext = decipher.update(ciphertext, 'base64', 'utf8');
cleartext += decipher.final('utf8');
return cleartext;
}
export {
generateKeyPair,
encryptAsymmetric,
decryptAsymmetric,
encryptSymmetric,
decryptSymmetric,
encryptSymmetric128BitHexKeyUTF8,
decryptSymmetric128BitHexKeyUTF8
};
+210
View File
@@ -0,0 +1,210 @@
import crypto from 'crypto';
import { encryptSymmetric128BitHexKeyUTF8 } from '../crypto';
import { EESecretService } from '../../ee/services';
import { SecretVersion } from '../../ee/models';
import {
Secret,
ISecret,
SecretBlindIndexData,
Workspace,
Bot,
BackupPrivateKey,
IntegrationAuth
} from '../../models';
import { getEncryptionKey, getRootEncryptionKey } from '../../config';
import {
ALGORITHM_AES_256_GCM,
ENCODING_SCHEME_UTF8
} from '../../variables';
/**
*
*/
export const backfillSecretVersions = async () => {
await Secret.updateMany(
{ version: { $exists: false } },
{ $set: { version: 1 } }
);
const unversionedSecrets: ISecret[] = await Secret.aggregate([
{
$lookup: {
from: "secretversions",
localField: "_id",
foreignField: "secret",
as: "versions",
},
},
{
$match: {
versions: { $size: 0 },
},
},
]);
if (unversionedSecrets.length > 0) {
await EESecretService.addSecretVersions({
secretVersions: unversionedSecrets.map(
(s, idx) =>
new SecretVersion({
...s,
secret: s._id,
version: s.version ? s.version : 1,
isDeleted: false,
workspace: s.workspace,
environment: s.environment,
})
),
});
}
}
export const backfillSecretBlindIndexData = async () => {
const workspaceIdsBlindIndexed = await SecretBlindIndexData.distinct('workspace');
const workspaceIdsToBlindIndex = await Workspace.distinct('_id', {
_id: {
$nin: workspaceIdsBlindIndexed
}
});
const secretBlindIndexDataToInsert = await Promise.all(
workspaceIdsToBlindIndex.map(async (workspaceToBlindIndex) => {
const salt = crypto.randomBytes(16).toString('base64');
const {
ciphertext: encryptedSaltCiphertext,
iv: saltIV,
tag: saltTag
} = encryptSymmetric128BitHexKeyUTF8({
plaintext: salt,
key: await getEncryptionKey()
});
const secretBlindIndexData = new SecretBlindIndexData({
workspace: workspaceToBlindIndex,
encryptedSaltCiphertext,
saltIV,
saltTag
})
return secretBlindIndexData;
})
);
if (secretBlindIndexDataToInsert.length > 0) {
await SecretBlindIndexData.insertMany(secretBlindIndexDataToInsert);
}
}
export const backfillEncryptionMetadata = async () => {
// backfill bot encryption metadata
await Bot.updateMany(
{
algorithm: {
$exists: false
},
keySize: {
$exists: false
},
keyEncoding: {
$exists: false
}
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keySize: 256,
keyEncoding: ENCODING_SCHEME_UTF8
}
}
);
// backfill secret blind index encryption metadata
await SecretBlindIndexData.updateMany(
{
algorithm: {
$exists: false
},
keySize: {
$exists: false
},
keyEncoding: {
$exists: false
}
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keySize: 256,
keyEncoding: ENCODING_SCHEME_UTF8
}
}
);
// backfill backup private key encryption metadata
await BackupPrivateKey.updateMany(
{
algorithm: {
$exists: false
},
keySize: {
$exists: false
},
keyEncoding: {
$exists: false
}
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
keySize: 256,
keyEncoding: ENCODING_SCHEME_UTF8
}
}
);
// backfill integration auth encryption metadata
await IntegrationAuth.updateMany(
{
},
{
$set: {
algorithm: ALGORITHM_AES_256_GCM,
}
}
);
// TODO: blind indices
// TODO: secret versions and snapshots etc.
// TODO: re-encrypt keys logic
// TODO: how do you handle different parts of the software
// encrypting under different schemes?
// const encryptionKey = await getEncryptionKey();
// const rootEncryptionKey = await getRootEncryptionKey();
// console.log('rootEncryptionKey: ', rootEncryptionKey);
// if (encryptionKey && rootEncryptionKey) {
// // case: both the old encryption key and new encryption key are present
// // -> perform migration if needed
// console.log('rootEncryptionKey is defined');
// const bots = await Bot.find({
// algorithm: ALGORITHM_AES_256_GCM,
// keySize: 256,
// keyEncoding: ENCODING_SCHEME_UTF8
// }, 'encryptedPrivateKey iv tag');
// if (bots.length > 0) {
// // TODO: unencrypt and re-encrypt
// // TODO: unencrypt and re-encrypt blind-indices
// // probably then need to move this function
// console.log('bots: ', bots);
// }
// }
}
+51
View File
@@ -0,0 +1,51 @@
import * as Sentry from '@sentry/node';
import { DatabaseService } from '../../services';
import { setTransporter } from '../../helpers/nodemailer';
import { initSmtp } from '../../services/smtp';
import { createTestUserForDevelopment } from '../addDevelopmentUser'
// eslint-disable-next-line @typescript-eslint/no-var-requires
const { patchRouterParam } = require('../patchAsyncRoutes');
import {
backfillSecretVersions,
backfillSecretBlindIndexData,
backfillEncryptionMetadata
} from './backfill';
import {
getNodeEnv,
getMongoURL,
getSentryDSN
} from '../../config';
/**
* Prepare Infisical upon startup. This includes tasks like:
* - Initializing SMTP configuration
* - Initializing the database connection
* - Initializing Sentry
* - Backfilling data
*/
export const setup = async () => {
// initializing SMTP configuration
setTransporter(await initSmtp());
// initializing the database connection
await DatabaseService.initDatabase(await getMongoURL());
// backfilling data
await backfillSecretVersions();
await backfillSecretBlindIndexData();
await backfillEncryptionMetadata();
// initializing Sentry
if ((await getNodeEnv()) !== 'development') {
Sentry.init({
dsn: await getSentryDSN(),
tracesSampleRate: 1.0,
debug: await getNodeEnv() === 'production' ? false : true,
environment: await getNodeEnv()
});
}
patchRouterParam();
await createTestUserForDevelopment();
}