Begin refactoring backfilling and preparation operations into setup and start adding encryption metadata to models

backend/src/helpers/bot.ts

@@ -13,10 +13,10 @@ import {
 } from "../models";
 import {
   generateKeyPair,
-  encryptSymmetric,
-  decryptSymmetric,
-  decryptAsymmetric,
-} from "../utils/crypto";
+  encryptSymmetric128BitHexKeyUTF8,
+  decryptSymmetric128BitHexKeyUTF8,
+  decryptAsymmetric
+} from '../utils/crypto';
 import {
   SECRET_SHARED,
   AUTH_MODE_JWT,
@@ -26,7 +26,6 @@ import {
 } from "../variables";
 import { getEncryptionKey } from "../config";
 import { BotNotFoundError, UnauthorizedRequestError } from "../utils/errors";
-import { validateMembership } from "../helpers/membership";
 import { validateUserClientForWorkspace } from "../helpers/user";
 import { validateServiceAccountClientForWorkspace } from "../helpers/serviceAccount";
 
@@ -120,7 +119,7 @@ const createBot = async ({
   workspaceId: Types.ObjectId;
 }) => {
   const { publicKey, privateKey } = generateKeyPair();
-  const { ciphertext, iv, tag } = encryptSymmetric({
+  const { ciphertext, iv, tag } = encryptSymmetric128BitHexKeyUTF8({
     plaintext: privateKey,
     key: await getEncryptionKey(),
   });
@@ -161,14 +160,14 @@ const getSecretsHelper = async ({
   });
 
   secrets.forEach((secret: ISecret) => {
-    const secretKey = decryptSymmetric({
+    const secretKey = decryptSymmetric128BitHexKeyUTF8({
       ciphertext: secret.secretKeyCiphertext,
       iv: secret.secretKeyIV,
       tag: secret.secretKeyTag,
       key,
     });
 
-    const secretValue = decryptSymmetric({
+    const secretValue = decryptSymmetric128BitHexKeyUTF8({
       ciphertext: secret.secretValueCiphertext,
       iv: secret.secretValueIV,
       tag: secret.secretValueTag,
@@ -202,7 +201,7 @@ const getKey = async ({ workspaceId }: { workspaceId: string }) => {
   if (!bot) throw new Error("Failed to find bot");
   if (!bot.isActive) throw new Error("Bot is not active");
 
-  const privateKeyBot = decryptSymmetric({
+  const privateKeyBot = decryptSymmetric128BitHexKeyUTF8({
     ciphertext: bot.encryptedPrivateKey,
     iv: bot.iv,
     tag: bot.tag,
@@ -234,7 +233,7 @@ const encryptSymmetricHelper = async ({
   plaintext: string;
 }) => {
   const key = await getKey({ workspaceId: workspaceId.toString() });
-  const { ciphertext, iv, tag } = encryptSymmetric({
+  const { ciphertext, iv, tag } = encryptSymmetric128BitHexKeyUTF8({
     plaintext,
     key,
   });
@@ -266,7 +265,7 @@ const decryptSymmetricHelper = async ({
   tag: string;
 }) => {
   const key = await getKey({ workspaceId: workspaceId.toString() });
-  const plaintext = decryptSymmetric({
+  const plaintext = decryptSymmetric128BitHexKeyUTF8({
     ciphertext,
     iv,
     tag,
@@ -281,5 +280,5 @@ export {
   createBot,
   getSecretsHelper,
   encryptSymmetricHelper,
-  decryptSymmetricHelper,
+  decryptSymmetricHelper
 };

backend/src/helpers/database.ts

@@ -1,6 +1,4 @@
 import mongoose from 'mongoose';
-import { EESecretService } from '../ee/services';
-import { SecretService } from '../services';
 import { getLogger } from '../utils/logger';
 
 /**
@@ -21,9 +19,7 @@ const initDatabaseHelper = async ({
         mongoose.Schema.Types.String.checkRequired(v => typeof v === 'string');
 
         (await getLogger("database")).info("Database connection established");
-        
-        await EESecretService.initSecretVersioning();
-        await SecretService.initSecretBlindIndexDataHelper();
+
     } catch (err) {
         (await getLogger("database")).error(`Unable to establish Database connection due to the error.\n${err}`);
     }

backend/src/helpers/secrets.ts

@@ -54,9 +54,9 @@ import {
 } from '../variables';
 import crypto from 'crypto';
 import * as argon2 from 'argon2';
-import { 
-    encryptSymmetric,
-    decryptSymmetric
+import {
+    encryptSymmetric128BitHexKeyUTF8,
+    decryptSymmetric128BitHexKeyUTF8
 } from '../utils/crypto';
 import { getEncryptionKey } from '../config';
 import { TelemetryService } from '../services';
@@ -220,48 +220,6 @@ const validateClientForSecrets = async ({
     });
 }
 
-/**
- * Initialize secret blind index data by setting previously
- * un-initialized projects to have secret blind index data
- * (Ensures that all projects have associated blind index data)
- */
-const initSecretBlindIndexDataHelper = async () => {
-    const workspaceIdsBlindIndexed = await SecretBlindIndexData.distinct('workspace');
-    const workspaceIdsToBlindIndex = await Workspace.distinct('_id', {
-        _id: {
-            $nin: workspaceIdsBlindIndexed
-        }
-    });
-    
-    const secretBlindIndexDataToInsert = await Promise.all(
-        workspaceIdsToBlindIndex.map(async (workspaceToBlindIndex) => {
-            const salt = crypto.randomBytes(16).toString('base64');
-
-            const { 
-                ciphertext: encryptedSaltCiphertext,
-                iv: saltIV,
-                tag: saltTag
-            } = encryptSymmetric({
-                plaintext: salt,
-                key: await getEncryptionKey()
-            });
-
-            const secretBlindIndexData = new SecretBlindIndexData({
-                workspace: workspaceToBlindIndex,
-                encryptedSaltCiphertext,
-                saltIV,
-                saltTag
-            })
-            
-            return secretBlindIndexData;
-        })
-    );
-    
-    if (secretBlindIndexDataToInsert.length > 0) {
-        await SecretBlindIndexData.insertMany(secretBlindIndexDataToInsert);
-    }
-}
-
 /**
  * Create secret blind index data containing encrypted blind index [salt]
  * for workspace with id [workspaceId]
@@ -280,7 +238,7 @@ const createSecretBlindIndexDataHelper = async ({
         ciphertext: encryptedSaltCiphertext,
         iv: saltIV,
         tag: saltTag
-    } = encryptSymmetric({
+    } = encryptSymmetric128BitHexKeyUTF8({
         plaintext: salt,
         key: await getEncryptionKey()
     });
@@ -314,7 +272,7 @@ const getSecretBlindIndexSaltHelper = async ({
     if (!secretBlindIndexData) throw SecretBlindIndexDataNotFoundError();
     
     // decrypt workspace salt
-    const salt = decryptSymmetric({
+    const salt = decryptSymmetric128BitHexKeyUTF8({
         ciphertext: secretBlindIndexData.encryptedSaltCiphertext,
         iv: secretBlindIndexData.saltIV,
         tag: secretBlindIndexData.saltTag,
@@ -376,7 +334,7 @@ const generateSecretBlindIndexHelper = async ({
     if (!secretBlindIndexData) throw SecretBlindIndexDataNotFoundError();
     
     // decrypt workspace salt
-    const salt = decryptSymmetric({
+    const salt = decryptSymmetric128BitHexKeyUTF8({
         ciphertext: secretBlindIndexData.encryptedSaltCiphertext,
         iv: secretBlindIndexData.saltIV,
         tag: secretBlindIndexData.saltTag,
@@ -934,7 +892,6 @@ const deleteSecretHelper = async ({
 export {
     validateClientForSecret,
     validateClientForSecrets,
-    initSecretBlindIndexDataHelper,
     createSecretBlindIndexDataHelper,
     getSecretBlindIndexSaltHelper,
     generateSecretBlindIndexWithSaltHelper,

backend/src/helpers/workspace.ts

@@ -28,7 +28,6 @@ import {
 	AUTH_MODE_SERVICE_TOKEN,
 	AUTH_MODE_API_KEY
 } from '../variables';
-import { encryptSymmetric } from '../utils/crypto';
 import { SecretService } from '../services';
 
 /**