Finished migration

backend/src/db/migrations/20240202093209_ghost_user.ts

@@ -15,6 +15,7 @@ export async function up(knex: Knex): Promise<void> {
   if (!hasProjectVersionColumn) {
     await knex.schema.alterTable(TableName.Project, (t) => {
       t.string("version").defaultTo(ProjectVersion.V1).notNullable();
+      t.text("upgradeStatus").nullable();
     });
   }
 }
@@ -32,6 +33,7 @@ export async function down(knex: Knex): Promise<void> {
   if (hasProjectVersionColumn) {
     await knex.schema.alterTable(TableName.Project, (t) => {
       t.dropColumn("version");
+      t.dropColumn("upgradeStatus");
     });
   }
 }

backend/src/db/schemas/models.ts

@@ -117,6 +117,24 @@ export enum ProjectVersion {
   V2 = "v2"
 }
 
+// Case study:
+/*
+
+Q: If the upgrade status is null and the project version is V1:
+A: It would mean that the project has not been attempted to be upgraded
+
+Q: If the upgrade status is not null (FAILED or IN_PROGRESS), we a status
+
+Q: if the project status is null, and the project version is v2, we should display nothing cuz its upgraded!
+
+*/
+
+export enum ProjectUpgradeStatus {
+  InProgress = "IN_PROGRESS",
+  Completed = "Project upgrade completed.", // Will be null if completed. So a completed status is not needed
+  Failed = "FAILED"
+}
+
 export enum IdentityAuthMethod {
   Univeral = "universal-auth"
 }

backend/src/db/schemas/projects.ts

@@ -15,7 +15,8 @@ export const ProjectsSchema = z.object({
   orgId: z.string().uuid(),
   createdAt: z.date(),
   updatedAt: z.date(),
-  version: z.string().default("v1")
+  version: z.string().default("v1"),
+  upgradeStatus: z.string().nullable().optional()
 });
 
 export type TProjects = z.infer<typeof ProjectsSchema>;

backend/src/ee/services/saml-config/saml-config-service.ts

@@ -338,7 +338,8 @@ export const samlConfigServiceFactory = ({
             email,
             firstName,
             lastName,
-            authMethods: [AuthMethod.EMAIL]
+            authMethods: [AuthMethod.EMAIL],
+            ghost: false
           },
           tx
         );

backend/src/queue/queue-service.ts

@@ -1,6 +1,7 @@
 import { Job, JobsOptions, Queue, QueueOptions, RepeatOptions, Worker, WorkerListener } from "bullmq";
 import Redis from "ioredis";
 
+import { SecretKeyEncoding } from "@app/db/schemas";
 import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types";
 import {
   TScanFullRepoEventPayload,
@@ -15,7 +16,8 @@ export enum QueueName {
   IntegrationSync = "sync-integrations",
   SecretWebhook = "secret-webhook",
   SecretFullRepoScan = "secret-full-repo-scan",
-  SecretPushEventScan = "secret-push-event-scan"
+  SecretPushEventScan = "secret-push-event-scan",
+  UpgradeProjectToGhost = "upgrade-project-to-ghost"
 }
 
 export enum QueueJobs {
@@ -25,7 +27,8 @@ export enum QueueJobs {
   AuditLogPrune = "audit-log-prune-job",
   SecWebhook = "secret-webhook-trigger",
   IntegrationSync = "secret-integration-pull",
-  SecretScan = "secret-scan"
+  SecretScan = "secret-scan",
+  UpgradeProjectToGhost = "upgrade-project-to-ghost-job"
 }
 
 export type TQueueJobTypes = {
@@ -64,6 +67,20 @@ export type TQueueJobTypes = {
     payload: TScanFullRepoEventPayload;
   };
   [QueueName.SecretPushEventScan]: { name: QueueJobs.SecretScan; payload: TScanPushEventPayload };
+
+  [QueueName.UpgradeProjectToGhost]: {
+    name: QueueJobs.UpgradeProjectToGhost;
+    payload: {
+      projectId: string;
+      startedByUserId: string;
+      encryptedPrivateKey: {
+        encryptedKey: string;
+        encryptedKeyIv: string;
+        encryptedKeyTag: string;
+        keyEncoding: SecretKeyEncoding;
+      };
+    };
+  };
 };
 
 export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;

backend/src/server/routes/index.ts

@@ -65,6 +65,7 @@ import { orgRoleDALFactory } from "@app/services/org/org-role-dal";
 import { orgRoleServiceFactory } from "@app/services/org/org-role-service";
 import { orgServiceFactory } from "@app/services/org/org-service";
 import { projectDALFactory } from "@app/services/project/project-dal";
+import { projectQueueFactory } from "@app/services/project/project-queue";
 import { projectServiceFactory } from "@app/services/project/project-service";
 import { projectBotDALFactory } from "@app/services/project-bot/project-bot-dal";
 import { projectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
@@ -299,19 +300,33 @@ export const registerRoutes = async (
     projectKeyDAL,
     projectMembershipDAL
   });
+
+  const projectQueueService = projectQueueFactory({
+    queueService,
+    secretDAL,
+    folderDAL,
+    projectDAL,
+    orgDAL,
+    orgService,
+    projectEnvDAL,
+    userDAL,
+    secretVersionDAL,
+    projectKeyDAL,
+    projectBotDAL,
+    projectMembershipDAL,
+    secretApprovalRequestDAL,
+    secretApprovalSecretDAL: sarSecretDAL
+  });
+
   const projectService = projectServiceFactory({
     permissionService,
     projectDAL,
+    projectQueue: projectQueueService,
     secretBlindIndexDAL,
     identityProjectDAL,
     identityOrgMembershipDAL,
     projectBotDAL,
-    secretDAL,
-    orgDAL,
-    secretApprovalRequestDAL,
-    secretApprovalSecretDAL: sarSecretDAL,
     projectKeyDAL,
-    secretVersionDAL,
     userDAL,
     projectEnvDAL,
     orgService,

backend/src/server/routes/v2/project-router.ts

@@ -74,7 +74,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
         userPrivateKey: z.string().trim()
       }),
       response: {
-        200: z.object({})
+        200: z.void()
       }
     },
     onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY]),
@@ -88,6 +88,31 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => {
     }
   });
 
+  server.route({
+    url: "/:projectId/upgrade/status",
+    method: "GET",
+    schema: {
+      params: z.object({
+        projectId: z.string().trim()
+      }),
+      response: {
+        200: z.object({
+          status: z.string().nullable()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY]),
+    handler: async (req) => {
+      const status = await server.services.project.getProjectUpgradeStatus({
+        projectId: req.params.projectId,
+        actor: req.permission.type,
+        actorId: req.permission.id
+      });
+
+      return { status };
+    }
+  });
+
   /* Create new project */
   server.route({
     method: "POST",

backend/src/services/auth/auth-signup-service.ts

@@ -50,7 +50,7 @@ export const authSignupServiceFactory = ({
       throw new Error("Failed to send verification code for complete account");
     }
     if (!user) {
-      user = await userDAL.create({ authMethods: [AuthMethod.EMAIL], email });
+      user = await userDAL.create({ authMethods: [AuthMethod.EMAIL], email, ghost: false });
     }
     if (!user) throw new Error("Failed to create user");
 

backend/src/services/project/project-dal.ts

@@ -1,5 +1,5 @@
 import { TDbClient } from "@app/db";
-import { ProjectsSchema, TableName } from "@app/db/schemas";
+import { ProjectsSchema, ProjectUpgradeStatus, TableName, TProjectsUpdate } from "@app/db/schemas";
 import { DatabaseError } from "@app/lib/errors";
 import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
 
@@ -66,6 +66,18 @@ export const projectDALFactory = (db: TDbClient) => {
     }
   };
 
+  const setProjectUpgradeStatus = async (projectId: string, status: ProjectUpgradeStatus) => {
+    try {
+      const data: TProjectsUpdate = {
+        upgradeStatus: status
+      } as const;
+
+      await db(TableName.Project).where({ id: projectId }).update(data);
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Set project upgrade status" });
+    }
+  };
+
   const findAllProjectsByIdentity = async (identityId: string) => {
     try {
       const workspaces = await db(TableName.IdentityProjectMembership)
@@ -149,6 +161,7 @@ export const projectDALFactory = (db: TDbClient) => {
   return {
     ...projectOrm,
     findAllProjects,
+    setProjectUpgradeStatus,
     findAllProjectsByIdentity,
     findProjectGhostUser,
     findProjectById

backend/src/services/project/project-queue.ts

@@ -0,0 +1,410 @@
+/* eslint-disable no-await-in-loop */
+import {
+  ProjectMembershipRole,
+  ProjectUpgradeStatus,
+  ProjectVersion,
+  SecretKeyEncoding,
+  TSecrets
+} from "@app/db/schemas";
+import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal";
+import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal";
+import { RequestState } from "@app/ee/services/secret-approval-request/secret-approval-request-types";
+import {
+  decryptAsymmetric,
+  encryptSymmetric128BitHexKeyUTF8,
+  infisicalSymmetricDecrypt,
+  infisicalSymmetricEncypt
+} from "@app/lib/crypto/encryption";
+import { BadRequestError } from "@app/lib/errors";
+import { logger } from "@app/lib/logger";
+import { createProjectKey, createWsMembers } from "@app/lib/project";
+import { decryptSecrets, SecretDocType, TPartialSecret } from "@app/lib/secret";
+import { QueueJobs, QueueName, TQueueJobTypes, TQueueServiceFactory } from "@app/queue";
+
+import { TOrgDALFactory } from "../org/org-dal";
+import { TOrgServiceFactory } from "../org/org-service";
+import { TProjectBotDALFactory } from "../project-bot/project-bot-dal";
+import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
+import { TProjectKeyDALFactory } from "../project-key/project-key-dal";
+import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal";
+import { TSecretDALFactory } from "../secret/secret-dal";
+import { TSecretVersionDALFactory } from "../secret/secret-version-dal";
+import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
+import { TUserDALFactory } from "../user/user-dal";
+import { TProjectDALFactory } from "./project-dal";
+
+export type TProjectQueueFactory = ReturnType<typeof projectQueueFactory>;
+
+type TProjectQueueFactoryDep = {
+  queueService: TQueueServiceFactory;
+  secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "bulkUpdateNoVersionIncrement">;
+  folderDAL: Pick<TSecretFolderDALFactory, "find">;
+  secretDAL: Pick<TSecretDALFactory, "find" | "bulkUpdateNoVersionIncrement">;
+  projectKeyDAL: Pick<TProjectKeyDALFactory, "findLatestProjectKey" | "find" | "create" | "delete" | "insertMany">;
+  secretApprovalRequestDAL: Pick<TSecretApprovalRequestDALFactory, "find">;
+  secretApprovalSecretDAL: Pick<TSecretApprovalRequestSecretDALFactory, "find" | "bulkUpdateNoVersionIncrement">;
+  projectBotDAL: Pick<TProjectBotDALFactory, "findOne" | "delete" | "create">;
+  orgService: Pick<TOrgServiceFactory, "addGhostUser">;
+  projectMembershipDAL: Pick<TProjectMembershipDALFactory, "create">;
+  userDAL: Pick<TUserDALFactory, "findUserEncKeyByUserId">;
+
+  projectEnvDAL: Pick<TProjectEnvDALFactory, "find">;
+  projectDAL: Pick<TProjectDALFactory, "findOne" | "transaction" | "updateById" | "setProjectUpgradeStatus" | "find">;
+  orgDAL: Pick<TOrgDALFactory, "findMembership">;
+};
+
+export const projectQueueFactory = ({
+  queueService,
+  secretDAL,
+  folderDAL,
+  userDAL,
+  secretVersionDAL,
+  secretApprovalRequestDAL,
+  secretApprovalSecretDAL,
+  projectKeyDAL,
+  projectBotDAL,
+  projectEnvDAL,
+  orgDAL,
+  projectDAL,
+  orgService,
+  projectMembershipDAL
+}: TProjectQueueFactoryDep) => {
+  const upgradeProject = async (dto: TQueueJobTypes["upgrade-project-to-ghost"]["payload"]) => {
+    await queueService.queue(QueueName.UpgradeProjectToGhost, QueueJobs.UpgradeProjectToGhost, dto, {
+      attempts: 1,
+      removeOnComplete: true,
+      removeOnFail: {
+        count: 5 // keep the most recent jobs
+      }
+    });
+  };
+
+  queueService.start(QueueName.UpgradeProjectToGhost, async ({ data }) => {
+    try {
+      const [project] = await projectDAL.find({
+        id: data.projectId,
+        version: ProjectVersion.V1,
+        $in: {
+          upgradeStatus: [ProjectUpgradeStatus.Failed, null]
+        }
+      });
+
+      const oldProjectKey = await projectKeyDAL.findLatestProjectKey(data.startedByUserId, data.projectId);
+
+      if (!project || !oldProjectKey) {
+        throw new BadRequestError({
+          message: "Project or project key not found"
+        });
+      }
+
+      await projectDAL.setProjectUpgradeStatus(data.projectId, ProjectUpgradeStatus.InProgress);
+      const userPrivateKey = infisicalSymmetricDecrypt({
+        keyEncoding: data.encryptedPrivateKey.keyEncoding,
+        ciphertext: data.encryptedPrivateKey.encryptedKey,
+        iv: data.encryptedPrivateKey.encryptedKeyIv,
+        tag: data.encryptedPrivateKey.encryptedKeyTag
+      });
+
+      const projectEnvs = await projectEnvDAL.find({
+        projectId: project.id
+      });
+
+      const projectFolders = await folderDAL.find({
+        $in: {
+          envId: projectEnvs.map((env) => env.id)
+        }
+      });
+
+      // Get all the secrets within the project (as encrypted)
+      const secrets: TPartialSecret[] = [];
+      for (const folder of projectFolders) {
+        const folderSecrets = await secretDAL.find({ folderId: folder.id });
+
+        const folderSecretVersions = await secretVersionDAL.find(
+          {
+            folderId: folder.id
+          },
+          // Only get the latest 100 secret versions for each folder.
+          {
+            limit: 100
+          }
+        );
+        const approvalRequests = await secretApprovalRequestDAL.find({
+          status: RequestState.Open,
+          folderId: folder.id
+        });
+        const approvalSecrets = await secretApprovalSecretDAL.find({
+          $in: {
+            requestId: approvalRequests.map((el) => el.id)
+          }
+        });
+
+        secrets.push(...folderSecrets.map((el) => ({ ...el, docType: SecretDocType.Secret })));
+        secrets.push(...folderSecretVersions.map((el) => ({ ...el, docType: SecretDocType.SecretVersion })));
+        secrets.push(...approvalSecrets.map((el) => ({ ...el, docType: SecretDocType.ApprovalSecret })));
+      }
+
+      const decryptedSecrets = decryptSecrets(secrets, userPrivateKey, oldProjectKey);
+
+      if (secrets.length !== decryptedSecrets.length) {
+        throw new Error("Failed to decrypt some secret versions");
+      }
+
+      // Get the existing bot and the existing project keys for the members of the project
+      const existingBot = await projectBotDAL.findOne({ projectId: project.id }).catch(() => null);
+      const existingProjectKeys = await projectKeyDAL.find({ projectId: project.id });
+
+      // TRANSACTION START
+      await projectDAL.transaction(async (tx) => {
+        await projectDAL.updateById(project.id, { version: ProjectVersion.V2 }, tx);
+
+        // Create a ghost user
+        const ghostUser = await orgService.addGhostUser(project.orgId, tx);
+
+        // Create a project key
+        const { key: newEncryptedProjectKey, iv: newEncryptedProjectKeyIv } = createProjectKey({
+          publicKey: ghostUser.keys.publicKey,
+          privateKey: ghostUser.keys.plainPrivateKey
+        });
+
+        // Create a new project key for the GHOST
+        await projectKeyDAL.create(
+          {
+            projectId: project.id,
+            receiverId: ghostUser.user.id,
+            encryptedKey: newEncryptedProjectKey,
+            nonce: newEncryptedProjectKeyIv,
+            senderId: ghostUser.user.id
+          },
+          tx
+        );
+
+        // Create a membership for the ghost user
+        await projectMembershipDAL.create(
+          {
+            projectId: project.id,
+            userId: ghostUser.user.id,
+            role: ProjectMembershipRole.Admin
+          },
+          tx
+        );
+
+        // If a bot already exists, delete it
+        if (existingBot) {
+          await projectBotDAL.delete({ id: existingBot.id }, tx);
+        }
+
+        // Delete all the existing project keys
+        await projectKeyDAL.delete(
+          {
+            projectId: project.id,
+            $in: {
+              id: existingProjectKeys.map((key) => key.id)
+            }
+          },
+          tx
+        );
+
+        const ghostUserLatestKey = await projectKeyDAL.findLatestProjectKey(ghostUser.user.id, project.id, tx);
+
+        if (!ghostUserLatestKey) {
+          throw new Error("User latest key not found (V2 Upgrade)");
+        }
+
+        const newProjectMembers: {
+          encryptedKey: string;
+          nonce: string;
+          senderId: string;
+          receiverId: string;
+          projectId: string;
+        }[] = [];
+
+        for (const key of existingProjectKeys) {
+          const user = await userDAL.findUserEncKeyByUserId(key.receiverId);
+          const [orgMembership] = await orgDAL.findMembership({ userId: key.receiverId, orgId: project.orgId });
+
+          if (!user || !orgMembership) {
+            throw new Error(`User with ID ${key.receiverId} was not found during upgrade, or user is not in org.`);
+          }
+
+          const [newMember] = createWsMembers({
+            decryptKey: ghostUserLatestKey,
+            userPrivateKey: ghostUser.keys.plainPrivateKey,
+            members: [
+              {
+                userPublicKey: user.publicKey,
+                orgMembershipId: orgMembership.id,
+                projectMembershipRole: ProjectMembershipRole.Admin
+              }
+            ]
+          });
+
+          newProjectMembers.push({
+            encryptedKey: newMember.workspaceEncryptedKey,
+            nonce: newMember.workspaceEncryptedNonce,
+            senderId: ghostUser.user.id,
+            receiverId: user.id,
+            projectId: project.id
+          });
+        }
+
+        // Create project keys for all the old members
+        await projectKeyDAL.insertMany(newProjectMembers, tx);
+
+        // Encrypt the bot private key (which is the same as the ghost user)
+        const { iv, tag, ciphertext, encoding, algorithm } = infisicalSymmetricEncypt(ghostUser.keys.plainPrivateKey);
+
+        // 5. Create a bot for the project
+        const newBot = await projectBotDAL.create(
+          {
+            name: "Infisical Bot (Ghost)",
+            projectId: project.id,
+            tag,
+            iv,
+            encryptedPrivateKey: ciphertext,
+            isActive: true,
+            publicKey: ghostUser.keys.publicKey,
+            senderId: ghostUser.user.id,
+            encryptedProjectKey: newEncryptedProjectKey,
+            encryptedProjectKeyNonce: newEncryptedProjectKeyIv,
+            algorithm,
+            keyEncoding: encoding
+          },
+          tx
+        );
+
+        const botPrivateKey = infisicalSymmetricDecrypt({
+          keyEncoding: newBot.keyEncoding as SecretKeyEncoding,
+          iv: newBot.iv,
+          tag: newBot.tag,
+          ciphertext: newBot.encryptedPrivateKey
+        });
+
+        const botKey = decryptAsymmetric({
+          ciphertext: newBot.encryptedProjectKey!,
+          privateKey: botPrivateKey,
+          nonce: newBot.encryptedProjectKeyNonce!,
+          publicKey: ghostUser.keys.publicKey
+        });
+
+        type TPartialSecret = Pick<
+          TSecrets,
+          | "id"
+          | "secretKeyCiphertext"
+          | "secretKeyIV"
+          | "secretKeyTag"
+          | "secretValueCiphertext"
+          | "secretValueIV"
+          | "secretValueTag"
+          | "secretCommentCiphertext"
+          | "secretCommentIV"
+          | "secretCommentTag"
+        >;
+
+        const updatedSecrets: TPartialSecret[] = [];
+        const updatedSecretVersions: TPartialSecret[] = [];
+        const updatedSecretApprovals: TPartialSecret[] = [];
+        for (const rawSecret of decryptedSecrets) {
+          const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretKey, botKey);
+          const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretValue || "", botKey);
+          const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretComment || "", botKey);
+
+          const payload = {
+            id: rawSecret.id,
+            secretKeyCiphertext: secretKeyEncrypted.ciphertext,
+            secretKeyIV: secretKeyEncrypted.iv,
+            secretKeyTag: secretKeyEncrypted.tag,
+            secretValueCiphertext: secretValueEncrypted.ciphertext,
+            secretValueIV: secretValueEncrypted.iv,
+            secretValueTag: secretValueEncrypted.tag,
+            secretCommentCiphertext: secretCommentEncrypted.ciphertext,
+            secretCommentIV: secretCommentEncrypted.iv,
+            secretCommentTag: secretCommentEncrypted.tag
+          } as const;
+
+          if (rawSecret.docType === SecretDocType.Secret) {
+            updatedSecrets.push(payload);
+          } else if (rawSecret.docType === SecretDocType.SecretVersion) {
+            updatedSecretVersions.push(payload);
+          } else if (rawSecret.docType === SecretDocType.ApprovalSecret) {
+            updatedSecretApprovals.push(payload);
+          } else {
+            throw new Error("Unknown secret type");
+          }
+        }
+
+        const secretUpdates = await secretDAL.bulkUpdateNoVersionIncrement(
+          [
+            ...updatedSecrets.map((secret) => ({
+              filter: { id: secret.id },
+              data: {
+                ...secret,
+                id: undefined
+              }
+            }))
+          ],
+          tx
+        );
+
+        const secretVersionUpdates = await secretVersionDAL.bulkUpdateNoVersionIncrement(
+          [
+            ...updatedSecretVersions.map((version) => ({
+              filter: { id: version.id },
+              data: {
+                ...version,
+                id: undefined
+              }
+            }))
+          ],
+          tx
+        );
+
+        const secretApprovalUpdates = await secretApprovalSecretDAL.bulkUpdateNoVersionIncrement(
+          [
+            ...updatedSecretApprovals.map((approval) => ({
+              filter: {
+                id: approval.id
+              },
+              data: {
+                ...approval,
+                id: undefined
+              }
+            }))
+          ],
+          tx
+        );
+
+        if (secretUpdates.length !== updatedSecrets.length) {
+          throw new Error("Failed to update some secrets");
+        }
+        if (secretVersionUpdates.length !== updatedSecretVersions.length) {
+          throw new Error("Failed to update some secret versions");
+        }
+        if (secretApprovalUpdates.length !== updatedSecretApprovals.length) {
+          throw new Error("Failed to update some secret approvals");
+        }
+
+        throw new Error("Transaction was successful!");
+      });
+    } catch (err) {
+      const project = await projectDAL.findOne({ id: data.projectId, version: ProjectVersion.V1 }).catch(() => null);
+
+      if (!project) {
+        logger.error("Failed to upgrade project, because no project was found", data);
+      } else {
+        await projectDAL.setProjectUpgradeStatus(data.projectId, ProjectUpgradeStatus.Failed);
+        logger.error("Failed to upgrade project", data, err);
+      }
+
+      throw err;
+    }
+  });
+
+  queueService.listen(QueueName.UpgradeProjectToGhost, "failed", (job, err) => {
+    logger.error("Upgrade project failed", job?.data, err);
+  });
+
+  return {
+    upgradeProject
+  };
+};

backend/src/services/project/project-service.ts

@@ -1,45 +1,36 @@
+/* eslint-disable @typescript-eslint/no-redeclare */
 /* eslint-disable no-console */
 /* eslint-disable no-await-in-loop */
 import { ForbiddenError } from "@casl/ability";
 import slugify from "@sindresorhus/slugify";
 
-import { ProjectMembershipRole, ProjectVersion, SecretKeyEncoding, TSecrets } from "@app/db/schemas";
+import { ProjectMembershipRole, ProjectVersion } from "@app/db/schemas";
 import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
 import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
-import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal";
-import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal";
-import { RequestState } from "@app/ee/services/secret-approval-request/secret-approval-request-types";
 import { isAtLeastAsPrivileged } from "@app/lib/casl";
 import { getConfig } from "@app/lib/config/env";
 import { createSecretBlindIndex } from "@app/lib/crypto";
-import {
-  decryptAsymmetric,
-  encryptSymmetric128BitHexKeyUTF8,
-  infisicalSymmetricDecrypt,
-  infisicalSymmetricEncypt
-} from "@app/lib/crypto/encryption";
+import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption";
 import { BadRequestError, ForbiddenRequestError } from "@app/lib/errors";
 import { alphaNumericNanoId } from "@app/lib/nanoid";
 import { createProjectKey, createWsMembers } from "@app/lib/project";
-import { decryptSecrets, SecretDocType, TPartialSecret } from "@app/lib/secret";
+import { TProjectPermission } from "@app/lib/types";
 
 import { ActorType } from "../auth/auth-type";
 import { TIdentityOrgDALFactory } from "../identity/identity-org-dal";
 import { TIdentityProjectDALFactory } from "../identity-project/identity-project-dal";
-import { TOrgDALFactory } from "../org/org-dal";
 import { TOrgServiceFactory } from "../org/org-service";
 import { TProjectBotDALFactory } from "../project-bot/project-bot-dal";
 import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
 import { TProjectKeyDALFactory } from "../project-key/project-key-dal";
 import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal";
-import { TSecretDALFactory } from "../secret/secret-dal";
-import { TSecretVersionDALFactory } from "../secret/secret-version-dal";
 import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-index-dal";
 import { ROOT_FOLDER_NAME, TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TUserDALFactory } from "../user/user-dal";
 import { TProjectDALFactory } from "./project-dal";
+import { TProjectQueueFactory } from "./project-queue";
 import { TCreateProjectDTO, TDeleteProjectDTO, TGetProjectDTO, TUpgradeProjectDTO } from "./project-types";
 
 export const DEFAULT_PROJECT_ENVS = [
@@ -50,22 +41,18 @@ export const DEFAULT_PROJECT_ENVS = [
 
 type TProjectServiceFactoryDep = {
   projectDAL: TProjectDALFactory;
+  projectQueue: TProjectQueueFactory;
   userDAL: TUserDALFactory;
   folderDAL: TSecretFolderDALFactory;
   projectEnvDAL: Pick<TProjectEnvDALFactory, "insertMany" | "find">;
-  secretVersionDAL: TSecretVersionDALFactory;
   identityOrgMembershipDAL: TIdentityOrgDALFactory;
   identityProjectDAL: TIdentityProjectDALFactory;
   projectKeyDAL: Pick<TProjectKeyDALFactory, "create" | "findLatestProjectKey" | "delete" | "find" | "insertMany">;
   projectBotDAL: Pick<TProjectBotDALFactory, "create" | "findById" | "delete" | "findOne">;
   projectMembershipDAL: Pick<TProjectMembershipDALFactory, "create" | "findProjectGhostUser" | "findOne">;
-  orgDAL: TOrgDALFactory;
-  secretApprovalRequestDAL: TSecretApprovalRequestDALFactory;
-  secretApprovalSecretDAL: TSecretApprovalRequestSecretDALFactory;
   secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "create">;
   permissionService: TPermissionServiceFactory;
   orgService: Pick<TOrgServiceFactory, "addGhostUser">;
-  secretDAL: TSecretDALFactory;
   licenseService: Pick<TLicenseServiceFactory, "getPlan">;
 };
 
@@ -73,19 +60,15 @@ export type TProjectServiceFactory = ReturnType<typeof projectServiceFactory>;
 
 export const projectServiceFactory = ({
   projectDAL,
+  projectQueue,
   projectKeyDAL,
-  secretApprovalRequestDAL,
-  secretApprovalSecretDAL,
   permissionService,
   userDAL,
   folderDAL,
   orgService,
-  orgDAL,
   identityProjectDAL,
-  secretVersionDAL,
   projectBotDAL,
   identityOrgMembershipDAL,
-  secretDAL,
   secretBlindIndexDAL,
   projectMembershipDAL,
   projectEnvDAL,
@@ -396,303 +379,33 @@ export const projectServiceFactory = ({
 		25. API route returns 200 OK.
 		*/
 
-    const project = await projectDAL.findOne({ id: projectId, version: ProjectVersion.V1 });
-    const oldProjectKey = await projectKeyDAL.findLatestProjectKey(actorId, projectId);
-    if (!project || !oldProjectKey) {
+    const encryptedPrivateKey = infisicalSymmetricEncypt(userPrivateKey);
+
+    await projectQueue.upgradeProject({
+      projectId,
+      startedByUserId: actorId,
+      encryptedPrivateKey: {
+        encryptedKey: encryptedPrivateKey.ciphertext,
+        encryptedKeyIv: encryptedPrivateKey.iv,
+        encryptedKeyTag: encryptedPrivateKey.tag,
+        keyEncoding: encryptedPrivateKey.encoding
+      }
+    });
+  };
+
+  const getProjectUpgradeStatus = async ({ projectId, actor, actorId }: TProjectPermission) => {
+    const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId);
+    ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Secrets);
+
+    const project = await projectDAL.findProjectById(projectId);
+
+    if (!project) {
       throw new BadRequestError({
-        message: "Project or project key not found"
+        message: `Project with id ${projectId} not found`
       });
     }
 
-    const projectEnvs = await projectEnvDAL.find({
-      projectId: project.id
-    });
-
-    console.log(
-      "projectEnvs",
-      projectEnvs.map((e) => e.name)
-    );
-
-    const projectFolders = await folderDAL.find({
-      $in: {
-        envId: projectEnvs.map((env) => env.id)
-      }
-    });
-
-    // Get all the secrets within the project (as encrypted)
-    const secrets: TPartialSecret[] = [];
-    for (const folder of projectFolders) {
-      const folderSecrets = await secretDAL.find({ folderId: folder.id });
-
-      const folderSecretVersions = await secretVersionDAL.find({
-        folderId: folder.id
-      });
-      const approvalRequests = await secretApprovalRequestDAL.find({
-        status: RequestState.Open,
-        folderId: folder.id
-      });
-      const approvalSecrets = await secretApprovalSecretDAL.find({
-        $in: {
-          requestId: approvalRequests.map((el) => el.id)
-        }
-      });
-
-      secrets.push(...folderSecrets.map((el) => ({ ...el, docType: SecretDocType.Secret })));
-      secrets.push(...folderSecretVersions.map((el) => ({ ...el, docType: SecretDocType.SecretVersion })));
-      secrets.push(...approvalSecrets.map((el) => ({ ...el, docType: SecretDocType.ApprovalSecret })));
-    }
-
-    const decryptedSecrets = decryptSecrets(secrets, userPrivateKey, oldProjectKey);
-
-    if (secrets.length !== decryptedSecrets.length) {
-      throw new Error("Failed to decrypt some secret versions");
-    }
-
-    // Get the existing bot and the existing project keys for the members of the project
-    const existingBot = await projectBotDAL.findOne({ projectId: project.id }).catch(() => null);
-    const existingProjectKeys = await projectKeyDAL.find({ projectId: project.id });
-
-    // TRANSACTION START
-    await projectDAL.transaction(async (tx) => {
-      await projectDAL.updateById(project.id, { version: ProjectVersion.V2 }, tx);
-
-      // Create a ghost user
-      const ghostUser = await orgService.addGhostUser(project.orgId, tx);
-
-      // Create a project key
-      const { key: newEncryptedProjectKey, iv: newEncryptedProjectKeyIv } = createProjectKey({
-        publicKey: ghostUser.keys.publicKey,
-        privateKey: ghostUser.keys.plainPrivateKey
-      });
-
-      console.log("Creating new project key for ghost user");
-      // Create a new project key for the GHOST
-      await projectKeyDAL.create(
-        {
-          projectId: project.id,
-          receiverId: ghostUser.user.id,
-          encryptedKey: newEncryptedProjectKey,
-          nonce: newEncryptedProjectKeyIv,
-          senderId: ghostUser.user.id
-        },
-        tx
-      );
-
-      // Create a membership for the ghost user
-      await projectMembershipDAL.create(
-        {
-          projectId: project.id,
-          userId: ghostUser.user.id,
-          role: ProjectMembershipRole.Admin
-        },
-        tx
-      );
-
-      // If a bot already exists, delete it
-      if (existingBot) {
-        console.log("Deleting existing bot");
-        await projectBotDAL.delete({ id: existingBot.id }, tx);
-      }
-
-      console.log("Deleting old project keys");
-      // Delete all the existing project keys
-      await projectKeyDAL.delete(
-        {
-          projectId: project.id,
-          $in: {
-            id: existingProjectKeys.map((key) => key.id)
-          }
-        },
-        tx
-      );
-
-      console.log("Finding latest key for ghost user");
-      const ghostUserLatestKey = await projectKeyDAL.findLatestProjectKey(ghostUser.user.id, project.id, tx);
-
-      if (!ghostUserLatestKey) {
-        throw new Error("User latest key not found (V2 Upgrade)");
-      }
-
-      console.log("Creating new project keys for old members");
-
-      const newProjectMembers: {
-        encryptedKey: string;
-        nonce: string;
-        senderId: string;
-        receiverId: string;
-        projectId: string;
-      }[] = [];
-
-      for (const key of existingProjectKeys) {
-        const user = await userDAL.findUserEncKeyByUserId(key.receiverId);
-        const [orgMembership] = await orgDAL.findMembership({ userId: key.receiverId, orgId: project.orgId });
-
-        if (!user || !orgMembership) {
-          throw new Error(`User with ID ${key.receiverId} was not found during upgrade, or user is not in org.`);
-        }
-
-        const [newMember] = createWsMembers({
-          decryptKey: ghostUserLatestKey,
-          userPrivateKey: ghostUser.keys.plainPrivateKey,
-          members: [
-            {
-              userPublicKey: user.publicKey,
-              orgMembershipId: orgMembership.id,
-              projectMembershipRole: ProjectMembershipRole.Admin
-            }
-          ]
-        });
-
-        newProjectMembers.push({
-          encryptedKey: newMember.workspaceEncryptedKey,
-          nonce: newMember.workspaceEncryptedNonce,
-          senderId: ghostUser.user.id,
-          receiverId: user.id,
-          projectId: project.id
-        });
-      }
-
-      // Create project keys for all the old members
-      await projectKeyDAL.insertMany(newProjectMembers, tx);
-
-      // Encrypt the bot private key (which is the same as the ghost user)
-      const { iv, tag, ciphertext, encoding, algorithm } = infisicalSymmetricEncypt(ghostUser.keys.plainPrivateKey);
-
-      // 5. Create a bot for the project
-      const newBot = await projectBotDAL.create(
-        {
-          name: "Infisical Bot (Ghost)",
-          projectId: project.id,
-          tag,
-          iv,
-          encryptedPrivateKey: ciphertext,
-          isActive: true,
-          publicKey: ghostUser.keys.publicKey,
-          senderId: ghostUser.user.id,
-          encryptedProjectKey: newEncryptedProjectKey,
-          encryptedProjectKeyNonce: newEncryptedProjectKeyIv,
-          algorithm,
-          keyEncoding: encoding
-        },
-        tx
-      );
-
-      console.log("Updating secrets with new project key");
-      console.log("Got decrypted secrets");
-
-      const botPrivateKey = infisicalSymmetricDecrypt({
-        keyEncoding: newBot.keyEncoding as SecretKeyEncoding,
-        iv: newBot.iv,
-        tag: newBot.tag,
-        ciphertext: newBot.encryptedPrivateKey
-      });
-
-      const botKey = decryptAsymmetric({
-        ciphertext: newBot.encryptedProjectKey!,
-        privateKey: botPrivateKey,
-        nonce: newBot.encryptedProjectKeyNonce!,
-        publicKey: ghostUser.keys.publicKey
-      });
-
-      type TPartialSecret = Pick<
-        TSecrets,
-        | "id"
-        | "secretKeyCiphertext"
-        | "secretKeyIV"
-        | "secretKeyTag"
-        | "secretValueCiphertext"
-        | "secretValueIV"
-        | "secretValueTag"
-        | "secretCommentCiphertext"
-        | "secretCommentIV"
-        | "secretCommentTag"
-      >;
-
-      const updatedSecrets: TPartialSecret[] = [];
-      const updatedSecretVersions: TPartialSecret[] = [];
-      const updatedSecretApprovals: TPartialSecret[] = [];
-      for (const rawSecret of decryptedSecrets) {
-        const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretKey, botKey);
-        const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretValue || "", botKey);
-        const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretComment || "", botKey);
-
-        const payload = {
-          id: rawSecret.id,
-          secretKeyCiphertext: secretKeyEncrypted.ciphertext,
-          secretKeyIV: secretKeyEncrypted.iv,
-          secretKeyTag: secretKeyEncrypted.tag,
-          secretValueCiphertext: secretValueEncrypted.ciphertext,
-          secretValueIV: secretValueEncrypted.iv,
-          secretValueTag: secretValueEncrypted.tag,
-          secretCommentCiphertext: secretCommentEncrypted.ciphertext,
-          secretCommentIV: secretCommentEncrypted.iv,
-          secretCommentTag: secretCommentEncrypted.tag
-        } as const;
-
-        if (rawSecret.docType === SecretDocType.Secret) {
-          updatedSecrets.push(payload);
-        } else if (rawSecret.docType === SecretDocType.SecretVersion) {
-          updatedSecretVersions.push(payload);
-        } else if (rawSecret.docType === SecretDocType.ApprovalSecret) {
-          updatedSecretApprovals.push(payload);
-        } else {
-          throw new Error("Unknown secret type");
-        }
-      }
-
-      const secretUpdates = await secretDAL.bulkUpdateNoVersionIncrement(
-        [
-          ...updatedSecrets.map((secret) => ({
-            filter: { id: secret.id },
-            data: {
-              ...secret,
-              id: undefined
-            }
-          }))
-        ],
-        tx
-      );
-
-      const secretVersionUpdates = await secretVersionDAL.bulkUpdateNoVersionIncrement(
-        [
-          ...updatedSecretVersions.map((version) => ({
-            filter: { id: version.id },
-            data: {
-              ...version,
-              id: undefined
-            }
-          }))
-        ],
-        tx
-      );
-
-      const secretApprovalUpdates = await secretApprovalSecretDAL.bulkUpdateNoVersionIncrement(
-        [
-          ...updatedSecretApprovals.map((approval) => ({
-            filter: {
-              id: approval.id
-            },
-            data: {
-              ...approval,
-              id: undefined
-            }
-          }))
-        ],
-        tx
-      );
-
-      if (secretUpdates.length !== updatedSecrets.length) {
-        throw new Error("Failed to update some secrets");
-      }
-      if (secretVersionUpdates.length !== updatedSecretVersions.length) {
-        throw new Error("Failed to update some secret versions");
-      }
-      if (secretApprovalUpdates.length !== updatedSecretApprovals.length) {
-        throw new Error("Failed to update some secret approvals");
-      }
-
-      throw new Error("Transaction was successful");
-    });
+    return project.upgradeStatus || null;
   };
 
   return {
@@ -700,6 +413,7 @@ export const projectServiceFactory = ({
     deleteProject,
     getProjects,
     findProjectGhostUser,
+    getProjectUpgradeStatus,
     getAProject,
     toggleAutoCapitalization,
     updateName,

backend/src/services/super-admin/super-admin-service.ts

@@ -70,6 +70,7 @@ export const superAdminServiceFactory = ({
           lastName,
           email,
           superAdmin: true,
+          ghost: false,
           isAccepted: true,
           authMethods: [AuthMethod.EMAIL]
         },