/* eslint-disable no-await-in-loop */ import { IntegrationAuthsSchema, ProjectMembershipRole, ProjectUpgradeStatus, ProjectVersion, SecretApprovalRequestsSecretsSchema, SecretKeyEncoding, SecretsSchema, SecretVersionsSchema, TIntegrationAuths, TSecretApprovalRequestsSecrets, TSecrets, TSecretVersions } from "@app/db/schemas"; import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal"; import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; import { RequestState } from "@app/ee/services/secret-approval-request/secret-approval-request-types"; import { decryptIntegrationAuths, decryptSecretApprovals, decryptSecrets, decryptSecretVersions } from "@app/lib/crypto"; import { decryptAsymmetric, encryptSymmetric128BitHexKeyUTF8, infisicalSymmetricDecrypt, infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { logger } from "@app/lib/logger"; import { QueueJobs, QueueName, TQueueJobTypes, TQueueServiceFactory } from "@app/queue"; import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal"; import { TOrgDALFactory } from "../org/org-dal"; import { TOrgServiceFactory } from "../org/org-service"; import { TProjectBotDALFactory } from "../project-bot/project-bot-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectKeyDALFactory } from "../project-key/project-key-dal"; import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal"; import { TSecretDALFactory } from "../secret/secret-dal"; import { TSecretVersionDALFactory } from "../secret/secret-version-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TUserDALFactory } from "../user/user-dal"; import { TProjectDALFactory } from "./project-dal"; import { assignWorkspaceKeysToMembers, createProjectKey } from "./project-fns"; export type TProjectQueueFactory = ReturnType; type TProjectQueueFactoryDep = { queueService: TQueueServiceFactory; secretVersionDAL: Pick; folderDAL: Pick; secretDAL: Pick; projectKeyDAL: Pick; secretApprovalRequestDAL: Pick; secretApprovalSecretDAL: Pick; projectBotDAL: Pick; orgService: Pick; projectMembershipDAL: Pick; projectUserMembershipRoleDAL: Pick; integrationAuthDAL: TIntegrationAuthDALFactory; userDAL: Pick; projectEnvDAL: Pick; projectDAL: Pick; orgDAL: Pick; }; export const projectQueueFactory = ({ queueService, secretDAL, folderDAL, userDAL, secretVersionDAL, integrationAuthDAL, secretApprovalRequestDAL, secretApprovalSecretDAL, projectKeyDAL, projectBotDAL, projectEnvDAL, orgDAL, projectDAL, orgService, projectMembershipDAL, projectUserMembershipRoleDAL }: TProjectQueueFactoryDep) => { const upgradeProject = async (dto: TQueueJobTypes["upgrade-project-to-ghost"]["payload"]) => { await queueService.queue(QueueName.UpgradeProjectToGhost, QueueJobs.UpgradeProjectToGhost, dto, { attempts: 1, removeOnComplete: true, removeOnFail: { count: 5 // keep the most recent jobs } }); }; queueService.start(QueueName.UpgradeProjectToGhost, async ({ data }) => { try { const [project] = await projectDAL.find({ id: data.projectId, version: ProjectVersion.V1 }); const oldProjectKey = await projectKeyDAL.findLatestProjectKey(data.startedByUserId, data.projectId); if (!project) { throw new Error("Project not found"); } if (!oldProjectKey) { throw new Error("Old project key not found"); } if (project.upgradeStatus !== ProjectUpgradeStatus.Failed && project.upgradeStatus !== null) { throw new Error("Project upgrade status is not valid"); } await projectDAL.setProjectUpgradeStatus(data.projectId, ProjectUpgradeStatus.InProgress); // Set the status to in progress. This is important to prevent multiple upgrades at the same time. // eslint-disable-next-line no-promise-executor-return // await new Promise((resolve) => setTimeout(resolve, 50_000)); const userPrivateKey = infisicalSymmetricDecrypt({ keyEncoding: data.encryptedPrivateKey.keyEncoding, ciphertext: data.encryptedPrivateKey.encryptedKey, iv: data.encryptedPrivateKey.encryptedKeyIv, tag: data.encryptedPrivateKey.encryptedKeyTag }); const decryptedPlainProjectKey = decryptAsymmetric({ ciphertext: oldProjectKey.encryptedKey, nonce: oldProjectKey.nonce, publicKey: oldProjectKey.sender.publicKey, privateKey: userPrivateKey }); const projectEnvs = await projectEnvDAL.find({ projectId: project.id }); const projectFolders = await folderDAL.find({ $in: { envId: projectEnvs.map((env) => env.id) } }); // Get all the secrets within the project (as encrypted) const projectIntegrationAuths = await integrationAuthDAL.find({ projectId: project.id }); const secrets: TSecrets[] = []; const secretVersions: TSecretVersions[] = []; const approvalSecrets: TSecretApprovalRequestsSecrets[] = []; const folderSecretVersionIdsToDelete: string[] = []; for (const folder of projectFolders) { const folderSecrets = await secretDAL.find({ folderId: folder.id }); const folderSecretVersions = await secretVersionDAL.find( { folderId: folder.id }, // Only get the latest 700 secret versions for each folder. { limit: 1000, sort: [["createdAt", "desc"]] } ); const deletedSecretVersions = await secretVersionDAL.find( { folderId: folder.id }, { // Get all the secret versions that are not the latest 700 offset: 1000 } ); folderSecretVersionIdsToDelete.push(...deletedSecretVersions.map((el) => el.id)); const approvalRequests = await secretApprovalRequestDAL.find({ status: RequestState.Open, folderId: folder.id }); const secretApprovals = await secretApprovalSecretDAL.find({ $in: { requestId: approvalRequests.map((el) => el.id) } }); secrets.push(...folderSecrets); secretVersions.push(...folderSecretVersions); approvalSecrets.push(...secretApprovals); } const decryptedSecrets = decryptSecrets(secrets, userPrivateKey, oldProjectKey); const decryptedSecretVersions = decryptSecretVersions(secretVersions, userPrivateKey, oldProjectKey); const decryptedApprovalSecrets = decryptSecretApprovals(approvalSecrets, userPrivateKey, oldProjectKey); const decryptedIntegrationAuths = decryptIntegrationAuths(projectIntegrationAuths, userPrivateKey, oldProjectKey); // Get the existing bot and the existing project keys for the members of the project const existingBot = await projectBotDAL.findOne({ projectId: project.id }).catch(() => null); const existingProjectKeys = await projectKeyDAL.find({ projectId: project.id }); // TRANSACTION START await projectDAL.transaction(async (tx) => { await projectDAL.updateById(project.id, { version: ProjectVersion.V2 }, tx); // Create a ghost user const ghostUser = await orgService.addGhostUser(project.orgId, tx); // Create a project key const { key: newEncryptedProjectKey, iv: newEncryptedProjectKeyIv } = createProjectKey({ plainProjectKey: decryptedPlainProjectKey, publicKey: ghostUser.keys.publicKey, privateKey: ghostUser.keys.plainPrivateKey }); // Create a new project key for the GHOST await projectKeyDAL.create( { projectId: project.id, receiverId: ghostUser.user.id, encryptedKey: newEncryptedProjectKey, nonce: newEncryptedProjectKeyIv, senderId: ghostUser.user.id }, tx ); // Create a membership for the ghost user const projectMembership = await projectMembershipDAL.create( { projectId: project.id, userId: ghostUser.user.id }, tx ); await projectUserMembershipRoleDAL.create( { projectMembershipId: projectMembership.id, role: ProjectMembershipRole.Admin }, tx ); // If a bot already exists, delete it if (existingBot) { await projectBotDAL.delete({ id: existingBot.id }, tx); } // Delete all the existing project keys await projectKeyDAL.delete( { projectId: project.id, $in: { id: existingProjectKeys.map((key) => key.id) } }, tx ); const ghostUserLatestKey = await projectKeyDAL.findLatestProjectKey(ghostUser.user.id, project.id, tx); if (!ghostUserLatestKey) { throw new Error("User latest key not found (V2 Upgrade)"); } const newProjectMembers: { encryptedKey: string; nonce: string; senderId: string; receiverId: string; projectId: string; }[] = []; for (const key of existingProjectKeys) { const user = await userDAL.findUserEncKeyByUserId(key.receiverId); const [orgMembership] = await orgDAL.findMembership({ userId: key.receiverId, orgId: project.orgId }); if (!user) { throw new Error(`User with ID ${key.receiverId} was not found during upgrade.`); } if (!orgMembership) { // This can happen. Since we don't remove project memberships and project keys when a user is removed from an org, this is a valid case. logger.info("User is not in organization", { userId: key.receiverId, orgId: project.orgId, projectId: project.id }); // eslint-disable-next-line no-continue continue; } const [newMember] = assignWorkspaceKeysToMembers({ decryptKey: ghostUserLatestKey, userPrivateKey: ghostUser.keys.plainPrivateKey, members: [ { userPublicKey: user.publicKey, orgMembershipId: orgMembership.id, projectMembershipRole: ProjectMembershipRole.Admin } ] }); newProjectMembers.push({ encryptedKey: newMember.workspaceEncryptedKey, nonce: newMember.workspaceEncryptedNonce, senderId: ghostUser.user.id, receiverId: user.id, projectId: project.id }); } // Create project keys for all the old members await projectKeyDAL.insertMany(newProjectMembers, tx); // Encrypt the bot private key (which is the same as the ghost user) const { iv, tag, ciphertext, encoding, algorithm } = infisicalSymmetricEncypt(ghostUser.keys.plainPrivateKey); // 5. Create a bot for the project const newBot = await projectBotDAL.create( { name: "Infisical Bot (Ghost)", projectId: project.id, tag, iv, encryptedPrivateKey: ciphertext, isActive: true, publicKey: ghostUser.keys.publicKey, senderId: ghostUser.user.id, encryptedProjectKey: newEncryptedProjectKey, encryptedProjectKeyNonce: newEncryptedProjectKeyIv, algorithm, keyEncoding: encoding }, tx ); const botPrivateKey = infisicalSymmetricDecrypt({ keyEncoding: newBot.keyEncoding as SecretKeyEncoding, iv: newBot.iv, tag: newBot.tag, ciphertext: newBot.encryptedPrivateKey }); const botKey = decryptAsymmetric({ ciphertext: newBot.encryptedProjectKey!, privateKey: botPrivateKey, nonce: newBot.encryptedProjectKeyNonce!, publicKey: ghostUser.keys.publicKey }); const updatedSecrets: TSecrets[] = []; const updatedSecretVersions: TSecretVersions[] = []; const updatedSecretApprovals: TSecretApprovalRequestsSecrets[] = []; const updatedIntegrationAuths: TIntegrationAuths[] = []; for (const rawSecret of decryptedSecrets) { const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.decrypted.secretKey, botKey); const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.decrypted.secretValue || "", botKey); const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8( rawSecret.decrypted.secretComment || "", botKey ); const payload: TSecrets = { ...rawSecret.original, keyEncoding: SecretKeyEncoding.UTF8, secretKeyCiphertext: secretKeyEncrypted.ciphertext, secretKeyIV: secretKeyEncrypted.iv, secretKeyTag: secretKeyEncrypted.tag, secretValueCiphertext: secretValueEncrypted.ciphertext, secretValueIV: secretValueEncrypted.iv, secretValueTag: secretValueEncrypted.tag, secretCommentCiphertext: secretCommentEncrypted.ciphertext, secretCommentIV: secretCommentEncrypted.iv, secretCommentTag: secretCommentEncrypted.tag } as const; if (!SecretsSchema.safeParse(payload).success) { throw new Error(`Invalid secret payload: ${JSON.stringify(payload)}`); } updatedSecrets.push(payload); } for (const rawSecretVersion of decryptedSecretVersions) { const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecretVersion.decrypted.secretKey, botKey); const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8( rawSecretVersion.decrypted.secretValue || "", botKey ); const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8( rawSecretVersion.decrypted.secretComment || "", botKey ); const payload: TSecretVersions = { ...rawSecretVersion.original, keyEncoding: SecretKeyEncoding.UTF8, secretKeyCiphertext: secretKeyEncrypted.ciphertext, secretKeyIV: secretKeyEncrypted.iv, secretKeyTag: secretKeyEncrypted.tag, secretValueCiphertext: secretValueEncrypted.ciphertext, secretValueIV: secretValueEncrypted.iv, secretValueTag: secretValueEncrypted.tag, secretCommentCiphertext: secretCommentEncrypted.ciphertext, secretCommentIV: secretCommentEncrypted.iv, secretCommentTag: secretCommentEncrypted.tag } as const; if (!SecretVersionsSchema.safeParse(payload).success) { throw new Error(`Invalid secret version payload: ${JSON.stringify(payload)}`); } updatedSecretVersions.push(payload); } for (const rawSecretApproval of decryptedApprovalSecrets) { const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecretApproval.decrypted.secretKey, botKey); const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8( rawSecretApproval.decrypted.secretValue || "", botKey ); const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8( rawSecretApproval.decrypted.secretComment || "", botKey ); const payload: TSecretApprovalRequestsSecrets = { ...rawSecretApproval.original, keyEncoding: SecretKeyEncoding.UTF8, secretKeyCiphertext: secretKeyEncrypted.ciphertext, secretKeyIV: secretKeyEncrypted.iv, secretKeyTag: secretKeyEncrypted.tag, secretValueCiphertext: secretValueEncrypted.ciphertext, secretValueIV: secretValueEncrypted.iv, secretValueTag: secretValueEncrypted.tag, secretCommentCiphertext: secretCommentEncrypted.ciphertext, secretCommentIV: secretCommentEncrypted.iv, secretCommentTag: secretCommentEncrypted.tag } as const; if (!SecretApprovalRequestsSecretsSchema.safeParse(payload).success) { throw new Error(`Invalid secret approval payload: ${JSON.stringify(payload)}`); } updatedSecretApprovals.push(payload); } for (const integrationAuth of decryptedIntegrationAuths) { const access = encryptSymmetric128BitHexKeyUTF8(integrationAuth.decrypted.access, botKey); const accessId = encryptSymmetric128BitHexKeyUTF8(integrationAuth.decrypted.accessId, botKey); const refresh = encryptSymmetric128BitHexKeyUTF8(integrationAuth.decrypted.refresh, botKey); const payload: TIntegrationAuths = { ...integrationAuth.original, keyEncoding: SecretKeyEncoding.UTF8, accessCiphertext: access.ciphertext, accessIV: access.iv, accessTag: access.tag, accessIdCiphertext: accessId.ciphertext, accessIdIV: accessId.iv, accessIdTag: accessId.tag, refreshCiphertext: refresh.ciphertext, refreshIV: refresh.iv, refreshTag: refresh.tag } as const; if (!IntegrationAuthsSchema.safeParse(payload).success) { throw new Error(`Invalid integration auth payload: ${JSON.stringify(payload)}`); } updatedIntegrationAuths.push(payload); } if (updatedSecrets.length !== secrets.length) { throw new Error("Failed to update some secrets"); } if (updatedSecretVersions.length !== secretVersions.length) { throw new Error("Failed to update some secret versions"); } if (updatedSecretApprovals.length !== approvalSecrets.length) { throw new Error("Failed to update some secret approvals"); } if (updatedIntegrationAuths.length !== projectIntegrationAuths.length) { throw new Error("Failed to update some integration auths"); } const secretUpdates = await secretDAL.bulkUpdateNoVersionIncrement(updatedSecrets, tx); const secretVersionUpdates = await secretVersionDAL.bulkUpdateNoVersionIncrement(updatedSecretVersions, tx); const secretApprovalUpdates = await secretApprovalSecretDAL.bulkUpdateNoVersionIncrement( updatedSecretApprovals, tx ); const integrationAuthUpdates = await integrationAuthDAL.bulkUpdate( updatedIntegrationAuths.map((el) => ({ filter: { id: el.id }, data: { ...el, id: undefined } })), tx ); // Delete all secret versions that are no longer needed. We only store the latest 100 versions for each secret. await secretVersionDAL.delete( { $in: { id: folderSecretVersionIdsToDelete } }, tx ); if ( secretUpdates.length !== updatedSecrets.length || secretVersionUpdates.length !== updatedSecretVersions.length || secretApprovalUpdates.length !== updatedSecretApprovals.length || integrationAuthUpdates.length !== updatedIntegrationAuths.length ) { throw new Error("Parts of the upgrade failed. Some secrets were not updated"); } await projectDAL.setProjectUpgradeStatus(data.projectId, null, tx); // await new Promise((resolve) => setTimeout(resolve, 15_000)); // throw new Error("Transaction was successful!"); }); } catch (err) { const [project] = await projectDAL .find({ id: data.projectId, version: ProjectVersion.V1 }) .catch(() => [null]); if (!project) { logger.error("Failed to upgrade project, because no project was found", data); } else { await projectDAL.setProjectUpgradeStatus(data.projectId, ProjectUpgradeStatus.Failed); logger.error("Failed to upgrade project", err, { extra: { project, jobData: data } }); } throw err; } }); queueService.listen(QueueName.UpgradeProjectToGhost, "failed", (job, err) => { logger.error(err, "Upgrade project failed", job?.data); }); return { upgradeProject }; };