From 031a2416a9a45b19c0c6c9fea9a0da85b553b58f Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:37:39 +0400 Subject: [PATCH 001/157] Schemas (mostly linting) --- backend/src/db/schemas/projects.ts | 3 ++- backend/src/db/schemas/users.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/src/db/schemas/projects.ts b/backend/src/db/schemas/projects.ts index 296fa421e..8c20ca703 100644 --- a/backend/src/db/schemas/projects.ts +++ b/backend/src/db/schemas/projects.ts @@ -14,7 +14,8 @@ export const ProjectsSchema = z.object({ autoCapitalization: z.boolean().default(true).nullable().optional(), orgId: z.string().uuid(), createdAt: z.date(), - updatedAt: z.date() + updatedAt: z.date(), + e2ee: z.boolean().default(true) }); export type TProjects = z.infer; diff --git a/backend/src/db/schemas/users.ts b/backend/src/db/schemas/users.ts index 4a29de510..e90b06187 100644 --- a/backend/src/db/schemas/users.ts +++ b/backend/src/db/schemas/users.ts @@ -19,7 +19,8 @@ export const UsersSchema = z.object({ mfaMethods: z.string().array().nullable().optional(), devices: z.unknown().nullable().optional(), createdAt: z.date(), - updatedAt: z.date() + updatedAt: z.date(), + ghost: z.boolean().default(false) }); export type TUsers = z.infer; From 3612e5834cba03b15903a6f3e8bdd2c5bc6e6393 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:37:58 +0400 Subject: [PATCH 002/157] Ghost user migration --- .../migrations/20240202093209_ghost_user.ts | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 backend/src/db/migrations/20240202093209_ghost_user.ts diff --git a/backend/src/db/migrations/20240202093209_ghost_user.ts b/backend/src/db/migrations/20240202093209_ghost_user.ts new file mode 100644 index 000000000..7fa23d245 --- /dev/null +++ b/backend/src/db/migrations/20240202093209_ghost_user.ts @@ -0,0 +1,31 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasColumn(TableName.Users, "ghost"))) { + await knex.schema.alterTable(TableName.Users, (t) => { + t.boolean("ghost").defaultTo(false).notNullable(); + }); + } + + if (!(await knex.schema.hasColumn(TableName.Project, "e2ee"))) { + await knex.schema.alterTable(TableName.Project, (t) => { + t.boolean("e2ee").defaultTo(true).notNullable(); + }); + } +} + +export async function down(knex: Knex): Promise { + if (await knex.schema.hasColumn(TableName.Users, "ghost")) { + await knex.schema.alterTable(TableName.Users, (t) => { + t.dropColumn("ghost"); + }); + } + + if (await knex.schema.hasColumn(TableName.Project, "e2ee")) { + await knex.schema.alterTable(TableName.Project, (t) => { + t.dropColumn("e2ee"); + }); + } +} From 8e3cddc1eaf52dadd0609c5ab8f7ab234b1bb2c1 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:39:21 +0400 Subject: [PATCH 003/157] TS erros --- backend/src/db/seeds/1-user.ts | 4 ++++ backend/src/db/seeds/2-org.ts | 3 +++ 2 files changed, 7 insertions(+) diff --git a/backend/src/db/seeds/1-user.ts b/backend/src/db/seeds/1-user.ts index 8e2b00b90..af40dbbbc 100644 --- a/backend/src/db/seeds/1-user.ts +++ b/backend/src/db/seeds/1-user.ts @@ -1,3 +1,6 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +// @ts-nocheck + import { Knex } from "knex"; import { AuthMethod } from "../../services/auth/auth-type"; @@ -19,6 +22,7 @@ export async function seed(knex: Knex): Promise { id: seedData1.id, email: seedData1.email, superAdmin: true, + ghost: false, firstName: "test", lastName: "", authMethods: [AuthMethod.EMAIL], diff --git a/backend/src/db/seeds/2-org.ts b/backend/src/db/seeds/2-org.ts index ba2f65a36..a2990054b 100644 --- a/backend/src/db/seeds/2-org.ts +++ b/backend/src/db/seeds/2-org.ts @@ -1,3 +1,6 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +// @ts-nocheck + import { Knex } from "knex"; import { OrgMembershipRole, OrgMembershipStatus, TableName } from "../schemas"; From aec0e8618268199ca0e35de4d7d32a11740a274b Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:39:55 +0400 Subject: [PATCH 004/157] Crypto stuff --- backend/src/lib/crypto/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/lib/crypto/index.ts b/backend/src/lib/crypto/index.ts index 62278de1d..be896a8bc 100644 --- a/backend/src/lib/crypto/index.ts +++ b/backend/src/lib/crypto/index.ts @@ -10,3 +10,4 @@ export { generateAsymmetricKeyPair } from "./encryption"; export { generateSrpServerKey, srpCheckClientProof } from "./srp"; +export { decodeBase64, encodeBase64 } from "tweetnacl-util"; From 80a802386c8aea26fa5ed871e4dba9750ff66f0e Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:40:28 +0400 Subject: [PATCH 005/157] Helper functions for adding workspace members on serverside --- backend/src/lib/project/index.ts | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 backend/src/lib/project/index.ts diff --git a/backend/src/lib/project/index.ts b/backend/src/lib/project/index.ts new file mode 100644 index 000000000..20a63cbd5 --- /dev/null +++ b/backend/src/lib/project/index.ts @@ -0,0 +1,39 @@ +import { ProjectMembershipRole, TProjectKeys } from "@app/db/schemas"; + +import { decryptAsymmetric, encryptAsymmetric } from "../crypto"; + +type AddUserToWsDTO = { + decryptKey: TProjectKeys & { sender: { publicKey: string } }; + userPrivateKey: string; + members: { + orgMembershipId: string; + projectMembershipRole: ProjectMembershipRole; + userPublicKey: string; + }[]; +}; + +export const createWsMembers = async ({ members, decryptKey, userPrivateKey }: AddUserToWsDTO) => { + const key = decryptAsymmetric({ + ciphertext: decryptKey.encryptedKey, + nonce: decryptKey.nonce, + publicKey: decryptKey.sender.publicKey, + privateKey: userPrivateKey + }); + + const newWsMembers = members.map(({ orgMembershipId, userPublicKey, projectMembershipRole }) => { + const { ciphertext: inviteeCipherText, nonce: inviteeNonce } = encryptAsymmetric( + key, + userPublicKey, + userPrivateKey + ); + + return { + orgMembershipId, + projectRole: projectMembershipRole, + workspaceEncryptedKey: inviteeCipherText, + workspaceEncryptedNonce: inviteeNonce + }; + }); + + return newWsMembers; +}; From fe21ba0e54f006d2c4ccf237ddda5da194cb99ec Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:40:47 +0400 Subject: [PATCH 006/157] Ghost user! --- backend/src/server/routes/v3/index.ts | 2 + .../src/server/routes/v3/project-router.ts | 126 ++++++++++++++++++ .../src/services/auth/auth-login-service.ts | 2 +- backend/src/services/org/org-dal.ts | 42 +++++- backend/src/services/org/org-service.ts | 49 ++++++- .../project-bot/project-bot-service.ts | 78 +++++------ .../services/project-bot/project-bot-types.ts | 8 ++ .../project-key/project-key-service.ts | 3 +- .../project-membership-dal.ts | 5 +- .../project-membership-service.ts | 15 ++- .../project-membership-types.ts | 2 + .../src/services/project/project-service.ts | 10 +- .../utilities/cryptography/crypto.ts | 35 ++--- frontend/src/hooks/api/workspace/queries.tsx | 10 +- frontend/src/hooks/api/workspace/types.ts | 3 +- .../src/pages/org/[id]/overview/index.tsx | 67 ++++------ 16 files changed, 332 insertions(+), 125 deletions(-) create mode 100644 backend/src/server/routes/v3/project-router.ts diff --git a/backend/src/server/routes/v3/index.ts b/backend/src/server/routes/v3/index.ts index 6a061a635..21d57d359 100644 --- a/backend/src/server/routes/v3/index.ts +++ b/backend/src/server/routes/v3/index.ts @@ -1,4 +1,5 @@ import { registerLoginRouter } from "./login-router"; +import { registerProjectRouter } from "./project-router"; import { registerSecretBlindIndexRouter } from "./secret-blind-index-router"; import { registerSecretRouter } from "./secret-router"; import { registerSignupRouter } from "./signup-router"; @@ -10,4 +11,5 @@ export const registerV3Routes = async (server: FastifyZodProvider) => { await server.register(registerUserRouter, { prefix: "/users" }); await server.register(registerSecretRouter, { prefix: "/secrets" }); await server.register(registerSecretBlindIndexRouter, { prefix: "/workspaces" }); + await server.register(registerProjectRouter, { prefix: "/projects" }); }; diff --git a/backend/src/server/routes/v3/project-router.ts b/backend/src/server/routes/v3/project-router.ts new file mode 100644 index 000000000..5f83167a7 --- /dev/null +++ b/backend/src/server/routes/v3/project-router.ts @@ -0,0 +1,126 @@ +import crypto from "crypto"; +import { z } from "zod"; + +import { ProjectMembershipRole, ProjectsSchema } from "@app/db/schemas"; +import { encryptAsymmetric } from "@app/lib/crypto"; +import { createWsMembers } from "@app/lib/project"; +import { authRateLimit } from "@app/server/config/rateLimiter"; + +const projectWithEnv = ProjectsSchema.merge( + z.object({ + _id: z.string(), + environments: z.object({ name: z.string(), slug: z.string(), id: z.string() }).array() + }) +); + +export const registerProjectRouter = async (server: FastifyZodProvider) => { + /* Create new project */ + server.route({ + method: "POST", + url: "/", + config: { + rateLimit: authRateLimit + }, + schema: { + body: z.object({ + projectName: z.string().trim(), + inviteAllOrgMembers: z.boolean(), + organizationId: z.string().trim() + }), + response: { + 200: z.object({ + workspace: projectWithEnv + }) + } + }, + handler: async (req) => { + // 1. create the ghost user and add it to the org as admin + const ghost = await server.services.org.addGhostUser(req.body.organizationId); + + // 2. create the workspace + const workspace = await server.services.project.createProject({ + actorId: ghost.user.id, + actor: req.permission.type, + orgId: req.body.organizationId, + workspaceName: req.body.projectName + }); + + // 3. create a random key that we'll use as the project key + const randomBytes = crypto.randomBytes(16).toString("hex"); + + const ghostPrivateKey = ghost.keys.plainPrivateKey; + + const { ciphertext: encryptedProjectKey, nonce: encryptedProjectKeyIv } = encryptAsymmetric( + randomBytes, + ghost.keys.publicKey, + ghostPrivateKey + ); + + // 3. create workspace keys for the ghost user + await server.services.projectKey.uploadProjectKeys({ + projectId: workspace.id, + actor: req.permission.type, + actorId: ghost.user.id, + nonce: encryptedProjectKeyIv, + receiverId: ghost.user.id, + encryptedKey: encryptedProjectKey + }); + + // 4. create a project bot + const bot = await server.services.projectBot.findBotByProjectId({ + actorId: ghost.user.id, + actor: req.permission.type, + projectId: workspace.id + }); + + // 5. activate the bot + await server.services.projectBot.setBotActiveState({ + botKey: { + encryptedKey: encryptedProjectKey, + nonce: encryptedProjectKeyIv + }, + actorId: ghost.user.id, + isActive: true, + actor: req.permission.type, + botId: bot.id + }); + + // 6. get the current user & org membership + const user = await server.services.user.getMe(req.permission.id); + const userOrgMembership = await server.services.permission.getUserOrgPermission(user.id, req.body.organizationId); + + // 7. Get the latest key from the ghost! + const latestKey = await server.services.projectKey.getLatestProjectKey({ + actorId: ghost.user.id, + actor: req.permission.type, + projectId: workspace.id + }); + + if (!latestKey) throw new Error("Failed to get latest key"); + + // 8. Create workspace members for the current user + + const projectAdmin = await createWsMembers({ + decryptKey: latestKey, + members: [ + { + userPublicKey: user.publicKey, + orgMembershipId: userOrgMembership.membership.id, + projectMembershipRole: ProjectMembershipRole.Admin // <-- Make the first user an admin + } + ], + userPrivateKey: ghostPrivateKey + }); + + // 9. Add the current user to the workspace + await server.services.projectMembership.addUsersToProject({ + projectId: workspace.id, + actorId: ghost.user.id, + actor: req.permission.type, + members: projectAdmin + }); + + return { workspace }; + } + }); +}; diff --git a/backend/src/services/auth/auth-login-service.ts b/backend/src/services/auth/auth-login-service.ts index 6e4d60bba..231a99e19 100644 --- a/backend/src/services/auth/auth-login-service.ts +++ b/backend/src/services/auth/auth-login-service.ts @@ -275,7 +275,7 @@ export const authLoginServiceFactory = ({ userDAL, tokenService, smtpService }: if (isOauthSignUpDisabled) throw new BadRequestError({ message: "User signup disabled", name: "Oauth 2 login" }); if (!user) { - user = await userDAL.create({ email, firstName, lastName, authMethods: [authMethod] }); + user = await userDAL.create({ email, firstName, lastName, authMethods: [authMethod], ghost: false }); } const isLinkingRequired = !user?.authMethods?.includes(authMethod); const isUserCompleted = user.isAccepted; diff --git a/backend/src/services/org/org-dal.ts b/backend/src/services/org/org-dal.ts index bb3cbbd95..e708cefdd 100644 --- a/backend/src/services/org/org-dal.ts +++ b/backend/src/services/org/org-dal.ts @@ -76,7 +76,8 @@ export const orgDALFactory = (db: TDbClient) => { db.ref("lastName").withSchema(TableName.Users), db.ref("id").withSchema(TableName.Users).as("userId"), db.ref("publicKey").withSchema(TableName.UserEncryptionKey) - ); + ) + .where({ ghost: false }); // MAKE SURE USER IS NOT A GHOST USER return members.map(({ email, firstName, lastName, userId, publicKey, ...data }) => ({ ...data, user: { email, firstName, lastName, id: userId, publicKey } @@ -86,6 +87,43 @@ export const orgDALFactory = (db: TDbClient) => { } }; + const findOrgGhostUser = async (orgId: string) => { + try { + const [member] = await db(TableName.OrgMembership) + .where({ orgId }) + .join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`) + .leftJoin(TableName.UserEncryptionKey, `${TableName.UserEncryptionKey}.userId`, `${TableName.Users}.id`) + .select( + db.ref("id").withSchema(TableName.OrgMembership), + db.ref("orgId").withSchema(TableName.OrgMembership), + db.ref("role").withSchema(TableName.OrgMembership), + db.ref("roleId").withSchema(TableName.OrgMembership), + db.ref("status").withSchema(TableName.OrgMembership), + db.ref("email").withSchema(TableName.Users), + db.ref("id").withSchema(TableName.Users).as("userId"), + db.ref("publicKey").withSchema(TableName.UserEncryptionKey) + ) + .where({ ghost: true }); + return member; + } catch (error) { + return null; + } + }; + + const ghostUserExists = async (orgId: string) => { + try { + const [member] = await db(TableName.OrgMembership) + .where({ orgId }) + .join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`) + .leftJoin(TableName.UserEncryptionKey, `${TableName.UserEncryptionKey}.userId`, `${TableName.Users}.id`) + .select(db.ref("id").withSchema(TableName.Users).as("userId")) + .where({ ghost: true }); + return !!member; + } catch (error) { + return false; + } + }; + const create = async (dto: TOrganizationsInsert, tx?: Knex) => { try { const [organization] = await (tx || db)(TableName.Organization).insert(dto).returning("*"); @@ -191,6 +229,8 @@ export const orgDALFactory = (db: TDbClient) => { findAllOrgMembers, findOrgById, findAllOrgsByUserId, + ghostUserExists, + findOrgGhostUser, create, updateById, deleteById, diff --git a/backend/src/services/org/org-service.ts b/backend/src/services/org/org-service.ts index 1d78ef0b9..3b20b9a14 100644 --- a/backend/src/services/org/org-service.ts +++ b/backend/src/services/org/org-service.ts @@ -1,5 +1,6 @@ import { ForbiddenError } from "@casl/ability"; import slugify from "@sindresorhus/slugify"; +import crypto from "crypto"; import jwt from "jsonwebtoken"; import { OrgMembershipRole, OrgMembershipStatus } from "@app/db/schemas"; @@ -11,6 +12,7 @@ import { TSamlConfigDALFactory } from "@app/ee/services/saml-config/saml-config- import { getConfig } from "@app/lib/config/env"; import { generateAsymmetricKeyPair } from "@app/lib/crypto"; import { generateSymmetricKey, infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; +import { generateUserSrpKeys } from "@app/lib/crypto/srp"; import { BadRequestError, UnauthorizedError } from "@app/lib/errors"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { isDisposableEmail } from "@app/lib/validator"; @@ -118,6 +120,49 @@ export const orgServiceFactory = ({ return workspaces.filter((workspace) => organizationWorkspaceIds.has(workspace.id)); }; + const addGhostUser = async (orgId: string) => { + const email = `ghost@${orgId}.com`; + const password = crypto.randomBytes(128).toString("hex"); + + const user = await userDAL.create({ + ghost: true, + authMethods: [AuthMethod.EMAIL], + email: `ghost@${orgId}.com`, + isAccepted: true + }); + + const encKeys = await generateUserSrpKeys(email, password); + + await userDAL.upsertUserEncryptionKey(user.id, { + encryptionVersion: 2, + protectedKey: encKeys.protectedKey, + protectedKeyIV: encKeys.protectedKeyIV, + protectedKeyTag: encKeys.protectedKeyTag, + publicKey: encKeys.publicKey, + encryptedPrivateKey: encKeys.encryptedPrivateKey, + iv: encKeys.encryptedPrivateKeyIV, + tag: encKeys.encryptedPrivateKeyTag, + salt: encKeys.salt, + verifier: encKeys.verifier + }); + + const createMembershipData = { + orgId, + userId: user.id, + role: OrgMembershipRole.Admin, + status: OrgMembershipStatus.Accepted + }; + + console.log("createMembershipData", createMembershipData); + + await orgDAL.createMembership(createMembershipData); + + return { + user, + keys: encKeys + }; + }; + /* * Update organization details * */ @@ -338,7 +383,8 @@ export const orgServiceFactory = ({ { email: inviteeEmail, isAccepted: false, - authMethods: [AuthMethod.EMAIL] + authMethods: [AuthMethod.EMAIL], + ghost: false }, tx ); @@ -491,6 +537,7 @@ export const orgServiceFactory = ({ deleteOrganizationById, deleteOrgMembership, findAllWorkspaces, + addGhostUser, updateOrgMembership, // incident contacts findIncidentContacts, diff --git a/backend/src/services/project-bot/project-bot-service.ts b/backend/src/services/project-bot/project-bot-service.ts index 5ead160f2..a9514c1e3 100644 --- a/backend/src/services/project-bot/project-bot-service.ts +++ b/backend/src/services/project-bot/project-bot-service.ts @@ -1,4 +1,5 @@ import { ForbiddenError } from "@casl/ability"; +import { Knex } from "knex"; import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; @@ -6,17 +7,16 @@ import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services import { getConfig } from "@app/lib/config/env"; import { decryptAsymmetric, - decryptSymmetric, - decryptSymmetric128BitHexKeyUTF8, encryptSymmetric, encryptSymmetric128BitHexKeyUTF8, generateAsymmetricKeyPair } from "@app/lib/crypto"; +import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; import { BadRequestError } from "@app/lib/errors"; import { TProjectPermission } from "@app/lib/types"; import { TProjectBotDALFactory } from "./project-bot-dal"; -import { TSetActiveStateDTO } from "./project-bot-types"; +import { TGetPrivateKeyDTO, TSetActiveStateDTO } from "./project-bot-types"; type TProjectBotServiceFactoryDep = { permissionService: Pick; @@ -26,48 +26,35 @@ type TProjectBotServiceFactoryDep = { export type TProjectBotServiceFactory = ReturnType; export const projectBotServiceFactory = ({ projectBotDAL, permissionService }: TProjectBotServiceFactoryDep) => { - const getBotKey = async (projectId: string) => { - const appCfg = getConfig(); - const encryptionKey = appCfg.ENCRYPTION_KEY; - const rootEncryptionKey = appCfg.ROOT_ENCRYPTION_KEY; + const getBotPrivateKey = async ({ encoding, nonce, tag, encryptedPrivateKey }: TGetPrivateKeyDTO) => + infisicalSymmetricDecrypt({ + keyEncoding: encoding, + iv: nonce, + tag, + ciphertext: encryptedPrivateKey + }); + const getBotKey = async (projectId: string) => { const bot = await projectBotDAL.findOne({ projectId }); if (!bot) throw new BadRequestError({ message: "failed to find bot key" }); if (!bot.isActive) throw new BadRequestError({ message: "Bot is not active" }); if (!bot.encryptedProjectKeyNonce || !bot.encryptedProjectKey) throw new BadRequestError({ message: "Encryption key missing" }); - if (rootEncryptionKey && (bot.keyEncoding as SecretKeyEncoding) === SecretKeyEncoding.BASE64) { - const privateKeyBot = decryptSymmetric({ - iv: bot.iv, - tag: bot.tag, - ciphertext: bot.encryptedPrivateKey, - key: rootEncryptionKey - }); - return decryptAsymmetric({ - ciphertext: bot.encryptedProjectKey, - privateKey: privateKeyBot, - nonce: bot.encryptedProjectKeyNonce, - publicKey: bot.sender.publicKey - }); - } - if (encryptionKey && (bot.keyEncoding as SecretKeyEncoding) === SecretKeyEncoding.UTF8) { - const privateKeyBot = decryptSymmetric128BitHexKeyUTF8({ - iv: bot.iv, - tag: bot.tag, - ciphertext: bot.encryptedPrivateKey, - key: encryptionKey - }); - return decryptAsymmetric({ - ciphertext: bot.encryptedProjectKey, - privateKey: privateKeyBot, - nonce: bot.encryptedProjectKeyNonce, - publicKey: bot.sender.publicKey - }); - } + const privateKeyBot = await getBotPrivateKey({ + nonce: bot.iv, + tag: bot.tag, + encryptedPrivateKey: bot.encryptedPrivateKey, + encoding: bot.keyEncoding as SecretKeyEncoding + }); - throw new BadRequestError({ - message: "Failed to obtain bot copy of workspace key needed for operation" + console.log("privateKeyBot", privateKeyBot); + + return decryptAsymmetric({ + ciphertext: bot.encryptedProjectKey, + privateKey: privateKeyBot, + nonce: bot.encryptedProjectKeyNonce, + publicKey: bot.sender.publicKey }); }; @@ -131,12 +118,16 @@ export const projectBotServiceFactory = ({ projectBotDAL, permissionService }: T if (!botKey?.nonce || !botKey?.encryptedKey) { throw new BadRequestError({ message: "Failed to set bot active - missing bot key" }); } - const doc = await projectBotDAL.updateById(botId, { - isActive: true, - encryptedProjectKey: botKey.encryptedKey, - encryptedProjectKeyNonce: botKey.nonce, - senderId: actorId - }); + const doc = await projectBotDAL.updateById( + botId, + { + isActive: true, + encryptedProjectKey: botKey.encryptedKey, + encryptedProjectKeyNonce: botKey.nonce, + senderId: actorId + }, + tx + ); if (!doc) throw new BadRequestError({ message: "Failed to update bot active state" }); return doc; } @@ -153,6 +144,7 @@ export const projectBotServiceFactory = ({ projectBotDAL, permissionService }: T return { findBotByProjectId, setBotActiveState, + getBotPrivateKey, getBotKey }; }; diff --git a/backend/src/services/project-bot/project-bot-types.ts b/backend/src/services/project-bot/project-bot-types.ts index 94a2943eb..3d4f400de 100644 --- a/backend/src/services/project-bot/project-bot-types.ts +++ b/backend/src/services/project-bot/project-bot-types.ts @@ -1,3 +1,4 @@ +import { SecretKeyEncoding } from "@app/db/schemas"; import { TProjectPermission } from "@app/lib/types"; export type TSetActiveStateDTO = { @@ -8,3 +9,10 @@ export type TSetActiveStateDTO = { }; botId: string; } & Omit; + +export type TGetPrivateKeyDTO = { + encoding: SecretKeyEncoding; + nonce: string; + tag: string; + encryptedPrivateKey: string; +}; diff --git a/backend/src/services/project-key/project-key-service.ts b/backend/src/services/project-key/project-key-service.ts index fa77760a4..81b589bd0 100644 --- a/backend/src/services/project-key/project-key-service.ts +++ b/backend/src/services/project-key/project-key-service.ts @@ -1,4 +1,5 @@ import { ForbiddenError } from "@casl/ability"; +import { Knex } from "knex"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; @@ -43,7 +44,7 @@ export const projectKeyServiceFactory = ({ name: "Upload project keys" }); - await projectKeyDAL.create({ projectId, receiverId, encryptedKey, nonce, senderId: actorId }); + await projectKeyDAL.create({ projectId, receiverId, encryptedKey, nonce, senderId: actorId }, tx); }; const getLatestProjectKey = async ({ actorId, projectId, actor, actorOrgId }: TGetLatestProjectKeyDTO) => { diff --git a/backend/src/services/project-membership/project-membership-dal.ts b/backend/src/services/project-membership/project-membership-dal.ts index 22b9937a9..c6d8249e2 100644 --- a/backend/src/services/project-membership/project-membership-dal.ts +++ b/backend/src/services/project-membership/project-membership-dal.ts @@ -24,15 +24,16 @@ export const projectMembershipDALFactory = (db: TDbClient) => { db.ref("projectId").withSchema(TableName.ProjectMembership), db.ref("role").withSchema(TableName.ProjectMembership), db.ref("roleId").withSchema(TableName.ProjectMembership), + db.ref("ghost").withSchema(TableName.Users), db.ref("email").withSchema(TableName.Users), db.ref("publicKey").withSchema(TableName.UserEncryptionKey), db.ref("firstName").withSchema(TableName.Users), db.ref("lastName").withSchema(TableName.Users), db.ref("id").withSchema(TableName.Users).as("userId") ); - return members.map(({ email, firstName, lastName, publicKey, ...data }) => ({ + return members.map(({ email, firstName, lastName, publicKey, ghost, ...data }) => ({ ...data, - user: { email, firstName, lastName, id: data.userId, publicKey } + user: { email, firstName, lastName, id: data.userId, publicKey, ghost } })); } catch (error) { throw new DatabaseError({ error, name: "Find all project members" }); diff --git a/backend/src/services/project-membership/project-membership-service.ts b/backend/src/services/project-membership/project-membership-service.ts index 6b2123e5e..477891bf8 100644 --- a/backend/src/services/project-membership/project-membership-service.ts +++ b/backend/src/services/project-membership/project-membership-service.ts @@ -134,11 +134,16 @@ export const projectMembershipServiceFactory = ({ await projectMembershipDAL.transaction(async (tx) => { await projectMembershipDAL.insertMany( - orgMembers.map(({ userId }) => ({ - projectId, - userId: userId as string, - role: ProjectMembershipRole.Member - })), + orgMembers.map(({ userId, id: membershipId }) => { + const role = + members.find((i) => i.orgMembershipId === membershipId)?.projectRole || ProjectMembershipRole.Member; + + return { + projectId, + userId: userId as string, + role + }; + }), tx ); const encKeyGroupByOrgMembId = groupBy(members, (i) => i.orgMembershipId); diff --git a/backend/src/services/project-membership/project-membership-types.ts b/backend/src/services/project-membership/project-membership-types.ts index 6ee1d2965..6fe2977ee 100644 --- a/backend/src/services/project-membership/project-membership-types.ts +++ b/backend/src/services/project-membership/project-membership-types.ts @@ -1,3 +1,4 @@ +import { ProjectMembershipRole } from "@app/db/schemas"; import { TProjectPermission } from "@app/lib/types"; export type TGetProjectMembershipDTO = TProjectPermission; @@ -20,5 +21,6 @@ export type TAddUsersToWorkspaceDTO = { orgMembershipId: string; workspaceEncryptedKey: string; workspaceEncryptedNonce: string; + projectRole: ProjectMembershipRole; }[]; } & TProjectPermission; diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index 387c9bd78..f56c40fa2 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -66,10 +66,15 @@ export const projectServiceFactory = ({ const newProject = projectDAL.transaction(async (tx) => { const project = await projectDAL.create( - { name: workspaceName, orgId, slug: slugify(`${workspaceName}-${alphaNumericNanoId(4)}`) }, + { + name: workspaceName, + orgId, + slug: slugify(`${workspaceName}-${alphaNumericNanoId(4)}`), + e2ee: false + }, tx ); - // set user as admin member for proeject + // set user as admin member for project await projectMembershipDAL.create( { userId: actorId, @@ -78,6 +83,7 @@ export const projectServiceFactory = ({ }, tx ); + // generate the blind index for project await secretBlindIndexDAL.create( { diff --git a/frontend/src/components/utilities/cryptography/crypto.ts b/frontend/src/components/utilities/cryptography/crypto.ts index 4e2d981a7..0f0e4af22 100644 --- a/frontend/src/components/utilities/cryptography/crypto.ts +++ b/frontend/src/components/utilities/cryptography/crypto.ts @@ -13,12 +13,12 @@ nacl.util = require("tweetnacl-util"); */ const generateKeyPair = () => { const pair = nacl.box.keyPair(); - - return ({ - publicKey: nacl.util.encodeBase64(pair.publicKey), - privateKey: nacl.util.encodeBase64(pair.secretKey) - }); -} + + return { + publicKey: nacl.util.encodeBase64(pair.publicKey), + privateKey: nacl.util.encodeBase64(pair.secretKey) + }; +}; type EncryptAsymmetricProps = { plaintext: string; @@ -29,27 +29,19 @@ type EncryptAsymmetricProps = { /** * Verify that private key [privateKey] is the one that corresponds to * the public key [publicKey] - * @param {Object} + * @param {Object} * @param {String} - base64-encoded Nacl private key * @param {String} - base64-encoded Nacl public key */ -const verifyPrivateKey = ({ - privateKey, - publicKey -}: { - privateKey: string; - publicKey: string; -}) => { +const verifyPrivateKey = ({ privateKey, publicKey }: { privateKey: string; publicKey: string }) => { const derivedPublicKey = nacl.util.encodeBase64( - nacl.box.keyPair.fromSecretKey( - nacl.util.decodeBase64(privateKey) - ).publicKey + nacl.box.keyPair.fromSecretKey(nacl.util.decodeBase64(privateKey)).publicKey ); - + if (derivedPublicKey !== publicKey) { throw new Error("Failed to verify private key"); } -} +}; /** * Derive a key from password [password] and salt [salt] using Argon2id @@ -229,7 +221,8 @@ export { decryptAssymmetric, decryptSymmetric, deriveArgonKey, - encryptAssymmetric, + encryptAssymmetric, encryptSymmetric, generateKeyPair, - verifyPrivateKey}; + verifyPrivateKey +}; diff --git a/frontend/src/hooks/api/workspace/queries.tsx b/frontend/src/hooks/api/workspace/queries.tsx index f47a77537..89a436df2 100644 --- a/frontend/src/hooks/api/workspace/queries.tsx +++ b/frontend/src/hooks/api/workspace/queries.tsx @@ -158,19 +158,21 @@ export const useGetWorkspaceIntegrations = (workspaceId: string) => export const createWorkspace = ({ organizationId, - workspaceName + projectName, + inviteAllOrgMembers }: CreateWorkspaceDTO): Promise<{ data: { workspace: Workspace } }> => { - return apiRequest.post("/api/v1/workspace", { workspaceName, organizationId }); + return apiRequest.post("/api/v3/projects", { projectName, inviteAllOrgMembers, organizationId }); }; export const useCreateWorkspace = () => { const queryClient = useQueryClient(); return useMutation<{ data: { workspace: Workspace } }, {}, CreateWorkspaceDTO>({ - mutationFn: async ({ organizationId, workspaceName }) => + mutationFn: async ({ organizationId, projectName, inviteAllOrgMembers }) => createWorkspace({ organizationId, - workspaceName + projectName, + inviteAllOrgMembers }), onSuccess: () => { queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace); diff --git a/frontend/src/hooks/api/workspace/types.ts b/frontend/src/hooks/api/workspace/types.ts index 10d222a2a..3538e761b 100644 --- a/frontend/src/hooks/api/workspace/types.ts +++ b/frontend/src/hooks/api/workspace/types.ts @@ -25,7 +25,8 @@ export type NameWorkspaceSecretsDTO = { // mutation dto export type CreateWorkspaceDTO = { - workspaceName: string; + projectName: string; + inviteAllOrgMembers: boolean; organizationId: string; }; diff --git a/frontend/src/pages/org/[id]/overview/index.tsx b/frontend/src/pages/org/[id]/overview/index.tsx index 43d36d06b..ae1bf4b3d 100644 --- a/frontend/src/pages/org/[id]/overview/index.tsx +++ b/frontend/src/pages/org/[id]/overview/index.tsx @@ -1,7 +1,5 @@ // REFACTOR(akhilmhdh): This file needs to be split into multiple components too complex -import crypto from "crypto"; - import { useEffect, useState } from "react"; import { Controller, useForm } from "react-hook-form"; import { useTranslation } from "react-i18next"; @@ -23,7 +21,7 @@ import { faNetworkWired, faPlug, faPlus, - faUserPlus, + faUserPlus } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { yupResolver } from "@hookform/resolvers/yup"; @@ -33,7 +31,6 @@ import * as yup from "yup"; import { useNotificationContext } from "@app/components/context/Notifications/NotificationProvider"; import { OrgPermissionCan } from "@app/components/permissions"; import onboardingCheck from "@app/components/utilities/checks/OnboardingCheck"; -import { encryptAssymmetric } from "@app/components/utilities/cryptography/crypto"; import { Button, Checkbox, @@ -53,13 +50,12 @@ import { } from "@app/context"; import { withPermission } from "@app/hoc"; import { - fetchOrgUsers, - useAddUserToWs, + // fetchOrgUsers, + // useAddUserToWs, useCreateWorkspace, useRegisterUserAction, - useUploadWsKey } from "@app/hooks/api"; -import { fetchUserWsKey } from "@app/hooks/api/keys/queries"; +// import { fetchUserWsKey } from "@app/hooks/api/keys/queries"; import { useFetchServerStatus } from "@app/hooks/api/serverDetails"; import { usePopUp } from "@app/hooks/usePopUp"; @@ -475,7 +471,7 @@ const OrganizationPage = withPermission( const currentOrg = String(router.query.id); const orgWorkspaces = workspaces?.filter((workspace) => workspace.orgId === currentOrg) || []; const { createNotification } = useNotificationContext(); - const addWsUser = useAddUserToWs(); + // const addWsUser = useAddUserToWs(); const { popUp, handlePopUpOpen, handlePopUpClose, handlePopUpToggle } = usePopUp([ "addNewWs", @@ -497,7 +493,6 @@ const OrganizationPage = withPermission( const [searchFilter, setSearchFilter] = useState(""); const createWs = useCreateWorkspace(); const { user } = useUser(); - const uploadWsKey = useUploadWsKey(); const { data: serverDetails } = useFetchServerStatus(); const onCreateProject = async ({ name, addMembers }: TAddProjectFormData) => { @@ -510,45 +505,31 @@ const OrganizationPage = withPermission( } } = await createWs.mutateAsync({ organizationId: currentOrg, - workspaceName: name - }); - - const randomBytes = crypto.randomBytes(16).toString("hex"); - const PRIVATE_KEY = String(localStorage.getItem("PRIVATE_KEY")); - const { ciphertext, nonce } = encryptAssymmetric({ - plaintext: randomBytes, - publicKey: user.publicKey, - privateKey: PRIVATE_KEY - }); - - await uploadWsKey.mutateAsync({ - encryptedKey: ciphertext, - nonce, - userId: user?.id, - workspaceId: newWorkspaceId + inviteAllOrgMembers: addMembers, + projectName: name }); + /* if (addMembers) { // not using hooks because need at this point only const orgUsers = await fetchOrgUsers(currentOrg); const decryptKey = await fetchUserWsKey(newWorkspaceId); - const members = orgUsers - .filter( - ({ status, user: orgUser }) => status === "accepted" && user.email !== orgUser.email - ) - .map(({ user: orgUser, id: orgMembershipId }) => ({ - userPublicKey: orgUser.publicKey, - orgMembershipId - })); - if (members.length) { - await addWsUser.mutateAsync({ - workspaceId: newWorkspaceId, - decryptKey, - userPrivateKey: PRIVATE_KEY, - members - }); - } + + await addWsUser.mutateAsync({ + workspaceId: newWorkspaceId, + decryptKey, + userPrivateKey: PRIVATE_KEY, + members: orgUsers + .filter( + ({ status, user: orgUser }) => status === "accepted" && user.email !== orgUser.email + ) + .map(({ user: orgUser, id: orgMembershipId }) => ({ + userPublicKey: orgUser.publicKey, + orgMembershipId + })) + }); } + */ createNotification({ text: "Workspace created", type: "success" }); handlePopUpClose("addNewWs"); router.push(`/project/${newWorkspaceId}/secrets/overview`); @@ -735,7 +716,7 @@ const OrganizationPage = withPermission( new Date().getTime() - new Date(user?.createdAt).getTime() < 30 * 24 * 60 * 60 * 1000 ) && ( -
+

Onboarding Guide

Date: Tue, 6 Feb 2024 03:17:33 +0400 Subject: [PATCH 007/157] Wired most of the frontend to support ghost users --- frontend/src/hooks/api/types.ts | 2 +- frontend/src/hooks/api/users/index.tsx | 2 +- frontend/src/hooks/api/users/mutation.tsx | 22 ++++++++-- frontend/src/hooks/api/users/types.ts | 7 +++- frontend/src/hooks/api/workspace/queries.tsx | 22 +++++----- frontend/src/hooks/api/workspace/types.ts | 2 +- frontend/src/layouts/AppLayout/AppLayout.tsx | 8 ++-- .../src/pages/org/[id]/overview/index.tsx | 41 +++++++++---------- .../MemberListTab/MemberListTab.tsx | 26 ++++++++---- 9 files changed, 80 insertions(+), 52 deletions(-) diff --git a/frontend/src/hooks/api/types.ts b/frontend/src/hooks/api/types.ts index 04d215898..6748db6ac 100644 --- a/frontend/src/hooks/api/types.ts +++ b/frontend/src/hooks/api/types.ts @@ -22,7 +22,7 @@ export * from "./secrets/types"; export type { CreateServiceTokenDTO, ServiceToken } from "./serviceTokens/types"; export type { SubscriptionPlan } from "./subscriptions/types"; export type { WsTag } from "./tags/types"; -export type { AddUserToWsDTO, OrgUser, TWorkspaceUser, User, UserEnc } from "./users/types"; +export type { AddUserToWsDTOE2EE, OrgUser, TWorkspaceUser, User, UserEnc } from "./users/types"; export type { TWebhook } from "./webhooks/types"; export type { CreateEnvironmentDTO, diff --git a/frontend/src/hooks/api/users/index.tsx b/frontend/src/hooks/api/users/index.tsx index 75dd36cc5..9c51948f2 100644 --- a/frontend/src/hooks/api/users/index.tsx +++ b/frontend/src/hooks/api/users/index.tsx @@ -1,4 +1,4 @@ -export { useAddUserToWs } from "./mutation"; +export { useAddUserToWsE2EE, useAddUserToWsNonE2EE } from "./mutation"; export { fetchOrgUsers, useAddUserToOrg, diff --git a/frontend/src/hooks/api/users/mutation.tsx b/frontend/src/hooks/api/users/mutation.tsx index 03276c2ef..f7c0e1558 100644 --- a/frontend/src/hooks/api/users/mutation.tsx +++ b/frontend/src/hooks/api/users/mutation.tsx @@ -7,12 +7,12 @@ import { import { apiRequest } from "@app/config/request"; import { workspaceKeys } from "../workspace/queries"; -import { AddUserToWsDTO } from "./types"; +import { AddUserToWsDTOE2EE, AddUserToWsDTONonE2EE } from "./types"; -export const useAddUserToWs = () => { +export const useAddUserToWsE2EE = () => { const queryClient = useQueryClient(); - return useMutation<{}, {}, AddUserToWsDTO>({ + return useMutation<{}, {}, AddUserToWsDTOE2EE>({ mutationFn: async ({ workspaceId, members, decryptKey, userPrivateKey }) => { // assymmetrically decrypt symmetric key with local private key const key = decryptAssymmetric({ @@ -45,3 +45,19 @@ export const useAddUserToWs = () => { } }); }; + +export const useAddUserToWsNonE2EE = () => { + const queryClient = useQueryClient(); + + return useMutation<{}, {}, AddUserToWsDTONonE2EE>({ + mutationFn: async ({ projectId, emails }) => { + const { data } = await apiRequest.post(`/api/v3/projects/${projectId}/memberships`, { + emails + }); + return data; + }, + onSuccess: (_, { projectId }) => { + queryClient.invalidateQueries(workspaceKeys.getWorkspaceUsers(projectId)); + } + }); +}; diff --git a/frontend/src/hooks/api/users/types.ts b/frontend/src/hooks/api/users/types.ts index ebddefb14..366ee93bf 100644 --- a/frontend/src/hooks/api/users/types.ts +++ b/frontend/src/hooks/api/users/types.ts @@ -63,7 +63,7 @@ export type TProjectMembership = { export type TWorkspaceUser = OrgUser; -export type AddUserToWsDTO = { +export type AddUserToWsDTOE2EE = { workspaceId: string; decryptKey: UserWsKeyPair; userPrivateKey: string; @@ -73,6 +73,11 @@ export type AddUserToWsDTO = { }[]; }; +export type AddUserToWsDTONonE2EE = { + projectId: string; + emails: string[]; +}; + export type UpdateOrgUserRoleDTO = { organizationId: string; membershipId: string; diff --git a/frontend/src/hooks/api/workspace/queries.tsx b/frontend/src/hooks/api/workspace/queries.tsx index 89a436df2..38879de9d 100644 --- a/frontend/src/hooks/api/workspace/queries.tsx +++ b/frontend/src/hooks/api/workspace/queries.tsx @@ -158,21 +158,19 @@ export const useGetWorkspaceIntegrations = (workspaceId: string) => export const createWorkspace = ({ organizationId, - projectName, - inviteAllOrgMembers -}: CreateWorkspaceDTO): Promise<{ data: { workspace: Workspace } }> => { - return apiRequest.post("/api/v3/projects", { projectName, inviteAllOrgMembers, organizationId }); + projectName +}: CreateWorkspaceDTO): Promise<{ data: { project: Workspace } }> => { + return apiRequest.post("/api/v3/projects", { projectName, organizationId }); }; export const useCreateWorkspace = () => { const queryClient = useQueryClient(); - return useMutation<{ data: { workspace: Workspace } }, {}, CreateWorkspaceDTO>({ - mutationFn: async ({ organizationId, projectName, inviteAllOrgMembers }) => + return useMutation<{ data: { project: Workspace } }, {}, CreateWorkspaceDTO>({ + mutationFn: async ({ organizationId, projectName }) => createWorkspace({ organizationId, - projectName, - inviteAllOrgMembers + projectName }), onSuccess: () => { queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace); @@ -287,11 +285,13 @@ export const useAddUserToWorkspace = () => { return useMutation({ mutationFn: async ({ email, workspaceId }: { email: string; workspaceId: string }) => { const { - data: { invitee, latestKey } - } = await apiRequest.post(`/api/v1/workspace/${workspaceId}/invite-signup`, { email }); + data: { invitees, latestKey } + } = await apiRequest.post(`/api/v1/workspace/${workspaceId}/invite-signup`, { + emails: [email] + }); return { - invitee, + invitees, latestKey }; }, diff --git a/frontend/src/hooks/api/workspace/types.ts b/frontend/src/hooks/api/workspace/types.ts index 3538e761b..8b825d062 100644 --- a/frontend/src/hooks/api/workspace/types.ts +++ b/frontend/src/hooks/api/workspace/types.ts @@ -3,6 +3,7 @@ export type Workspace = { id: string; name: string; orgId: string; + e2ee: boolean; autoCapitalization: boolean; environments: WorkspaceEnv[]; }; @@ -26,7 +27,6 @@ export type NameWorkspaceSecretsDTO = { // mutation dto export type CreateWorkspaceDTO = { projectName: string; - inviteAllOrgMembers: boolean; organizationId: string; }; diff --git a/frontend/src/layouts/AppLayout/AppLayout.tsx b/frontend/src/layouts/AppLayout/AppLayout.tsx index d175afe48..6f9d80e6b 100644 --- a/frontend/src/layouts/AppLayout/AppLayout.tsx +++ b/frontend/src/layouts/AppLayout/AppLayout.tsx @@ -62,7 +62,7 @@ import { import { usePopUp } from "@app/hooks"; import { fetchOrgUsers, - useAddUserToWs, + useAddUserToWsE2EE, useCreateWorkspace, useGetOrgTrialUrl, useGetSecretApprovalRequestCount, @@ -130,7 +130,7 @@ export const AppLayout = ({ children }: LayoutProps) => { const createWs = useCreateWorkspace(); const uploadWsKey = useUploadWsKey(); - const addWsUser = useAddUserToWs(); + const addWsUser = useAddUserToWsE2EE(); const infisicalPlatformVersion = process.env.NEXT_PUBLIC_INFISICAL_PLATFORM_VERSION; const { popUp, handlePopUpOpen, handlePopUpClose, handlePopUpToggle } = usePopUp([ @@ -224,11 +224,11 @@ export const AppLayout = ({ children }: LayoutProps) => { try { const { data: { - workspace: { id: newWorkspaceId } + project: { id: newWorkspaceId } } } = await createWs.mutateAsync({ organizationId: currentOrg?.id, - workspaceName: name + projectName: name }); const randomBytes = crypto.randomBytes(16).toString("hex"); diff --git a/frontend/src/pages/org/[id]/overview/index.tsx b/frontend/src/pages/org/[id]/overview/index.tsx index ae1bf4b3d..56cf50af6 100644 --- a/frontend/src/pages/org/[id]/overview/index.tsx +++ b/frontend/src/pages/org/[id]/overview/index.tsx @@ -53,7 +53,7 @@ import { // fetchOrgUsers, // useAddUserToWs, useCreateWorkspace, - useRegisterUserAction, + useRegisterUserAction } from "@app/hooks/api"; // import { fetchUserWsKey } from "@app/hooks/api/keys/queries"; import { useFetchServerStatus } from "@app/hooks/api/serverDetails"; @@ -471,7 +471,7 @@ const OrganizationPage = withPermission( const currentOrg = String(router.query.id); const orgWorkspaces = workspaces?.filter((workspace) => workspace.orgId === currentOrg) || []; const { createNotification } = useNotificationContext(); - // const addWsUser = useAddUserToWs(); + // const addWsUser = useAddUserToWs(); const { popUp, handlePopUpOpen, handlePopUpClose, handlePopUpToggle } = usePopUp([ "addNewWs", @@ -501,35 +501,32 @@ const OrganizationPage = withPermission( try { const { data: { - workspace: { id: newWorkspaceId } + project: { id: newWorkspaceId } } } = await createWs.mutateAsync({ organizationId: currentOrg, - inviteAllOrgMembers: addMembers, projectName: name }); - /* if (addMembers) { // not using hooks because need at this point only - const orgUsers = await fetchOrgUsers(currentOrg); - const decryptKey = await fetchUserWsKey(newWorkspaceId); - - await addWsUser.mutateAsync({ - workspaceId: newWorkspaceId, - decryptKey, - userPrivateKey: PRIVATE_KEY, - members: orgUsers - .filter( - ({ status, user: orgUser }) => status === "accepted" && user.email !== orgUser.email - ) - .map(({ user: orgUser, id: orgMembershipId }) => ({ - userPublicKey: orgUser.publicKey, - orgMembershipId - })) - }); + // const orgUsers = await fetchOrgUsers(currentOrg); + // const decryptKey = await fetchUserWsKey(newWorkspaceId); + // await addWsUser.mutateAsync({ + // workspaceId: newWorkspaceId, + // decryptKey, + // userPrivateKey: PRIVATE_KEY, + // members: orgUsers + // .filter( + // ({ status, user: orgUser }) => status === "accepted" && user.email !== orgUser.email + // ) + // .map(({ user: orgUser, id: orgMembershipId }) => ({ + // userPublicKey: orgUser.publicKey, + // orgMembershipId + // })) + // }); } - */ + createNotification({ text: "Workspace created", type: "success" }); handlePopUpClose("addNewWs"); router.push(`/project/${newWorkspaceId}/secrets/overview`); diff --git a/frontend/src/views/Project/MembersPage/components/MemberListTab/MemberListTab.tsx b/frontend/src/views/Project/MembersPage/components/MemberListTab/MemberListTab.tsx index 76f97741d..838ccce19 100644 --- a/frontend/src/views/Project/MembersPage/components/MemberListTab/MemberListTab.tsx +++ b/frontend/src/views/Project/MembersPage/components/MemberListTab/MemberListTab.tsx @@ -44,7 +44,8 @@ import { } from "@app/context"; import { usePopUp } from "@app/hooks"; import { - useAddUserToWs, + useAddUserToWsE2EE, + useAddUserToWsNonE2EE, useDeleteUserFromWorkspace, useGetOrgUsers, useGetProjectRoles, @@ -95,12 +96,14 @@ export const MemberListTab = () => { formState: { isSubmitting } } = useForm({ resolver: zodResolver(addMemberFormSchema) }); - const { mutateAsync: addUserToWorkspace } = useAddUserToWs(); + const { mutateAsync: addUserToWorkspace } = useAddUserToWsE2EE(); + const { mutateAsync: addUserToWorkspaceNonE2EE } = useAddUserToWsNonE2EE(); const { mutateAsync: uploadWsKey } = useUploadWsKey(); const { mutateAsync: removeUserFromWorkspace } = useDeleteUserFromWorkspace(); const { mutateAsync: updateUserWorkspaceRole } = useUpdateUserWorkspaceRole(); const onAddMember = async ({ orgMembershipId }: TAddMemberForm) => { + if (!currentWorkspace) return; if (!currentOrg?.id) return; // TODO(akhilmhdh): Move to memory storage const userPrivateKey = localStorage.getItem("PRIVATE_KEY"); @@ -114,12 +117,19 @@ export const MemberListTab = () => { if (!orgUser) return; try { - await addUserToWorkspace({ - workspaceId, - userPrivateKey, - decryptKey: wsKey, - members: [{ orgMembershipId, userPublicKey: orgUser.user.publicKey }] - }); + if (currentWorkspace.e2ee) { + await addUserToWorkspace({ + workspaceId, + userPrivateKey, + decryptKey: wsKey, + members: [{ orgMembershipId, userPublicKey: orgUser.user.publicKey }] + }); + } else { + await addUserToWorkspaceNonE2EE({ + projectId: workspaceId, + emails: [orgUser.user.email] + }); + } createNotification({ text: "Successfully added user to the project", type: "success" From 3ad0382cb0e7fb2769b21fe1f7c536fd56f49838 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 6 Feb 2024 03:20:20 +0400 Subject: [PATCH 008/157] Added DAL methods --- backend/src/services/org/org-dal.ts | 34 +++++++++++++++++++ .../services/project-bot/project-bot-dal.ts | 16 ++++++++- .../project-membership-dal.ts | 20 +++++++++-- backend/src/services/user/user-dal.ts | 12 +++++++ 4 files changed, 79 insertions(+), 3 deletions(-) diff --git a/backend/src/services/org/org-dal.ts b/backend/src/services/org/org-dal.ts index e708cefdd..20678b680 100644 --- a/backend/src/services/org/org-dal.ts +++ b/backend/src/services/org/org-dal.ts @@ -87,6 +87,39 @@ export const orgDALFactory = (db: TDbClient) => { } }; + const findOrgMembersByEmail = async (orgId: string, emails: string[]) => { + try { + const members = await db(TableName.OrgMembership) + .where({ orgId }) + .join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`) + .leftJoin( + TableName.UserEncryptionKey, + `${TableName.UserEncryptionKey}.userId`, + `${TableName.Users}.id` + ) + .select( + db.ref("id").withSchema(TableName.OrgMembership), + db.ref("inviteEmail").withSchema(TableName.OrgMembership), + db.ref("orgId").withSchema(TableName.OrgMembership), + db.ref("role").withSchema(TableName.OrgMembership), + db.ref("roleId").withSchema(TableName.OrgMembership), + db.ref("status").withSchema(TableName.OrgMembership), + db.ref("email").withSchema(TableName.Users), + db.ref("firstName").withSchema(TableName.Users), + db.ref("lastName").withSchema(TableName.Users), + db.ref("id").withSchema(TableName.Users).as("userId"), + db.ref("publicKey").withSchema(TableName.UserEncryptionKey) + ) + .whereIn("email", emails); + return members.map(({ email, firstName, lastName, userId, publicKey, ...data }) => ({ + ...data, + user: { email, firstName, lastName, id: userId, publicKey } + })); + } catch (error) { + throw new DatabaseError({ error, name: "Find all org members" }); + } + }; + const findOrgGhostUser = async (orgId: string) => { try { const [member] = await db(TableName.OrgMembership) @@ -230,6 +263,7 @@ export const orgDALFactory = (db: TDbClient) => { findOrgById, findAllOrgsByUserId, ghostUserExists, + findOrgMembersByEmail, findOrgGhostUser, create, updateById, diff --git a/backend/src/services/project-bot/project-bot-dal.ts b/backend/src/services/project-bot/project-bot-dal.ts index 7f342f0ae..74abf8f21 100644 --- a/backend/src/services/project-bot/project-bot-dal.ts +++ b/backend/src/services/project-bot/project-bot-dal.ts @@ -27,5 +27,19 @@ export const projectBotDALFactory = (db: TDbClient) => { } }; - return { ...projectBotOrm, findOne }; + const findProjectByBotId = async (botId: string) => { + try { + const project = await db(TableName.ProjectBot) + .where({ [`${TableName.ProjectBot}.id` as "id"]: botId }) + .join(TableName.Project, `${TableName.ProjectBot}.projectId`, `${TableName.Project}.id`) + .select(selectAllTableCols(TableName.Project)) + .first(); + + return project || null; + } catch (error) { + throw new DatabaseError({ error, name: "Find project by bot id" }); + } + }; + + return { ...projectBotOrm, findOne, findProjectByBotId }; }; diff --git a/backend/src/services/project-membership/project-membership-dal.ts b/backend/src/services/project-membership/project-membership-dal.ts index c6d8249e2..b07043719 100644 --- a/backend/src/services/project-membership/project-membership-dal.ts +++ b/backend/src/services/project-membership/project-membership-dal.ts @@ -1,7 +1,7 @@ import { TDbClient } from "@app/db"; import { TableName, TUserEncryptionKeys } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; -import { ormify } from "@app/lib/knex"; +import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TProjectMembershipDALFactory = ReturnType; @@ -31,6 +31,7 @@ export const projectMembershipDALFactory = (db: TDbClient) => { db.ref("lastName").withSchema(TableName.Users), db.ref("id").withSchema(TableName.Users).as("userId") ); + // .where({ ghost: false }); return members.map(({ email, firstName, lastName, publicKey, ghost, ...data }) => ({ ...data, user: { email, firstName, lastName, id: data.userId, publicKey, ghost } @@ -40,5 +41,20 @@ export const projectMembershipDALFactory = (db: TDbClient) => { } }; - return { ...projectMemberOrm, findAllProjectMembers }; + const findProjectGhostUser = async (projectId: string) => { + try { + const ghostUser = await db(TableName.ProjectMembership) + .where({ projectId }) + .join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`) + .select(selectAllTableCols(TableName.Users)) + .where({ ghost: true }) + .first(); + + return ghostUser; + } catch (error) { + throw new DatabaseError({ error, name: "Find project ghost user" }); + } + }; + + return { ...projectMemberOrm, findAllProjectMembers, findProjectGhostUser }; }; diff --git a/backend/src/services/user/user-dal.ts b/backend/src/services/user/user-dal.ts index 0de490399..7b54e7c10 100644 --- a/backend/src/services/user/user-dal.ts +++ b/backend/src/services/user/user-dal.ts @@ -47,6 +47,17 @@ export const userDALFactory = (db: TDbClient) => { } }; + const findUserByProjectMembershipId = async (projectMembershipId: string) => { + try { + return await db(TableName.ProjectMembership) + .where({ [`${TableName.ProjectMembership}.id` as "id"]: projectMembershipId }) + .join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`) + .first(); + } catch (error) { + throw new DatabaseError({ error, name: "Find user by project membership id" }); + } + }; + const createUserEncryption = async (data: TUserEncryptionKeysInsert, tx?: Knex) => { try { const [userEnc] = await (tx || db)(TableName.UserEncryptionKey).insert(data).returning("*"); @@ -111,6 +122,7 @@ export const userDALFactory = (db: TDbClient) => { findUserEncKeyByEmail, findUserEncKeyByUserId, updateUserEncryptionByUserId, + findUserByProjectMembershipId, upsertUserEncryptionKey, createUserEncryption, findOneUserAction, From 34d1bbc2ed6b6546c082c1c72d18862c07386d0b Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 6 Feb 2024 03:20:46 +0400 Subject: [PATCH 009/157] Added SRP helpers to serverside --- backend/src/lib/crypto/srp.ts | 102 ++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/backend/src/lib/crypto/srp.ts b/backend/src/lib/crypto/srp.ts index b05f3734d..452c54c2a 100644 --- a/backend/src/lib/crypto/srp.ts +++ b/backend/src/lib/crypto/srp.ts @@ -1,4 +1,12 @@ +import argon2, { argon2id } from "argon2"; +import crypto from "crypto"; import jsrp from "jsrp"; +import nacl from "tweetnacl"; +import { encodeBase64 } from "tweetnacl-util"; + +import { TUserEncryptionKeys } from "@app/db/schemas"; + +import { decryptSymmetric, encryptAsymmetric, encryptSymmetric } from "./encryption"; export const generateSrpServerKey = async (salt: string, verifier: string) => { // eslint-disable-next-line new-cap @@ -24,3 +32,97 @@ export const srpCheckClientProof = async ( server.setClientPublicKey(clientPublicKey); return server.checkClientProof(clientProof); }; + +// FOR GHOST USER STUFF +export const generateUserSrpKeys = async (email: string, password: string) => { + const pair = nacl.box.keyPair(); + const secretKeyUint8Array = pair.secretKey; + const publicKeyUint8Array = pair.publicKey; + const privateKey = encodeBase64(secretKeyUint8Array); + const publicKey = encodeBase64(publicKeyUint8Array); + + // eslint-disable-next-line + const client = new jsrp.client(); + await new Promise((resolve) => { + client.init({ username: email, password }, () => resolve(null)); + }); + const { salt, verifier } = await new Promise<{ salt: string; verifier: string }>((resolve, reject) => { + client.createVerifier((err, res) => { + if (err) return reject(err); + return resolve(res); + }); + }); + const derivedKey = await argon2.hash(password, { + salt: Buffer.from(salt), + memoryCost: 65536, + timeCost: 3, + parallelism: 1, + hashLength: 32, + type: argon2id, + raw: true + }); + if (!derivedKey) throw new Error("Failed to derive key from password"); + + const key = crypto.randomBytes(32); + + // create encrypted private key by encrypting the private + // key with the symmetric key [key] + const { + ciphertext: encryptedPrivateKey, + iv: encryptedPrivateKeyIV, + tag: encryptedPrivateKeyTag + } = encryptSymmetric(privateKey, key.toString("base64")); + + // create the protected key by encrypting the symmetric key + // [key] with the derived key + const { + ciphertext: protectedKey, + iv: protectedKeyIV, + tag: protectedKeyTag + } = encryptSymmetric(key.toString("hex"), derivedKey.toString("base64")); + + return { + protectedKey, + plainPrivateKey: privateKey, + protectedKeyIV, + protectedKeyTag, + publicKey, + encryptedPrivateKey, + encryptedPrivateKeyIV, + encryptedPrivateKeyTag, + salt, + verifier + }; +}; + +export const getUserPrivateKey = async (password: string, user: TUserEncryptionKeys) => { + const derivedKey = await argon2.hash(password, { + salt: Buffer.from(user.salt), + memoryCost: 65536, + timeCost: 3, + parallelism: 1, + hashLength: 32, + type: argon2id, + raw: true + }); + if (!derivedKey) throw new Error("Failed to derive key from password"); + const key = decryptSymmetric({ + ciphertext: user.protectedKey!, + iv: user.protectedKeyIV!, + tag: user.protectedKeyTag!, + key: derivedKey.toString("base64") + }); + const privateKey = decryptSymmetric({ + ciphertext: user.encryptedPrivateKey, + iv: user.iv, + tag: user.tag, + key + }); + return privateKey; +}; + +export const buildUserProjectKey = async (privateKey: string, publickey: string) => { + const randomBytes = crypto.randomBytes(16).toString("hex"); + const { nonce, ciphertext } = encryptAsymmetric(randomBytes, publickey, privateKey); + return { nonce, ciphertext }; +}; From a7140941ee8be0d8f3f407f4a14f517d6ae0ead5 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 6 Feb 2024 03:21:18 +0400 Subject: [PATCH 010/157] Fixes --- backend/src/lib/project/index.ts | 2 +- backend/src/server/routes/index.ts | 33 ++++++++++++++++-------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/backend/src/lib/project/index.ts b/backend/src/lib/project/index.ts index 20a63cbd5..f680dc7eb 100644 --- a/backend/src/lib/project/index.ts +++ b/backend/src/lib/project/index.ts @@ -12,7 +12,7 @@ type AddUserToWsDTO = { }[]; }; -export const createWsMembers = async ({ members, decryptKey, userPrivateKey }: AddUserToWsDTO) => { +export const createWsMembers = ({ members, decryptKey, userPrivateKey }: AddUserToWsDTO) => { const key = decryptAsymmetric({ ciphertext: decryptKey.encryptedKey, nonce: decryptKey.nonce, diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 741e2b703..53dd86762 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -280,15 +280,8 @@ export const registerRoutes = async ( secretScanningDAL, secretScanningQueue }); - const projectService = projectServiceFactory({ - permissionService, - projectDAL, - secretBlindIndexDAL, - projectEnvDAL, - projectMembershipDAL, - folderDAL, - licenseService - }); + const projectBotService = projectBotServiceFactory({ permissionService, projectBotDAL }); + const projectMembershipService = projectMembershipServiceFactory({ projectMembershipDAL, projectDAL, @@ -300,6 +293,21 @@ export const registerRoutes = async ( projectRoleDAL, licenseService }); + const projectKeyService = projectKeyServiceFactory({ + permissionService, + projectKeyDAL, + projectMembershipDAL + }); + const projectService = projectServiceFactory({ + permissionService, + projectDAL, + secretBlindIndexDAL, + projectEnvDAL, + orgService, + projectMembershipDAL, + folderDAL, + licenseService + }); const projectEnvService = projectEnvServiceFactory({ permissionService, projectEnvDAL, @@ -307,11 +315,7 @@ export const registerRoutes = async ( projectDAL, folderDAL }); - const projectKeyService = projectKeyServiceFactory({ - permissionService, - projectKeyDAL, - projectMembershipDAL - }); + const projectRoleService = projectRoleServiceFactory({ permissionService, projectRoleDAL }); const snapshotService = secretSnapshotServiceFactory({ @@ -348,7 +352,6 @@ export const registerRoutes = async ( secretImportDAL, secretDAL }); - const projectBotService = projectBotServiceFactory({ permissionService, projectBotDAL }); const integrationAuthService = integrationAuthServiceFactory({ integrationAuthDAL, integrationDAL, From 729ec7866a014fabcb954387b37fe96a607a97e1 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 6 Feb 2024 03:21:29 +0400 Subject: [PATCH 011/157] Update bot-router.ts --- backend/src/server/routes/v1/bot-router.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/backend/src/server/routes/v1/bot-router.ts b/backend/src/server/routes/v1/bot-router.ts index 507423b0d..ff451a17e 100644 --- a/backend/src/server/routes/v1/bot-router.ts +++ b/backend/src/server/routes/v1/bot-router.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { ProjectBotsSchema } from "@app/db/schemas"; +import { BadRequestError } from "@app/lib/errors"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; @@ -26,6 +27,16 @@ export const registerProjectBotRouter = async (server: FastifyZodProvider) => { }, onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { + const project = await server.services.project.getAProject({ + actorId: req.permission.id, + actor: req.permission.type, + projectId: req.params.projectId + }); + + if (!project.e2ee) { + throw new BadRequestError({ message: "Failed to find bot, project has E2EE disabled" }); + } + const bot = await server.services.projectBot.findBotByProjectId({ actor: req.permission.type, actorId: req.permission.id, @@ -66,6 +77,12 @@ export const registerProjectBotRouter = async (server: FastifyZodProvider) => { }, onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { + const project = await server.services.projectBot.findProjectByBotId(req.params.botId); + + if (project?.e2ee === false) { + throw new BadRequestError({ message: "Failed to set bot active, project has E2EE disabled" }); + } + const bot = await server.services.projectBot.setBotActiveState({ actor: req.permission.type, actorId: req.permission.id, From b9840ceba9f7e86a9b38b10a9a7be42925810ad1 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 6 Feb 2024 03:21:42 +0400 Subject: [PATCH 012/157] Add project role --- .../server/routes/v1/project-membership-router.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/backend/src/server/routes/v1/project-membership-router.ts b/backend/src/server/routes/v1/project-membership-router.ts index 09de72233..afa2135c0 100644 --- a/backend/src/server/routes/v1/project-membership-router.ts +++ b/backend/src/server/routes/v1/project-membership-router.ts @@ -1,6 +1,12 @@ import { z } from "zod"; -import { OrgMembershipsSchema, ProjectMembershipsSchema, UserEncryptionKeysSchema, UsersSchema } from "@app/db/schemas"; +import { + OrgMembershipsSchema, + ProjectMembershipRole, + ProjectMembershipsSchema, + UserEncryptionKeysSchema, + UsersSchema +} from "@app/db/schemas"; import { EventType } from "@app/ee/services/audit-log/audit-log-types"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; @@ -80,7 +86,10 @@ export const registerProjectMembershipRouter = async (server: FastifyZodProvider actor: req.permission.type, actorOrgId: req.permission.orgId, projectId: req.params.workspaceId, - members: req.body.members + members: req.body.members.map((member) => ({ + ...member, + projectRole: ProjectMembershipRole.Member + })) }); await server.services.auditLog.createAuditLog({ From db773864d56964b479c5f9c5dea4b4914542bd9f Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 6 Feb 2024 03:22:41 +0400 Subject: [PATCH 013/157] Update project-router.ts --- .../src/server/routes/v1/project-router.ts | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/backend/src/server/routes/v1/project-router.ts b/backend/src/server/routes/v1/project-router.ts index f0faaa0f4..f7616de4e 100644 --- a/backend/src/server/routes/v1/project-router.ts +++ b/backend/src/server/routes/v1/project-router.ts @@ -261,33 +261,36 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { }), response: { 200: z.object({ - invitee: UsersSchema, + invitees: UsersSchema.array(), latestKey: ProjectKeysSchema.optional() }) } }, onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { - const { invitee, latestKey } = await server.services.projectMembership.inviteUserToProject({ + const { invitees, latestKey } = await server.services.projectMembership.inviteUserToProject({ actorId: req.permission.id, actor: req.permission.type, actorOrgId: req.permission.orgId, projectId: req.params.workspaceId, - email: req.body.email + emails: [req.body.email] }); - await server.services.auditLog.createAuditLog({ - ...req.auditLogInfo, - projectId: req.params.workspaceId, - event: { - type: EventType.ADD_WORKSPACE_MEMBER, - metadata: { - userId: invitee.id, - email: invitee.email + for (const invitee of invitees) { + // eslint-disable-next-line no-await-in-loop + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.params.workspaceId, + event: { + type: EventType.ADD_WORKSPACE_MEMBER, + metadata: { + userId: invitee.id, + email: invitee.email + } } - } - }); - return { invitee, latestKey }; + }); + } + return { invitees, latestKey }; } }); From 84cab17f5c29f18de4a2329aa2648415345063cf Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 6 Feb 2024 03:23:01 +0400 Subject: [PATCH 014/157] Ghost user WIP --- .../src/server/routes/v3/project-router.ts | 198 ++++++++++++++---- backend/src/services/org/org-service.ts | 64 ++++-- backend/src/services/org/org-types.ts | 7 + .../project-bot/project-bot-service.ts | 54 +++-- .../services/project-bot/project-bot-types.ts | 18 +- .../project-membership-service.ts | 158 ++++++++------ .../project-membership-types.ts | 3 +- .../src/services/project/project-service.ts | 31 ++- 8 files changed, 373 insertions(+), 160 deletions(-) diff --git a/backend/src/server/routes/v3/project-router.ts b/backend/src/server/routes/v3/project-router.ts index 5f83167a7..db39c5677 100644 --- a/backend/src/server/routes/v3/project-router.ts +++ b/backend/src/server/routes/v3/project-router.ts @@ -1,10 +1,15 @@ +import { ForbiddenError } from "@casl/ability"; import crypto from "crypto"; import { z } from "zod"; import { ProjectMembershipRole, ProjectsSchema } from "@app/db/schemas"; +import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission"; +import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; import { encryptAsymmetric } from "@app/lib/crypto"; +import { BadRequestError } from "@app/lib/errors"; import { createWsMembers } from "@app/lib/project"; import { authRateLimit } from "@app/server/config/rateLimiter"; +import { ActorType } from "@app/services/auth/auth-type"; const projectWithEnv = ProjectsSchema.merge( z.object({ @@ -24,103 +29,212 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { schema: { body: z.object({ projectName: z.string().trim(), - inviteAllOrgMembers: z.boolean(), + inviteMemberEmails: z.array(z.string().email()).optional(), organizationId: z.string().trim() }), response: { 200: z.object({ - workspace: projectWithEnv + project: projectWithEnv }) } }, handler: async (req) => { - // 1. create the ghost user and add it to the org as admin - const ghost = await server.services.org.addGhostUser(req.body.organizationId); + const { permission } = await server.services.permission.getOrgPermission( + req.permission.type, + req.permission.id, + req.body.organizationId + ); + ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.Workspace); - // 2. create the workspace - const workspace = await server.services.project.createProject({ - actorId: ghost.user.id, + // 2. Create a new project (will set the e2ee db field to false). + const { project, ghostUser } = await server.services.project.createProject({ + actorId: req.permission.id, actor: req.permission.type, orgId: req.body.organizationId, workspaceName: req.body.projectName }); - // 3. create a random key that we'll use as the project key + // 3. Create a random key that we'll use as the project key. const randomBytes = crypto.randomBytes(16).toString("hex"); - const ghostPrivateKey = ghost.keys.plainPrivateKey; - + // 4. Encrypt the project key with the users key pair. const { ciphertext: encryptedProjectKey, nonce: encryptedProjectKeyIv } = encryptAsymmetric( randomBytes, - ghost.keys.publicKey, - ghostPrivateKey + ghostUser.keys.publicKey, + ghostUser.keys.plainPrivateKey ); - // 3. create workspace keys for the ghost user + // 4. Save the project key for the ghost user. await server.services.projectKey.uploadProjectKeys({ - projectId: workspace.id, + projectId: project.id, actor: req.permission.type, - actorId: ghost.user.id, + actorId: ghostUser.user.id, nonce: encryptedProjectKeyIv, - receiverId: ghost.user.id, + receiverId: ghostUser.user.id, encryptedKey: encryptedProjectKey }); - // 4. create a project bot + // 5. Create a bot for the project. const bot = await server.services.projectBot.findBotByProjectId({ - actorId: ghost.user.id, + actorId: ghostUser.user.id, actor: req.permission.type, - projectId: workspace.id + projectId: project.id, + + // We set the publicKey and privateKey of the bot to the same as the ghost user. + // We do this because we'll need to access the private key again later, when adding new members to the project. + publicKey: ghostUser.keys.publicKey, + privateKey: ghostUser.keys.plainPrivateKey }); - // 5. activate the bot + // 6. Activate the bot. await server.services.projectBot.setBotActiveState({ botKey: { encryptedKey: encryptedProjectKey, nonce: encryptedProjectKeyIv }, - actorId: ghost.user.id, + actorId: ghostUser.user.id, isActive: true, actor: req.permission.type, botId: bot.id }); - // 6. get the current user & org membership + // 7. get the current user & org membership const user = await server.services.user.getMe(req.permission.id); const userOrgMembership = await server.services.permission.getUserOrgPermission(user.id, req.body.organizationId); // 7. Get the latest key from the ghost! const latestKey = await server.services.projectKey.getLatestProjectKey({ - actorId: ghost.user.id, + actorId: ghostUser.user.id, actor: req.permission.type, - projectId: workspace.id + projectId: project.id }); if (!latestKey) throw new Error("Failed to get latest key"); - // 8. Create workspace members for the current user + // If the project is being created by a user, add the user to the project as an admin + if (req.permission.type === ActorType.USER) { + const projectAdmin = createWsMembers({ + decryptKey: latestKey, + members: [ + { + userPublicKey: user.publicKey, + orgMembershipId: userOrgMembership.membership.id, + projectMembershipRole: ProjectMembershipRole.Admin // <-- Make the first user an admin + } + ], + userPrivateKey: ghostUser.keys.plainPrivateKey + }); - const projectAdmin = await createWsMembers({ - decryptKey: latestKey, - members: [ - { - userPublicKey: user.publicKey, - orgMembershipId: userOrgMembership.membership.id, - projectMembershipRole: ProjectMembershipRole.Admin // <-- Make the first user an admin - } - ], - userPrivateKey: ghostPrivateKey - }); + await server.services.projectMembership.addUsersToProject({ + projectId: project.id, + actorId: ghostUser.user.id, + actor: req.permission.type, + members: projectAdmin + }); + } + // If the project is being created by an identity, add the identity to the project as an admin + else if (req.permission.type === ActorType.IDENTITY) { + await server.services.identityProject.createProjectIdentity({ + actor: ActorType.IDENTITY, + actorId: ghostUser.user.id, + identityId: req.permission.id, + projectId: project.id, + role: ProjectMembershipRole.Admin + }); + } - // 9. Add the current user to the workspace - await server.services.projectMembership.addUsersToProject({ - projectId: workspace.id, - actorId: ghost.user.id, + return { project }; + } + }); + + server.route({ + method: "POST", + url: "/:projectId/memberships", + config: { + rateLimit: authRateLimit + }, + schema: { + params: z.object({ + projectId: z.string() + }), + body: z.object({ + emails: z.string().email().array() + }) + }, + handler: async (req) => { + const { permission } = await server.services.permission.getProjectPermission( + req.permission.type, + req.permission.id, + req.params.projectId + ); + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Member); + + const project = await server.services.project.getAProject({ + actorId: req.permission.id, actor: req.permission.type, - members: projectAdmin + projectId: req.params.projectId }); - return { workspace }; + const ghostUser = await server.services.project.findProjectGhostUser(req.params.projectId); + + if (!ghostUser) { + throw new BadRequestError({ + message: "Failed" // TODO: Add a message + }); + } + + const latestKey = await server.services.projectKey.getLatestProjectKey({ + actorId: ghostUser.id, + actor: ActorType.USER, + projectId: req.params.projectId + }); + + if (!latestKey) { + throw new BadRequestError({ + message: "Failed to find project key" + }); + } + + const bot = await server.services.projectBot.findBotByProjectId({ + actor: req.permission.type, + actorId: req.permission.id, + projectId: req.params.projectId + }); + + // We get the bot private key, because the bot private key is the same as the ghost user's private key. + const botPrivateKey = server.services.projectBot.getBotPrivateKey({ bot }); + + const members = await server.services.org.findOrgMembersByEmail({ + actor: req.permission.type, + actorId: req.permission.id, + orgId: project.orgId, + emails: req.body.emails + }); + + if (members.length !== req.body.emails.length) { + throw new BadRequestError({ + message: "Some users are not part of the organization" + }); + } + + const wsMembers = createWsMembers({ + members: members.map((membership) => ({ + orgMembershipId: membership.id, + projectMembershipRole: ProjectMembershipRole.Member, + userPublicKey: membership.user.publicKey + })), + decryptKey: latestKey, + userPrivateKey: botPrivateKey + }); + + await server.services.projectMembership.addUsersToProject({ + projectId: req.params.projectId, + actorId: ghostUser.id, // We set the actor ID to the ghost user, because this is used as senderId in the project key sharing + actor: ActorType.USER, + members: wsMembers + }); + + return {}; } }); }; diff --git a/backend/src/services/org/org-service.ts b/backend/src/services/org/org-service.ts index 3b20b9a14..bbc8fe7a6 100644 --- a/backend/src/services/org/org-service.ts +++ b/backend/src/services/org/org-service.ts @@ -2,6 +2,8 @@ import { ForbiddenError } from "@casl/ability"; import slugify from "@sindresorhus/slugify"; import crypto from "crypto"; import jwt from "jsonwebtoken"; +import { Knex } from "knex"; +import { nanoid } from "nanoid"; import { OrgMembershipRole, OrgMembershipStatus } from "@app/db/schemas"; import { TProjects } from "@app/db/schemas/projects"; @@ -30,6 +32,7 @@ import { TOrgRoleDALFactory } from "./org-role-dal"; import { TDeleteOrgMembershipDTO, TFindAllWorkspacesDTO, + TFindOrgMembersByEmailDTO, TInviteUserToOrgDTO, TUpdateOrgDTO, TUpdateOrgMembershipDTO, @@ -95,6 +98,15 @@ export const orgServiceFactory = ({ return members; }; + const findOrgMembersByEmail = async ({ actor, actorId, orgId, emails }: TFindOrgMembersByEmailDTO) => { + const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId); + ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.Member); + + const members = await orgDAL.findOrgMembersByEmail(orgId, emails); + + return members; + }; + const findAllWorkspaces = async ({ actor, actorId, actorOrgId, orgId }: TFindAllWorkspacesDTO) => { const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorOrgId); ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Read, OrgPermissionSubjects.Workspace); @@ -120,31 +132,38 @@ export const orgServiceFactory = ({ return workspaces.filter((workspace) => organizationWorkspaceIds.has(workspace.id)); }; - const addGhostUser = async (orgId: string) => { - const email = `ghost@${orgId}.com`; + const addGhostUser = async (orgId: string, tx?: Knex) => { + const email = `ghost@${nanoid(8)}-${orgId}.com`; // We add a nanoid because the email is unique. And we have to create a new ghost user each time, so we can have access to the private key. const password = crypto.randomBytes(128).toString("hex"); - const user = await userDAL.create({ - ghost: true, - authMethods: [AuthMethod.EMAIL], - email: `ghost@${orgId}.com`, - isAccepted: true - }); + const user = await userDAL.create( + { + ghost: true, + authMethods: [AuthMethod.EMAIL], + email, + isAccepted: true + }, + tx + ); const encKeys = await generateUserSrpKeys(email, password); - await userDAL.upsertUserEncryptionKey(user.id, { - encryptionVersion: 2, - protectedKey: encKeys.protectedKey, - protectedKeyIV: encKeys.protectedKeyIV, - protectedKeyTag: encKeys.protectedKeyTag, - publicKey: encKeys.publicKey, - encryptedPrivateKey: encKeys.encryptedPrivateKey, - iv: encKeys.encryptedPrivateKeyIV, - tag: encKeys.encryptedPrivateKeyTag, - salt: encKeys.salt, - verifier: encKeys.verifier - }); + await userDAL.upsertUserEncryptionKey( + user.id, + { + encryptionVersion: 2, + protectedKey: encKeys.protectedKey, + protectedKeyIV: encKeys.protectedKeyIV, + protectedKeyTag: encKeys.protectedKeyTag, + publicKey: encKeys.publicKey, + encryptedPrivateKey: encKeys.encryptedPrivateKey, + iv: encKeys.encryptedPrivateKeyIV, + tag: encKeys.encryptedPrivateKeyTag, + salt: encKeys.salt, + verifier: encKeys.verifier + }, + tx + ); const createMembershipData = { orgId, @@ -153,9 +172,7 @@ export const orgServiceFactory = ({ status: OrgMembershipStatus.Accepted }; - console.log("createMembershipData", createMembershipData); - - await orgDAL.createMembership(createMembershipData); + await orgDAL.createMembership(createMembershipData, tx); return { user, @@ -533,6 +550,7 @@ export const orgServiceFactory = ({ inviteUserToOrganization, verifyUserToOrg, updateOrg, + findOrgMembersByEmail, createOrganization, deleteOrganizationById, deleteOrgMembership, diff --git a/backend/src/services/org/org-types.ts b/backend/src/services/org/org-types.ts index bbb325247..bd8fe2e95 100644 --- a/backend/src/services/org/org-types.ts +++ b/backend/src/services/org/org-types.ts @@ -30,6 +30,13 @@ export type TVerifyUserToOrgDTO = { code: string; }; +export type TFindOrgMembersByEmailDTO = { + actor: ActorType; + actorId: string; + orgId: string; + emails: string[]; +}; + export type TFindAllWorkspacesDTO = { actor: ActorType; actorId: string; diff --git a/backend/src/services/project-bot/project-bot-service.ts b/backend/src/services/project-bot/project-bot-service.ts index a9514c1e3..a26170be3 100644 --- a/backend/src/services/project-bot/project-bot-service.ts +++ b/backend/src/services/project-bot/project-bot-service.ts @@ -13,10 +13,9 @@ import { } from "@app/lib/crypto"; import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; import { BadRequestError } from "@app/lib/errors"; -import { TProjectPermission } from "@app/lib/types"; import { TProjectBotDALFactory } from "./project-bot-dal"; -import { TGetPrivateKeyDTO, TSetActiveStateDTO } from "./project-bot-types"; +import { TFindBotByProjectIdDTO, TGetPrivateKeyDTO, TSetActiveStateDTO } from "./project-bot-types"; type TProjectBotServiceFactoryDep = { permissionService: Pick; @@ -26,12 +25,12 @@ type TProjectBotServiceFactoryDep = { export type TProjectBotServiceFactory = ReturnType; export const projectBotServiceFactory = ({ projectBotDAL, permissionService }: TProjectBotServiceFactoryDep) => { - const getBotPrivateKey = async ({ encoding, nonce, tag, encryptedPrivateKey }: TGetPrivateKeyDTO) => + const getBotPrivateKey = ({ bot }: TGetPrivateKeyDTO) => infisicalSymmetricDecrypt({ - keyEncoding: encoding, - iv: nonce, - tag, - ciphertext: encryptedPrivateKey + keyEncoding: bot.keyEncoding as SecretKeyEncoding, + iv: bot.iv, + tag: bot.tag, + ciphertext: bot.encryptedPrivateKey }); const getBotKey = async (projectId: string) => { @@ -41,24 +40,24 @@ export const projectBotServiceFactory = ({ projectBotDAL, permissionService }: T if (!bot.encryptedProjectKeyNonce || !bot.encryptedProjectKey) throw new BadRequestError({ message: "Encryption key missing" }); - const privateKeyBot = await getBotPrivateKey({ - nonce: bot.iv, - tag: bot.tag, - encryptedPrivateKey: bot.encryptedPrivateKey, - encoding: bot.keyEncoding as SecretKeyEncoding - }); - - console.log("privateKeyBot", privateKeyBot); + const botPrivateKey = getBotPrivateKey({ bot }); return decryptAsymmetric({ ciphertext: bot.encryptedProjectKey, - privateKey: privateKeyBot, + privateKey: botPrivateKey, nonce: bot.encryptedProjectKeyNonce, publicKey: bot.sender.publicKey }); }; - const findBotByProjectId = async ({ actorId, actor, actorOrgId, projectId }: TProjectPermission) => { + const findBotByProjectId = async ({ + actorId, + actor, + projectId, + actorOrgId, + privateKey, + publicKey + }: TFindBotByProjectIdDTO) => { const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId, actorOrgId); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); const appCfg = getConfig(); @@ -67,9 +66,10 @@ export const projectBotServiceFactory = ({ projectBotDAL, permissionService }: T const doc = await projectBotDAL.findOne({ projectId }, tx); if (doc) return doc; - const { publicKey, privateKey } = generateAsymmetricKeyPair(); + const keys = privateKey && publicKey ? { privateKey, publicKey } : generateAsymmetricKeyPair(); + if (appCfg.ROOT_ENCRYPTION_KEY) { - const { iv, tag, ciphertext } = encryptSymmetric(privateKey, appCfg.ROOT_ENCRYPTION_KEY); + const { iv, tag, ciphertext } = encryptSymmetric(keys.privateKey, appCfg.ROOT_ENCRYPTION_KEY); return projectBotDAL.create( { name: "Infisical Bot", @@ -78,7 +78,7 @@ export const projectBotServiceFactory = ({ projectBotDAL, permissionService }: T iv, encryptedPrivateKey: ciphertext, isActive: false, - publicKey, + publicKey: keys.publicKey, algorithm: SecretEncryptionAlgo.AES_256_GCM, keyEncoding: SecretKeyEncoding.BASE64 }, @@ -86,7 +86,7 @@ export const projectBotServiceFactory = ({ projectBotDAL, permissionService }: T ); } if (appCfg.ENCRYPTION_KEY) { - const { iv, tag, ciphertext } = encryptSymmetric128BitHexKeyUTF8(privateKey, appCfg.ENCRYPTION_KEY); + const { iv, tag, ciphertext } = encryptSymmetric128BitHexKeyUTF8(keys.privateKey, appCfg.ENCRYPTION_KEY); return projectBotDAL.create( { name: "Infisical Bot", @@ -95,7 +95,7 @@ export const projectBotServiceFactory = ({ projectBotDAL, permissionService }: T iv, encryptedPrivateKey: ciphertext, isActive: false, - publicKey, + publicKey: keys.publicKey, algorithm: SecretEncryptionAlgo.AES_256_GCM, keyEncoding: SecretKeyEncoding.UTF8 }, @@ -107,6 +107,15 @@ export const projectBotServiceFactory = ({ projectBotDAL, permissionService }: T return bot; }; + const findProjectByBotId = async (botId: string) => { + try { + const bot = await projectBotDAL.findProjectByBotId(botId); + return bot; + } catch (e) { + throw new BadRequestError({ message: "Failed to find bot by ID" }); + } + }; + const setBotActiveState = async ({ actor, botId, botKey, actorId, actorOrgId, isActive }: TSetActiveStateDTO) => { const bot = await projectBotDAL.findById(botId); if (!bot) throw new BadRequestError({ message: "Bot not found" }); @@ -145,6 +154,7 @@ export const projectBotServiceFactory = ({ projectBotDAL, permissionService }: T findBotByProjectId, setBotActiveState, getBotPrivateKey, + findProjectByBotId, getBotKey }; }; diff --git a/backend/src/services/project-bot/project-bot-types.ts b/backend/src/services/project-bot/project-bot-types.ts index 3d4f400de..4431c6686 100644 --- a/backend/src/services/project-bot/project-bot-types.ts +++ b/backend/src/services/project-bot/project-bot-types.ts @@ -1,4 +1,5 @@ -import { SecretKeyEncoding } from "@app/db/schemas"; +// import { SecretKeyEncoding } from "@app/db/schemas"; +import { TProjectBots } from "@app/db/schemas"; import { TProjectPermission } from "@app/lib/types"; export type TSetActiveStateDTO = { @@ -10,9 +11,16 @@ export type TSetActiveStateDTO = { botId: string; } & Omit; +export type TFindBotByProjectIdDTO = { + privateKey?: string; + publicKey?: string; +} & TProjectPermission; + export type TGetPrivateKeyDTO = { - encoding: SecretKeyEncoding; - nonce: string; - tag: string; - encryptedPrivateKey: string; + // encoding: SecretKeyEncoding; + // nonce: string; + // tag: string; + // encryptedPrivateKey: string; + + bot: TProjectBots; }; diff --git a/backend/src/services/project-membership/project-membership-service.ts b/backend/src/services/project-membership/project-membership-service.ts index 477891bf8..0f628712b 100644 --- a/backend/src/services/project-membership/project-membership-service.ts +++ b/backend/src/services/project-membership/project-membership-service.ts @@ -1,6 +1,7 @@ +/* eslint-disable no-await-in-loop */ import { ForbiddenError } from "@casl/ability"; -import { OrgMembershipStatus, ProjectMembershipRole, TableName } from "@app/db/schemas"; +import { OrgMembershipStatus, ProjectMembershipRole, TableName, TUsers } from "@app/db/schemas"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; @@ -27,7 +28,7 @@ type TProjectMembershipServiceFactoryDep = { permissionService: Pick; smtpService: TSmtpService; projectMembershipDAL: TProjectMembershipDALFactory; - userDAL: Pick; + userDAL: Pick; projectRoleDAL: Pick; orgDAL: Pick; projectDAL: Pick; @@ -55,64 +56,78 @@ export const projectMembershipServiceFactory = ({ return projectMembershipDAL.findAllProjectMembers(projectId); }; - const inviteUserToProject = async ({ actorId, actor, actorOrgId, projectId, email }: TInviteUserToProjectDTO) => { + const inviteUserToProject = async ({ actorId, actor, actorOrgId, projectId, emails }: TInviteUserToProjectDTO) => { const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId, actorOrgId); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Member); - const invitee = await userDAL.findOne({ email }); - if (!invitee || !invitee.isAccepted) - throw new BadRequestError({ - message: "Faield to validate invitee", - name: "Invite user to project" + const invitees: TUsers[] = []; + + for (const email of emails) { + const invitee = await userDAL.findOne({ email }); + if (!invitee || !invitee.isAccepted) + throw new BadRequestError({ + message: "Faield to validate invitee", + name: "Invite user to project" + }); + + const inviteeMembership = await projectMembershipDAL.findOne({ + userId: invitee.id, + projectId + }); + if (inviteeMembership) + throw new BadRequestError({ + message: "Existing member of project", + name: "Invite user to project" + }); + + const project = await projectDAL.findById(projectId); + const inviteeMembershipOrg = await orgDAL.findMembership({ + userId: invitee.id, + orgId: project.orgId, + status: OrgMembershipStatus.Accepted + }); + if (!inviteeMembershipOrg) + throw new BadRequestError({ + message: "Failed to validate invitee org membership", + name: "Invite user to project" + }); + + await projectMembershipDAL.create({ + userId: invitee.id, + projectId, + role: ProjectMembershipRole.Member }); - const inviteeMembership = await projectMembershipDAL.findOne({ - userId: invitee.id, - projectId - }); - if (inviteeMembership) - throw new BadRequestError({ - message: "Existing member of project", - name: "Invite user to project" + const sender = await userDAL.findById(actorId); + const appCfg = getConfig(); + await smtpService.sendMail({ + template: SmtpTemplates.WorkspaceInvite, + subjectLine: "Infisical workspace invitation", + recipients: [invitee.email], + substitutions: { + inviterFirstName: sender.firstName, + inviterEmail: sender.email, + workspaceName: project.name, + callback_url: `${appCfg.SITE_URL}/login` + } }); - const project = await projectDAL.findById(projectId); - const inviteeMembershipOrg = await orgDAL.findMembership({ - userId: invitee.id, - orgId: project.orgId, - status: OrgMembershipStatus.Accepted - }); - if (!inviteeMembershipOrg) - throw new BadRequestError({ - message: "Failed to validate invitee org membership", - name: "Invite user to project" - }); + invitees.push(invitee); + } const latestKey = await projectKeyDAL.findLatestProjectKey(actorId, projectId); - await projectMembershipDAL.create({ - userId: invitee.id, - projectId, - role: ProjectMembershipRole.Member - }); - const sender = await userDAL.findById(actorId); - const appCfg = getConfig(); - await smtpService.sendMail({ - template: SmtpTemplates.WorkspaceInvite, - subjectLine: "Infisical workspace invitation", - recipients: [invitee.email], - substitutions: { - inviterFirstName: sender.firstName, - inviterEmail: sender.email, - workspaceName: project.name, - callback_url: `${appCfg.SITE_URL}/login` - } - }); - - return { invitee, latestKey }; + return { invitees, latestKey }; }; - const addUsersToProject = async ({ projectId, actorId, actor, actorOrgId, members }: TAddUsersToWorkspaceDTO) => { + const addUsersToProject = async ({ + projectId, + actorId, + actor, + actorOrgId, + members, + sendEmails = true + }: TAddUsersToWorkspaceDTO) => { const project = await projectDAL.findById(projectId); if (!project) throw new BadRequestError({ message: "Project not found" }); @@ -158,19 +173,22 @@ export const projectMembershipServiceFactory = ({ tx ); }); - const sender = await userDAL.findById(actorId); - const appCfg = getConfig(); - await smtpService.sendMail({ - template: SmtpTemplates.WorkspaceInvite, - subjectLine: "Infisical workspace invitation", - recipients: orgMembers.map(({ email }) => email).filter(Boolean), - substitutions: { - inviterFirstName: sender.firstName, - inviterEmail: sender.email, - workspaceName: project.name, - callback_url: `${appCfg.SITE_URL}/login` - } - }); + + if (sendEmails) { + const sender = await userDAL.findById(actorId); + const appCfg = getConfig(); + await smtpService.sendMail({ + template: SmtpTemplates.WorkspaceInvite, + subjectLine: "Infisical workspace invitation", + recipients: orgMembers.map(({ email }) => email).filter(Boolean), + substitutions: { + inviterFirstName: sender.firstName, + inviterEmail: sender.email, + workspaceName: project.name, + callback_url: `${appCfg.SITE_URL}/login` + } + }); + } return orgMembers; }; @@ -185,6 +203,15 @@ export const projectMembershipServiceFactory = ({ const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId, actorOrgId); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Edit, ProjectPermissionSub.Member); + const membershipUser = await userDAL.findUserByProjectMembershipId(membershipId); + + if (membershipUser?.ghost) { + throw new BadRequestError({ + message: "Unauthorized member update", + name: "Update project membership" + }); + } + const isCustomRole = !Object.values(ProjectMembershipRole).includes(role as ProjectMembershipRole); if (isCustomRole) { const customRole = await projectRoleDAL.findOne({ slug: role, projectId }); @@ -220,6 +247,15 @@ export const projectMembershipServiceFactory = ({ const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId, actorOrgId); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Delete, ProjectPermissionSub.Member); + const membershipUser = await userDAL.findUserByProjectMembershipId(membershipId); + + if (membershipUser?.ghost) { + throw new BadRequestError({ + message: "Cannot delete ghost", + name: "Delete project membership" + }); + } + const membership = await projectMembershipDAL.transaction(async (tx) => { const [deletedMembership] = await projectMembershipDAL.delete({ projectId, id: membershipId }, tx); await projectKeyDAL.delete({ receiverId: deletedMembership.userId, projectId }, tx); diff --git a/backend/src/services/project-membership/project-membership-types.ts b/backend/src/services/project-membership/project-membership-types.ts index 6fe2977ee..786072d56 100644 --- a/backend/src/services/project-membership/project-membership-types.ts +++ b/backend/src/services/project-membership/project-membership-types.ts @@ -4,7 +4,7 @@ import { TProjectPermission } from "@app/lib/types"; export type TGetProjectMembershipDTO = TProjectPermission; export type TInviteUserToProjectDTO = { - email: string; + emails: string[]; } & TProjectPermission; export type TUpdateProjectMembershipDTO = { @@ -17,6 +17,7 @@ export type TDeleteProjectMembershipDTO = { } & TProjectPermission; export type TAddUsersToWorkspaceDTO = { + sendEmails?: boolean; members: { orgMembershipId: string; workspaceEncryptedKey: string; diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index f56c40fa2..080278322 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -11,6 +11,7 @@ import { createSecretBlindIndex } from "@app/lib/crypto"; import { BadRequestError } from "@app/lib/errors"; import { alphaNumericNanoId } from "@app/lib/nanoid"; +import { TOrgServiceFactory } from "../org/org-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-index-dal"; @@ -28,7 +29,8 @@ type TProjectServiceFactoryDep = { projectDAL: TProjectDALFactory; folderDAL: Pick; projectEnvDAL: Pick; - projectMembershipDAL: Pick; + projectMembershipDAL: Pick; + orgService: Pick; secretBlindIndexDAL: Pick; permissionService: TPermissionServiceFactory; licenseService: Pick; @@ -40,6 +42,7 @@ export const projectServiceFactory = ({ projectDAL, permissionService, folderDAL, + orgService, secretBlindIndexDAL, projectMembershipDAL, projectEnvDAL, @@ -64,7 +67,9 @@ export const projectServiceFactory = ({ }); } - const newProject = projectDAL.transaction(async (tx) => { + const results = await projectDAL.transaction(async (tx) => { + const ghostUser = await orgService.addGhostUser(orgId, tx); + const project = await projectDAL.create( { name: workspaceName, @@ -74,10 +79,10 @@ export const projectServiceFactory = ({ }, tx ); - // set user as admin member for project + // set ghost user as admin of project await projectMembershipDAL.create( { - userId: actorId, + userId: ghostUser.user.id, role: ProjectMembershipRole.Admin, projectId: project.id }, @@ -106,10 +111,23 @@ export const projectServiceFactory = ({ tx ); // _id for backward compat - return { ...project, environments: envs, _id: project.id }; + return { + project: { + ...project, + environments: envs, + _id: project.id + }, + ghostUser + }; }); - return newProject; + return results; + }; + + const findProjectGhostUser = async (projectId: string) => { + const user = await projectMembershipDAL.findProjectGhostUser(projectId); + + return user; }; const deleteProject = async ({ actor, actorId, actorOrgId, projectId }: TDeleteProjectDTO) => { @@ -156,6 +174,7 @@ export const projectServiceFactory = ({ createProject, deleteProject, getProjects, + findProjectGhostUser, getAProject, toggleAutoCapitalization, updateName From 3b1fc4b1564ed5faf65d54a09eaa29e5b7d13d97 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 6 Feb 2024 19:27:53 +0400 Subject: [PATCH 015/157] More gohst user --- .../services/audit-log/audit-log-service.ts | 1 + .../src/ee/services/license/licence-fns.ts | 8 +- backend/src/lib/project/index.ts | 21 ++ backend/src/server/routes/index.ts | 6 + .../src/server/routes/v1/project-router.ts | 1 + .../src/server/routes/v3/project-router.ts | 196 ++---------------- .../project-bot/project-bot-service.ts | 64 ++---- .../services/project-key/project-key-dal.ts | 11 +- .../project-membership-service.ts | 137 +++++++++++- .../project-membership-types.ts | 5 + backend/src/services/project/project-dal.ts | 15 ++ .../src/services/project/project-service.ts | 169 ++++++++++++++- .../templates/workspaceInvitation.handlebars | 18 +- frontend/src/hooks/api/workspace/types.ts | 1 + .../src/pages/org/[id]/overview/index.tsx | 35 ++-- 15 files changed, 413 insertions(+), 275 deletions(-) diff --git a/backend/src/ee/services/audit-log/audit-log-service.ts b/backend/src/ee/services/audit-log/audit-log-service.ts index 1c7868fc2..c4d4aabc0 100644 --- a/backend/src/ee/services/audit-log/audit-log-service.ts +++ b/backend/src/ee/services/audit-log/audit-log-service.ts @@ -58,6 +58,7 @@ export const auditLogServiceFactory = ({ if (data.event.type !== EventType.LOGIN_IDENTITY_UNIVERSAL_AUTH) { if (!data.projectId && !data.orgId) throw new BadRequestError({ message: "Must either project id or org id" }); } + return auditLogQueue.pushToLog(data); }; diff --git a/backend/src/ee/services/license/licence-fns.ts b/backend/src/ee/services/license/licence-fns.ts index 9b1d4c203..b8b02ec54 100644 --- a/backend/src/ee/services/license/licence-fns.ts +++ b/backend/src/ee/services/license/licence-fns.ts @@ -9,11 +9,11 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({ _id: null, slug: null, tier: -1, - workspaceLimit: null, + workspaceLimit: 5000, workspacesUsed: 0, - memberLimit: null, + memberLimit: 5000, membersUsed: 0, - environmentLimit: null, + environmentLimit: 5000, environmentsUsed: 0, secretVersioning: true, pitRecovery: false, @@ -28,7 +28,7 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({ status: null, trial_end: null, has_used_trial: true, - secretApproval: false, + secretApproval: true, secretRotation: true }); diff --git a/backend/src/lib/project/index.ts b/backend/src/lib/project/index.ts index f680dc7eb..d4e97ba17 100644 --- a/backend/src/lib/project/index.ts +++ b/backend/src/lib/project/index.ts @@ -1,3 +1,5 @@ +import crypto from "crypto"; + import { ProjectMembershipRole, TProjectKeys } from "@app/db/schemas"; import { decryptAsymmetric, encryptAsymmetric } from "../crypto"; @@ -37,3 +39,22 @@ export const createWsMembers = ({ members, decryptKey, userPrivateKey }: AddUser return newWsMembers; }; + +type TCreateWorkspaceKeyDTO = { + publicKey: string; + privateKey: string; +}; + +export const createWorkspaceKey = ({ publicKey, privateKey }: TCreateWorkspaceKeyDTO) => { + // 3. Create a random key that we'll use as the project key. + const randomBytes = crypto.randomBytes(16).toString("hex"); + + // 4. Encrypt the project key with the users key pair. + const { ciphertext: encryptedProjectKey, nonce: encryptedProjectKeyIv } = encryptAsymmetric( + randomBytes, + publicKey, + privateKey + ); + + return { key: encryptedProjectKey, iv: encryptedProjectKeyIv }; +}; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 53dd86762..e793d2133 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -286,6 +286,7 @@ export const registerRoutes = async ( projectMembershipDAL, projectDAL, permissionService, + projectBotDAL, orgDAL, userDAL, smtpService, @@ -302,6 +303,11 @@ export const registerRoutes = async ( permissionService, projectDAL, secretBlindIndexDAL, + identityProjectDAL, + identityOrgMembershipDAL, + projectBotDAL, + projectKeyDAL, + userDAL, projectEnvDAL, orgService, projectMembershipDAL, diff --git a/backend/src/server/routes/v1/project-router.ts b/backend/src/server/routes/v1/project-router.ts index f7616de4e..873a2e78c 100644 --- a/backend/src/server/routes/v1/project-router.ts +++ b/backend/src/server/routes/v1/project-router.ts @@ -249,6 +249,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { } }); + // Is this actually used..? server.route({ url: "/:workspaceId/invite-signup", method: "POST", diff --git a/backend/src/server/routes/v3/project-router.ts b/backend/src/server/routes/v3/project-router.ts index db39c5677..c1759f718 100644 --- a/backend/src/server/routes/v3/project-router.ts +++ b/backend/src/server/routes/v3/project-router.ts @@ -1,15 +1,8 @@ -import { ForbiddenError } from "@casl/ability"; -import crypto from "crypto"; import { z } from "zod"; -import { ProjectMembershipRole, ProjectsSchema } from "@app/db/schemas"; -import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission"; -import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; -import { encryptAsymmetric } from "@app/lib/crypto"; -import { BadRequestError } from "@app/lib/errors"; -import { createWsMembers } from "@app/lib/project"; +import { ProjectMembershipsSchema, ProjectsSchema } from "@app/db/schemas"; +import { EventType } from "@app/ee/services/audit-log/audit-log-types"; import { authRateLimit } from "@app/server/config/rateLimiter"; -import { ActorType } from "@app/services/auth/auth-type"; const projectWithEnv = ProjectsSchema.merge( z.object({ @@ -29,7 +22,6 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { schema: { body: z.object({ projectName: z.string().trim(), - inviteMemberEmails: z.array(z.string().email()).optional(), organizationId: z.string().trim() }), response: { @@ -39,110 +31,13 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { } }, handler: async (req) => { - const { permission } = await server.services.permission.getOrgPermission( - req.permission.type, - req.permission.id, - req.body.organizationId - ); - ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.Workspace); - - // 2. Create a new project (will set the e2ee db field to false). - const { project, ghostUser } = await server.services.project.createProject({ + const project = await server.services.project.createProject({ actorId: req.permission.id, actor: req.permission.type, orgId: req.body.organizationId, workspaceName: req.body.projectName }); - // 3. Create a random key that we'll use as the project key. - const randomBytes = crypto.randomBytes(16).toString("hex"); - - // 4. Encrypt the project key with the users key pair. - const { ciphertext: encryptedProjectKey, nonce: encryptedProjectKeyIv } = encryptAsymmetric( - randomBytes, - ghostUser.keys.publicKey, - ghostUser.keys.plainPrivateKey - ); - - // 4. Save the project key for the ghost user. - await server.services.projectKey.uploadProjectKeys({ - projectId: project.id, - actor: req.permission.type, - actorId: ghostUser.user.id, - nonce: encryptedProjectKeyIv, - receiverId: ghostUser.user.id, - encryptedKey: encryptedProjectKey - }); - - // 5. Create a bot for the project. - const bot = await server.services.projectBot.findBotByProjectId({ - actorId: ghostUser.user.id, - actor: req.permission.type, - projectId: project.id, - - // We set the publicKey and privateKey of the bot to the same as the ghost user. - // We do this because we'll need to access the private key again later, when adding new members to the project. - publicKey: ghostUser.keys.publicKey, - privateKey: ghostUser.keys.plainPrivateKey - }); - - // 6. Activate the bot. - await server.services.projectBot.setBotActiveState({ - botKey: { - encryptedKey: encryptedProjectKey, - nonce: encryptedProjectKeyIv - }, - actorId: ghostUser.user.id, - isActive: true, - actor: req.permission.type, - botId: bot.id - }); - - // 7. get the current user & org membership - const user = await server.services.user.getMe(req.permission.id); - const userOrgMembership = await server.services.permission.getUserOrgPermission(user.id, req.body.organizationId); - - // 7. Get the latest key from the ghost! - const latestKey = await server.services.projectKey.getLatestProjectKey({ - actorId: ghostUser.user.id, - actor: req.permission.type, - projectId: project.id - }); - - if (!latestKey) throw new Error("Failed to get latest key"); - - // If the project is being created by a user, add the user to the project as an admin - if (req.permission.type === ActorType.USER) { - const projectAdmin = createWsMembers({ - decryptKey: latestKey, - members: [ - { - userPublicKey: user.publicKey, - orgMembershipId: userOrgMembership.membership.id, - projectMembershipRole: ProjectMembershipRole.Admin // <-- Make the first user an admin - } - ], - userPrivateKey: ghostUser.keys.plainPrivateKey - }); - - await server.services.projectMembership.addUsersToProject({ - projectId: project.id, - actorId: ghostUser.user.id, - actor: req.permission.type, - members: projectAdmin - }); - } - // If the project is being created by an identity, add the identity to the project as an admin - else if (req.permission.type === ActorType.IDENTITY) { - await server.services.identityProject.createProjectIdentity({ - actor: ActorType.IDENTITY, - actorId: ghostUser.user.id, - identityId: req.permission.id, - projectId: project.id, - role: ProjectMembershipRole.Admin - }); - } - return { project }; } }); @@ -159,82 +54,35 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { }), body: z.object({ emails: z.string().email().array() - }) + }), + response: { + 200: z.object({ + memberships: ProjectMembershipsSchema.array() + }) + } }, handler: async (req) => { - const { permission } = await server.services.permission.getProjectPermission( - req.permission.type, - req.permission.id, - req.params.projectId - ); - ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Member); - - const project = await server.services.project.getAProject({ + const memberships = await server.services.projectMembership.addUsersToProjectNonE2EE({ + projectId: req.params.projectId, actorId: req.permission.id, actor: req.permission.type, - projectId: req.params.projectId - }); - - const ghostUser = await server.services.project.findProjectGhostUser(req.params.projectId); - - if (!ghostUser) { - throw new BadRequestError({ - message: "Failed" // TODO: Add a message - }); - } - - const latestKey = await server.services.projectKey.getLatestProjectKey({ - actorId: ghostUser.id, - actor: ActorType.USER, - projectId: req.params.projectId - }); - - if (!latestKey) { - throw new BadRequestError({ - message: "Failed to find project key" - }); - } - - const bot = await server.services.projectBot.findBotByProjectId({ - actor: req.permission.type, - actorId: req.permission.id, - projectId: req.params.projectId - }); - - // We get the bot private key, because the bot private key is the same as the ghost user's private key. - const botPrivateKey = server.services.projectBot.getBotPrivateKey({ bot }); - - const members = await server.services.org.findOrgMembersByEmail({ - actor: req.permission.type, - actorId: req.permission.id, - orgId: project.orgId, emails: req.body.emails }); - if (members.length !== req.body.emails.length) { - throw new BadRequestError({ - message: "Some users are not part of the organization" - }); - } - - const wsMembers = createWsMembers({ - members: members.map((membership) => ({ - orgMembershipId: membership.id, - projectMembershipRole: ProjectMembershipRole.Member, - userPublicKey: membership.user.publicKey - })), - decryptKey: latestKey, - userPrivateKey: botPrivateKey - }); - - await server.services.projectMembership.addUsersToProject({ + await server.services.auditLog.createAuditLog({ projectId: req.params.projectId, - actorId: ghostUser.id, // We set the actor ID to the ghost user, because this is used as senderId in the project key sharing - actor: ActorType.USER, - members: wsMembers + ...req.auditLogInfo, + event: { + type: EventType.ADD_BATCH_WORKSPACE_MEMBER, + metadata: memberships.map(({ userId, id }) => ({ + userId: userId || "", + membershipId: id, + email: "" + })) + } }); - return {}; + return { memberships }; } }); }; diff --git a/backend/src/services/project-bot/project-bot-service.ts b/backend/src/services/project-bot/project-bot-service.ts index a26170be3..476fba829 100644 --- a/backend/src/services/project-bot/project-bot-service.ts +++ b/backend/src/services/project-bot/project-bot-service.ts @@ -1,17 +1,11 @@ import { ForbiddenError } from "@casl/ability"; import { Knex } from "knex"; -import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas"; +import { SecretKeyEncoding } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; -import { getConfig } from "@app/lib/config/env"; -import { - decryptAsymmetric, - encryptSymmetric, - encryptSymmetric128BitHexKeyUTF8, - generateAsymmetricKeyPair -} from "@app/lib/crypto"; -import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; +import { decryptAsymmetric, generateAsymmetricKeyPair } from "@app/lib/crypto"; +import { infisicalSymmetricDecrypt, infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { BadRequestError } from "@app/lib/errors"; import { TProjectBotDALFactory } from "./project-bot-dal"; @@ -60,7 +54,6 @@ export const projectBotServiceFactory = ({ projectBotDAL, permissionService }: T }: TFindBotByProjectIdDTO) => { const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId, actorOrgId); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations); - const appCfg = getConfig(); const bot = await projectBotDAL.transaction(async (tx) => { const doc = await projectBotDAL.findOne({ projectId }, tx); @@ -68,41 +61,22 @@ export const projectBotServiceFactory = ({ projectBotDAL, permissionService }: T const keys = privateKey && publicKey ? { privateKey, publicKey } : generateAsymmetricKeyPair(); - if (appCfg.ROOT_ENCRYPTION_KEY) { - const { iv, tag, ciphertext } = encryptSymmetric(keys.privateKey, appCfg.ROOT_ENCRYPTION_KEY); - return projectBotDAL.create( - { - name: "Infisical Bot", - projectId, - tag, - iv, - encryptedPrivateKey: ciphertext, - isActive: false, - publicKey: keys.publicKey, - algorithm: SecretEncryptionAlgo.AES_256_GCM, - keyEncoding: SecretKeyEncoding.BASE64 - }, - tx - ); - } - if (appCfg.ENCRYPTION_KEY) { - const { iv, tag, ciphertext } = encryptSymmetric128BitHexKeyUTF8(keys.privateKey, appCfg.ENCRYPTION_KEY); - return projectBotDAL.create( - { - name: "Infisical Bot", - projectId, - tag, - iv, - encryptedPrivateKey: ciphertext, - isActive: false, - publicKey: keys.publicKey, - algorithm: SecretEncryptionAlgo.AES_256_GCM, - keyEncoding: SecretKeyEncoding.UTF8 - }, - tx - ); - } - throw new BadRequestError({ message: "Failed to create bot due to missing encryption key" }); + const { iv, tag, ciphertext, encoding, algorithm } = infisicalSymmetricEncypt(keys.privateKey); + + return projectBotDAL.create( + { + name: "Infisical Bot", + projectId, + tag, + iv, + encryptedPrivateKey: ciphertext, + isActive: false, + publicKey: keys.publicKey, + algorithm, + keyEncoding: encoding + }, + tx + ); }); return bot; }; diff --git a/backend/src/services/project-key/project-key-dal.ts b/backend/src/services/project-key/project-key-dal.ts index 7423a48de..d1b4053d0 100644 --- a/backend/src/services/project-key/project-key-dal.ts +++ b/backend/src/services/project-key/project-key-dal.ts @@ -1,3 +1,5 @@ +import { Knex } from "knex"; + import { TDbClient } from "@app/db"; import { TableName, TProjectKeys } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; @@ -10,10 +12,11 @@ export const projectKeyDALFactory = (db: TDbClient) => { const findLatestProjectKey = async ( userId: string, - projectId: string + projectId: string, + tx?: Knex ): Promise<(TProjectKeys & { sender: { publicKey: string } }) | undefined> => { try { - const projectKey = await db(TableName.ProjectKeys) + const projectKey = await (tx || db)(TableName.ProjectKeys) .join(TableName.Users, `${TableName.ProjectKeys}.senderId`, `${TableName.Users}.id`) .join(TableName.UserEncryptionKey, `${TableName.UserEncryptionKey}.userId`, `${TableName.Users}.id`) .where({ projectId, receiverId: userId }) @@ -29,9 +32,9 @@ export const projectKeyDALFactory = (db: TDbClient) => { } }; - const findAllProjectUserPubKeys = async (projectId: string) => { + const findAllProjectUserPubKeys = async (projectId: string, tx?: Knex) => { try { - const pubKeys = await db(TableName.ProjectMembership) + const pubKeys = await (tx || db)(TableName.ProjectMembership) .where({ projectId }) .join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`) .join(TableName.UserEncryptionKey, `${TableName.Users}.id`, `${TableName.UserEncryptionKey}.userId`) diff --git a/backend/src/services/project-membership/project-membership-service.ts b/backend/src/services/project-membership/project-membership-service.ts index 0f628712b..584b7699d 100644 --- a/backend/src/services/project-membership/project-membership-service.ts +++ b/backend/src/services/project-membership/project-membership-service.ts @@ -1,16 +1,26 @@ /* eslint-disable no-await-in-loop */ import { ForbiddenError } from "@casl/ability"; -import { OrgMembershipStatus, ProjectMembershipRole, TableName, TUsers } from "@app/db/schemas"; +import { + OrgMembershipStatus, + ProjectMembershipRole, + SecretKeyEncoding, + TableName, + TProjectMemberships, + TUsers +} from "@app/db/schemas"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; import { getConfig } from "@app/lib/config/env"; +import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; import { BadRequestError } from "@app/lib/errors"; import { groupBy } from "@app/lib/fn"; +import { createWsMembers } from "@app/lib/project"; import { TOrgDALFactory } from "../org/org-dal"; import { TProjectDALFactory } from "../project/project-dal"; +import { TProjectBotDALFactory } from "../project-bot/project-bot-dal"; import { TProjectKeyDALFactory } from "../project-key/project-key-dal"; import { TProjectRoleDALFactory } from "../project-role/project-role-dal"; import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service"; @@ -18,6 +28,7 @@ import { TUserDALFactory } from "../user/user-dal"; import { TProjectMembershipDALFactory } from "./project-membership-dal"; import { TAddUsersToWorkspaceDTO, + TAddUsersToWorkspaceNonE2EEDTO, TDeleteProjectMembershipDTO, TGetProjectMembershipDTO, TInviteUserToProjectDTO, @@ -27,11 +38,12 @@ import { type TProjectMembershipServiceFactoryDep = { permissionService: Pick; smtpService: TSmtpService; + projectBotDAL: TProjectBotDALFactory; projectMembershipDAL: TProjectMembershipDALFactory; userDAL: Pick; projectRoleDAL: Pick; - orgDAL: Pick; - projectDAL: Pick; + orgDAL: Pick; + projectDAL: Pick; projectKeyDAL: Pick; licenseService: Pick; }; @@ -43,6 +55,7 @@ export const projectMembershipServiceFactory = ({ projectMembershipDAL, smtpService, projectRoleDAL, + projectBotDAL, orgDAL, userDAL, projectDAL, @@ -98,15 +111,12 @@ export const projectMembershipServiceFactory = ({ role: ProjectMembershipRole.Member }); - const sender = await userDAL.findById(actorId); const appCfg = getConfig(); await smtpService.sendMail({ template: SmtpTemplates.WorkspaceInvite, subjectLine: "Infisical workspace invitation", recipients: [invitee.email], substitutions: { - inviterFirstName: sender.firstName, - inviterEmail: sender.email, workspaceName: project.name, callback_url: `${appCfg.SITE_URL}/login` } @@ -175,15 +185,12 @@ export const projectMembershipServiceFactory = ({ }); if (sendEmails) { - const sender = await userDAL.findById(actorId); const appCfg = getConfig(); await smtpService.sendMail({ template: SmtpTemplates.WorkspaceInvite, subjectLine: "Infisical workspace invitation", recipients: orgMembers.map(({ email }) => email).filter(Boolean), substitutions: { - inviterFirstName: sender.firstName, - inviterEmail: sender.email, workspaceName: project.name, callback_url: `${appCfg.SITE_URL}/login` } @@ -192,6 +199,117 @@ export const projectMembershipServiceFactory = ({ return orgMembers; }; + const addUsersToProjectNonE2EE = async ({ + projectId, + actorId, + actor, + emails, + sendEmails = true + }: TAddUsersToWorkspaceNonE2EEDTO) => { + const project = await projectDAL.findById(projectId); + if (!project) throw new BadRequestError({ message: "Project not found" }); + + const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId); + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Member); + + const orgMembers = await orgDAL.findOrgMembersByEmail(project.orgId, emails); + + if (orgMembers.length !== emails.length) throw new BadRequestError({ message: "Some users are not part of org" }); + + const existingMembers = await projectMembershipDAL.find({ + projectId, + $in: { userId: orgMembers.map(({ user }) => user.id).filter(Boolean) } + }); + if (existingMembers.length) throw new BadRequestError({ message: "Some users are already part of project" }); + + const ghostUser = await projectDAL.findProjectGhostUser(projectId); + + if (!ghostUser) { + throw new BadRequestError({ + message: "Failed to find ghost user" + }); + } + + const ghostUserLatestKey = await projectKeyDAL.findLatestProjectKey(ghostUser.id, projectId); + + if (!ghostUserLatestKey) { + throw new BadRequestError({ + message: "Failed to find ghost user latest key" + }); + } + + const bot = await projectBotDAL.findOne({ projectId }); + + if (!bot) { + throw new BadRequestError({ + message: "Failed to find bot" + }); + } + + const botPrivateKey = infisicalSymmetricDecrypt({ + keyEncoding: bot.keyEncoding as SecretKeyEncoding, + iv: bot.iv, + tag: bot.tag, + ciphertext: bot.encryptedPrivateKey + }); + + const newWsMembers = createWsMembers({ + decryptKey: ghostUserLatestKey, + userPrivateKey: botPrivateKey, + members: orgMembers.map((membership) => ({ + orgMembershipId: membership.id, + projectMembershipRole: ProjectMembershipRole.Member, + userPublicKey: membership.user.publicKey + })) + }); + + const members: TProjectMemberships[] = []; + + await projectMembershipDAL.transaction(async (tx) => { + const result = await projectMembershipDAL.insertMany( + orgMembers.map(({ user, id: membershipId }) => { + const role = + orgMembers.find((membership) => membership.id === membershipId)?.role || ProjectMembershipRole.Member; + + return { + projectId, + userId: user.id, + role + }; + }), + tx + ); + + members.push(...result); + + const encKeyGroupByOrgMembId = groupBy(newWsMembers, (i) => i.orgMembershipId); + await projectKeyDAL.insertMany( + orgMembers.map(({ user, id }) => ({ + encryptedKey: encKeyGroupByOrgMembId[id][0].workspaceEncryptedKey, + nonce: encKeyGroupByOrgMembId[id][0].workspaceEncryptedNonce, + senderId: ghostUser.id, + receiverId: user.id, + projectId + })), + tx + ); + }); + + if (sendEmails) { + const appCfg = getConfig(); + await smtpService.sendMail({ + template: SmtpTemplates.WorkspaceInvite, + subjectLine: "Infisical workspace invitation", + recipients: orgMembers.map(({ user }) => user.email).filter(Boolean), + substitutions: { + workspaceName: project.name, + callback_url: `${appCfg.SITE_URL}/login` + } + }); + } + return members; + }; + const updateProjectMembership = async ({ actorId, actor, @@ -268,6 +386,7 @@ export const projectMembershipServiceFactory = ({ getProjectMemberships, inviteUserToProject, updateProjectMembership, + addUsersToProjectNonE2EE, deleteProjectMembership, addUsersToProject }; diff --git a/backend/src/services/project-membership/project-membership-types.ts b/backend/src/services/project-membership/project-membership-types.ts index 786072d56..4928ad522 100644 --- a/backend/src/services/project-membership/project-membership-types.ts +++ b/backend/src/services/project-membership/project-membership-types.ts @@ -25,3 +25,8 @@ export type TAddUsersToWorkspaceDTO = { projectRole: ProjectMembershipRole; }[]; } & TProjectPermission; + +export type TAddUsersToWorkspaceNonE2EEDTO = { + sendEmails?: boolean; + emails: string[]; +} & TProjectPermission; diff --git a/backend/src/services/project/project-dal.ts b/backend/src/services/project/project-dal.ts index 44ba57481..4d7c6072e 100644 --- a/backend/src/services/project/project-dal.ts +++ b/backend/src/services/project/project-dal.ts @@ -52,6 +52,20 @@ export const projectDALFactory = (db: TDbClient) => { } }; + const findProjectGhostUser = async (projectId: string) => { + try { + const ghostUser = await db(TableName.ProjectMembership) + .where({ projectId }) + .join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`) + .select(selectAllTableCols(TableName.Users)) + .where({ ghost: true }) + .first(); + return ghostUser; + } catch (error) { + throw new DatabaseError({ error, name: "Find project ghost user" }); + } + }; + const findAllProjectsByIdentity = async (identityId: string) => { try { const workspaces = await db(TableName.IdentityProjectMembership) @@ -136,6 +150,7 @@ export const projectDALFactory = (db: TDbClient) => { ...projectOrm, findAllProjects, findAllProjectsByIdentity, + findProjectGhostUser, findProjectById }; }; diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index 080278322..26fb5056f 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -6,16 +6,25 @@ import { TLicenseServiceFactory } from "@app/ee/services/license/license-service import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { isAtLeastAsPrivileged } from "@app/lib/casl"; import { getConfig } from "@app/lib/config/env"; import { createSecretBlindIndex } from "@app/lib/crypto"; -import { BadRequestError } from "@app/lib/errors"; +import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; +import { BadRequestError, ForbiddenRequestError } from "@app/lib/errors"; import { alphaNumericNanoId } from "@app/lib/nanoid"; +import { createWorkspaceKey, createWsMembers } from "@app/lib/project"; +import { ActorType } from "../auth/auth-type"; +import { TIdentityOrgDALFactory } from "../identity/identity-org-dal"; +import { TIdentityProjectDALFactory } from "../identity-project/identity-project-dal"; import { TOrgServiceFactory } from "../org/org-service"; +import { TProjectBotDALFactory } from "../project-bot/project-bot-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; +import { TProjectKeyDALFactory } from "../project-key/project-key-dal"; import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-index-dal"; import { ROOT_FOLDER_NAME, TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TUserDALFactory } from "../user/user-dal"; import { TProjectDALFactory } from "./project-dal"; import { TCreateProjectDTO, TDeleteProjectDTO, TGetProjectDTO } from "./project-types"; @@ -27,8 +36,13 @@ export const DEFAULT_PROJECT_ENVS = [ type TProjectServiceFactoryDep = { projectDAL: TProjectDALFactory; + userDAL: TUserDALFactory; folderDAL: Pick; projectEnvDAL: Pick; + identityOrgMembershipDAL: TIdentityOrgDALFactory; + identityProjectDAL: TIdentityProjectDALFactory; + projectKeyDAL: Pick; + projectBotDAL: Pick; projectMembershipDAL: Pick; orgService: Pick; secretBlindIndexDAL: Pick; @@ -40,9 +54,14 @@ export type TProjectServiceFactory = ReturnType; export const projectServiceFactory = ({ projectDAL, + projectKeyDAL, permissionService, + userDAL, folderDAL, orgService, + identityProjectDAL, + projectBotDAL, + identityOrgMembershipDAL, secretBlindIndexDAL, projectMembershipDAL, projectEnvDAL, @@ -52,7 +71,12 @@ export const projectServiceFactory = ({ * Create workspace. Make user the admin * */ const createProject = async ({ orgId, actor, actorId, actorOrgId, workspaceName }: TCreateProjectDTO) => { - const { permission } = await permissionService.getOrgPermission(actor, actorId, orgId, actorOrgId); + const { permission, membership: orgMembership } = await permissionService.getOrgPermission( + actor, + actorId, + orgId, + actorOrgId + ); ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.Workspace); const appCfg = getConfig(); @@ -110,14 +134,141 @@ export const projectServiceFactory = ({ envs.map(({ id }) => ({ name: ROOT_FOLDER_NAME, envId: id, version: 1 })), tx ); - // _id for backward compat - return { - project: { - ...project, - environments: envs, - _id: project.id + + // 3. Create a random key that we'll use as the project key. + const { key: encryptedProjectKey, iv: encryptedProjectKeyIv } = createWorkspaceKey({ + publicKey: ghostUser.keys.publicKey, + privateKey: ghostUser.keys.plainPrivateKey + }); + + // 4. Save the project key for the ghost user. + await projectKeyDAL.create( + { + projectId: project.id, + receiverId: ghostUser.user.id, + encryptedKey: encryptedProjectKey, + nonce: encryptedProjectKeyIv, + senderId: ghostUser.user.id }, - ghostUser + tx + ); + + const { iv, tag, ciphertext, encoding, algorithm } = infisicalSymmetricEncypt(ghostUser.keys.plainPrivateKey); + + // 5. Create & a bot for the project + await projectBotDAL.create( + { + name: "Infisical Bot (Ghost)", + projectId: project.id, + tag, + iv, + encryptedPrivateKey: ciphertext, + isActive: true, + publicKey: ghostUser.keys.publicKey, + algorithm, + keyEncoding: encoding + }, + tx + ); + + // Find the ghost users latest key + const latestKey = await projectKeyDAL.findLatestProjectKey(ghostUser.user.id, project.id, tx); + + if (!latestKey) { + throw new Error("Latest key not found for user"); + } + + // If the project is being created by a user, add the user to the project as an admin + if (actor === ActorType.USER) { + // Find public key of user + const user = await userDAL.findUserEncKeyByUserId(actorId); + + if (!user) { + throw new Error("User not found"); + } + + const [projectAdmin] = createWsMembers({ + decryptKey: latestKey, + userPrivateKey: ghostUser.keys.plainPrivateKey, + members: [ + { + userPublicKey: user.publicKey, + orgMembershipId: orgMembership.id, + projectMembershipRole: ProjectMembershipRole.Admin + } + ] + }); + + // Create a membership for the user + await projectMembershipDAL.create( + { + projectId: project.id, + userId: user.id, + role: projectAdmin.projectRole + }, + tx + ); + + // Create a project key for the user + await projectKeyDAL.create( + { + encryptedKey: projectAdmin.workspaceEncryptedKey, + nonce: projectAdmin.workspaceEncryptedNonce, + senderId: ghostUser.user.id, + receiverId: user.id, + projectId: project.id + }, + tx + ); + } + + // If the project is being created by an identity, add the identity to the project as an admin + else if (actor === ActorType.IDENTITY) { + // Find identity org membership + const identityOrgMembership = await identityOrgMembershipDAL.findOne( + { + identityId: actorId, + orgId: project.orgId + }, + tx + ); + + // If identity org membership not found, throw error + if (!identityOrgMembership) { + throw new BadRequestError({ + message: `Failed to find identity with id ${actorId}` + }); + } + + // Get the role permission for the identity + // IS THIS CORRECT? + const { permission: rolePermission, role: customRole } = await permissionService.getOrgPermissionByRole( + ProjectMembershipRole.Admin, + orgId + ); + + const hasPrivilege = isAtLeastAsPrivileged(permission, rolePermission); + if (!hasPrivilege) + throw new ForbiddenRequestError({ + message: "Failed to add identity to project with more privileged role" + }); + const isCustomRole = Boolean(customRole); + + await identityProjectDAL.create( + { + identityId: actorId, + projectId: project.id, + role: isCustomRole ? ProjectMembershipRole.Custom : ProjectMembershipRole.Admin, + roleId: customRole?.id + }, + tx + ); + } + + return { + ...project, + environments: envs, + _id: project.id }; }); diff --git a/backend/src/services/smtp/templates/workspaceInvitation.handlebars b/backend/src/services/smtp/templates/workspaceInvitation.handlebars index 60556555c..39a9b74ba 100644 --- a/backend/src/services/smtp/templates/workspaceInvitation.handlebars +++ b/backend/src/services/smtp/templates/workspaceInvitation.handlebars @@ -1,15 +1,15 @@ - - - - + + + Project Invitation - - + +

Join your team on Infisical

-

{{inviterFirstName}} ({{inviterEmail}}) has invited you to their Infisical project — {{workspaceName}}

+

You have been invited to a new Infisical project — {{workspaceName}}

Join now

What is Infisical?

-

Infisical is an easy-to-use end-to-end encrypted tool that enables developers to sync and manage their secrets and configs.

- +

Infisical is an easy-to-use end-to-end encrypted tool that enables developers to sync and manage their secrets + and configs.

+ \ No newline at end of file diff --git a/frontend/src/hooks/api/workspace/types.ts b/frontend/src/hooks/api/workspace/types.ts index 8b825d062..240a9e858 100644 --- a/frontend/src/hooks/api/workspace/types.ts +++ b/frontend/src/hooks/api/workspace/types.ts @@ -6,6 +6,7 @@ export type Workspace = { e2ee: boolean; autoCapitalization: boolean; environments: WorkspaceEnv[]; + slug: string; }; export type WorkspaceEnv = { diff --git a/frontend/src/pages/org/[id]/overview/index.tsx b/frontend/src/pages/org/[id]/overview/index.tsx index 56cf50af6..1b574611c 100644 --- a/frontend/src/pages/org/[id]/overview/index.tsx +++ b/frontend/src/pages/org/[id]/overview/index.tsx @@ -50,8 +50,8 @@ import { } from "@app/context"; import { withPermission } from "@app/hoc"; import { - // fetchOrgUsers, - // useAddUserToWs, + fetchOrgUsers, + useAddUserToWsNonE2EE, useCreateWorkspace, useRegisterUserAction } from "@app/hooks/api"; @@ -471,7 +471,7 @@ const OrganizationPage = withPermission( const currentOrg = String(router.query.id); const orgWorkspaces = workspaces?.filter((workspace) => workspace.orgId === currentOrg) || []; const { createNotification } = useNotificationContext(); - // const addWsUser = useAddUserToWs(); + const addUsersToProject = useAddUserToWsNonE2EE(); const { popUp, handlePopUpOpen, handlePopUpClose, handlePopUpToggle } = usePopUp([ "addNewWs", @@ -498,10 +498,11 @@ const OrganizationPage = withPermission( const onCreateProject = async ({ name, addMembers }: TAddProjectFormData) => { // type check if (!currentOrg) return; + if (!user) return; try { const { data: { - project: { id: newWorkspaceId } + project: { id: newProjectId } } } = await createWs.mutateAsync({ organizationId: currentOrg, @@ -509,27 +510,19 @@ const OrganizationPage = withPermission( }); if (addMembers) { - // not using hooks because need at this point only - // const orgUsers = await fetchOrgUsers(currentOrg); - // const decryptKey = await fetchUserWsKey(newWorkspaceId); - // await addWsUser.mutateAsync({ - // workspaceId: newWorkspaceId, - // decryptKey, - // userPrivateKey: PRIVATE_KEY, - // members: orgUsers - // .filter( - // ({ status, user: orgUser }) => status === "accepted" && user.email !== orgUser.email - // ) - // .map(({ user: orgUser, id: orgMembershipId }) => ({ - // userPublicKey: orgUser.publicKey, - // orgMembershipId - // })) - // }); + const orgUsers = await fetchOrgUsers(currentOrg); + + await addUsersToProject.mutateAsync({ + emails: orgUsers + .map((member) => member.user.email) + .filter((email) => email !== user.email), + projectId: newProjectId + }); } createNotification({ text: "Workspace created", type: "success" }); handlePopUpClose("addNewWs"); - router.push(`/project/${newWorkspaceId}/secrets/overview`); + router.push(`/project/${newProjectId}/secrets/overview`); } catch (err) { console.error(err); createNotification({ text: "Failed to create workspace", type: "error" }); From 8a11eebab8520c47693f956165ae1ce9b5ad8467 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 6 Feb 2024 19:28:42 +0400 Subject: [PATCH 016/157] Typo --- .../services/project-membership/project-membership-service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/services/project-membership/project-membership-service.ts b/backend/src/services/project-membership/project-membership-service.ts index 584b7699d..e95401d11 100644 --- a/backend/src/services/project-membership/project-membership-service.ts +++ b/backend/src/services/project-membership/project-membership-service.ts @@ -79,7 +79,7 @@ export const projectMembershipServiceFactory = ({ const invitee = await userDAL.findOne({ email }); if (!invitee || !invitee.isAccepted) throw new BadRequestError({ - message: "Faield to validate invitee", + message: "Failed to validate invitee", name: "Invite user to project" }); From f47fdfe386e12a480633c5143d0adfc66d6f4630 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 6 Feb 2024 19:29:09 +0400 Subject: [PATCH 017/157] Update licence-fns.ts --- backend/src/ee/services/license/licence-fns.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/ee/services/license/licence-fns.ts b/backend/src/ee/services/license/licence-fns.ts index b8b02ec54..9b1d4c203 100644 --- a/backend/src/ee/services/license/licence-fns.ts +++ b/backend/src/ee/services/license/licence-fns.ts @@ -9,11 +9,11 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({ _id: null, slug: null, tier: -1, - workspaceLimit: 5000, + workspaceLimit: null, workspacesUsed: 0, - memberLimit: 5000, + memberLimit: null, membersUsed: 0, - environmentLimit: 5000, + environmentLimit: null, environmentsUsed: 0, secretVersioning: true, pitRecovery: false, @@ -28,7 +28,7 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({ status: null, trial_end: null, has_used_trial: true, - secretApproval: true, + secretApproval: false, secretRotation: true }); From f2c36c58f929ef628dae03c670d074d6e860eb09 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 9 Feb 2024 21:23:15 +0400 Subject: [PATCH 018/157] Fix project seed (seeds old projects that can be upgraded) --- backend/src/db/seed-data.ts | 12 +++++++++- backend/src/db/seeds/1-user.ts | 12 ++++++++-- backend/src/db/seeds/3-project.ts | 40 +++++++++++++++++++++++++++---- 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/backend/src/db/seed-data.ts b/backend/src/db/seed-data.ts index 63821e15e..6406e44ab 100644 --- a/backend/src/db/seed-data.ts +++ b/backend/src/db/seed-data.ts @@ -1,3 +1,4 @@ +/* eslint-disable import/no-mutable-exports */ import crypto from "node:crypto"; import argon2, { argon2id } from "argon2"; @@ -15,9 +16,12 @@ import { import { TSecrets, TUserEncryptionKeys } from "./schemas"; +export let userPrivateKey: string | undefined; +export let userPublicKey: string | undefined; + export const seedData1 = { id: "3dafd81d-4388-432b-a4c5-f735616868c1", - email: "test@localhost.local", + email: process.env.TEST_USER_EMAIL || "test@localhost.local", password: process.env.TEST_USER_PASSWORD || "testInfisical@1", organization: { id: "180870b7-f464-4740-8ffe-9d11c9245ea7", @@ -42,6 +46,12 @@ export const seedData1 = { }, token: { id: "a9dfafba-a3b7-42e3-8618-91abb702fd36" + }, + + // We set these values during user creation, and later re-use them during project seeding. + encryptionKeys: { + publicKey: "", + privateKey: "" } }; diff --git a/backend/src/db/seeds/1-user.ts b/backend/src/db/seeds/1-user.ts index af40dbbbc..5f9fcd509 100644 --- a/backend/src/db/seeds/1-user.ts +++ b/backend/src/db/seeds/1-user.ts @@ -3,9 +3,12 @@ import { Knex } from "knex"; +// eslint-disable-next-line @typescript-eslint/no-unused-vars +import { generateUserSrpKeys } from "@app/lib/crypto/srp"; + import { AuthMethod } from "../../services/auth/auth-type"; import { TableName } from "../schemas"; -import { generateUserSrpKeys, seedData1 } from "../seed-data"; +import { seedData1 } from "../seed-data"; export async function seed(knex: Knex): Promise { // Deletes ALL existing entries @@ -34,7 +37,7 @@ export async function seed(knex: Knex): Promise { ]) .returning("*"); - const encKeys = await generateUserSrpKeys(seedData1.password); + const encKeys = await generateUserSrpKeys(seedData1.email, seedData1.password); // password: testInfisical@1 await knex(TableName.UserEncryptionKey).insert([ { @@ -64,4 +67,9 @@ export async function seed(knex: Knex): Promise { refreshVersion: 1, lastUsed: new Date() }); + + seedData1.encryptionKeys = { + publicKey: encKeys.publicKey, + privateKey: encKeys.plainPrivateKey + }; } diff --git a/backend/src/db/seeds/3-project.ts b/backend/src/db/seeds/3-project.ts index fea71b557..af74140dc 100644 --- a/backend/src/db/seeds/3-project.ts +++ b/backend/src/db/seeds/3-project.ts @@ -2,7 +2,9 @@ import crypto from "node:crypto"; import { Knex } from "knex"; -import { encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; +import { getConfig, initEnvConfig } from "@app/lib/config/env"; +import { createSecretBlindIndex, encryptAsymmetric, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; +import { initLogger } from "@app/lib/logger"; import { OrgMembershipRole, SecretEncryptionAlgo, SecretKeyEncoding, TableName } from "../schemas"; import { buildUserProjectKey, getUserPrivateKey, seedData1 } from "../seed-data"; @@ -14,6 +16,8 @@ export const DEFAULT_PROJECT_ENVS = [ ]; export async function seed(knex: Knex): Promise { + initEnvConfig(await initLogger()); + const appCfg = getConfig(); // Deletes ALL existing entries await knex(TableName.Project).del(); await knex(TableName.Environment).del(); @@ -24,12 +28,40 @@ export async function seed(knex: Knex): Promise { name: seedData1.project.name, orgId: seedData1.organization.id, slug: "first-project", - // eslint-disable-next-line - // @ts-ignore - id: seedData1.project.id + // @ts-expect-error exluded type id needs to be inserted here to keep it testable + id: seedData1.project.id, + version: "v1" }) .returning("*"); + const blindIndex = createSecretBlindIndex(appCfg.ROOT_ENCRYPTION_KEY, appCfg.ENCRYPTION_KEY); + + await knex(TableName.SecretBlindIndex).insert({ + projectId: project.id, + algorithm: blindIndex.algorithm, + keyEncoding: blindIndex.keyEncoding, + saltIV: blindIndex.iv, + encryptedSaltCipherText: blindIndex.ciphertext, + saltTag: blindIndex.tag + }); + + const randomBytes = crypto.randomBytes(16).toString("hex"); // Project key + // const encKeys = await generateUserSrpKeys(seedData1.email, seedData1.password); // User keys + + const { ciphertext: encryptedProjectKey, nonce: encryptedProjectKeyIv } = encryptAsymmetric( + randomBytes, + seedData1.encryptionKeys.publicKey, + seedData1.encryptionKeys.privateKey + ); + + await knex(TableName.ProjectKeys).insert({ + projectId: project.id, + senderId: seedData1.id, + receiverId: seedData1.id, + encryptedKey: encryptedProjectKey, + nonce: encryptedProjectKeyIv + }); + await knex(TableName.ProjectMembership).insert({ projectId: project.id, role: OrgMembershipRole.Admin, From 8fd99855bdaa4001c6974be6c31b11165ef0a14b Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 9 Feb 2024 21:23:31 +0400 Subject: [PATCH 019/157] Ghost --- .../migrations/20240202093209_ghost_user.ts | 20 ++++++++++++------- backend/src/db/schemas/models.ts | 5 +++++ backend/src/db/schemas/projects.ts | 2 +- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/backend/src/db/migrations/20240202093209_ghost_user.ts b/backend/src/db/migrations/20240202093209_ghost_user.ts index 7fa23d245..72181cd83 100644 --- a/backend/src/db/migrations/20240202093209_ghost_user.ts +++ b/backend/src/db/migrations/20240202093209_ghost_user.ts @@ -1,31 +1,37 @@ import { Knex } from "knex"; -import { TableName } from "../schemas"; +import { ProjectVersion, TableName } from "../schemas"; export async function up(knex: Knex): Promise { - if (!(await knex.schema.hasColumn(TableName.Users, "ghost"))) { + const hasGhostUserColumn = await knex.schema.hasColumn(TableName.Users, "ghost"); + const hasProjectVersionColumn = await knex.schema.hasColumn(TableName.Project, "version"); + + if (!hasGhostUserColumn) { await knex.schema.alterTable(TableName.Users, (t) => { t.boolean("ghost").defaultTo(false).notNullable(); }); } - if (!(await knex.schema.hasColumn(TableName.Project, "e2ee"))) { + if (!hasProjectVersionColumn) { await knex.schema.alterTable(TableName.Project, (t) => { - t.boolean("e2ee").defaultTo(true).notNullable(); + t.string("version").defaultTo(ProjectVersion.V1).notNullable(); }); } } export async function down(knex: Knex): Promise { - if (await knex.schema.hasColumn(TableName.Users, "ghost")) { + const hasGhostUserColumn = await knex.schema.hasColumn(TableName.Users, "ghost"); + const hasProjectVersionColumn = await knex.schema.hasColumn(TableName.Project, "version"); + + if (hasGhostUserColumn) { await knex.schema.alterTable(TableName.Users, (t) => { t.dropColumn("ghost"); }); } - if (await knex.schema.hasColumn(TableName.Project, "e2ee")) { + if (hasProjectVersionColumn) { await knex.schema.alterTable(TableName.Project, (t) => { - t.dropColumn("e2ee"); + t.dropColumn("version"); }); } } diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index e817d6ed0..4109c516c 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -112,6 +112,11 @@ export enum SecretType { Personal = "personal" } +export enum ProjectVersion { + V1 = "v1", + V2 = "v2" +} + export enum IdentityAuthMethod { Univeral = "universal-auth" } diff --git a/backend/src/db/schemas/projects.ts b/backend/src/db/schemas/projects.ts index 8c20ca703..fceb18388 100644 --- a/backend/src/db/schemas/projects.ts +++ b/backend/src/db/schemas/projects.ts @@ -15,7 +15,7 @@ export const ProjectsSchema = z.object({ orgId: z.string().uuid(), createdAt: z.date(), updatedAt: z.date(), - e2ee: z.boolean().default(true) + version: z.string().default("v1") }); export type TProjects = z.infer; From 0cfab8ab6bd81e347bd17ae6ad2c41d6502c3bee Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 9 Feb 2024 22:04:32 +0400 Subject: [PATCH 020/157] Update index.ts --- backend/src/lib/project/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/lib/project/index.ts b/backend/src/lib/project/index.ts index d4e97ba17..e0ed0d70b 100644 --- a/backend/src/lib/project/index.ts +++ b/backend/src/lib/project/index.ts @@ -40,12 +40,12 @@ export const createWsMembers = ({ members, decryptKey, userPrivateKey }: AddUser return newWsMembers; }; -type TCreateWorkspaceKeyDTO = { +type TCreateProjectKeyDTO = { publicKey: string; privateKey: string; }; -export const createWorkspaceKey = ({ publicKey, privateKey }: TCreateWorkspaceKeyDTO) => { +export const createProjectKey = ({ publicKey, privateKey }: TCreateProjectKeyDTO) => { // 3. Create a random key that we'll use as the project key. const randomBytes = crypto.randomBytes(16).toString("hex"); From 8333250b0b5a8a1e8869c332079df7092af2ef90 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 9 Feb 2024 22:28:37 +0400 Subject: [PATCH 021/157] Ghost user and migration finished! --- .../secret-approval-request-secret-dal.ts | 32 +- backend/src/lib/secret/index.ts | 126 ++++++ backend/src/server/routes/index.ts | 5 + backend/src/server/routes/v1/bot-router.ts | 4 +- backend/src/server/routes/v1/index.ts | 1 + backend/src/server/routes/v2/index.ts | 2 + .../project-membership-router.ts} | 41 +- .../src/server/routes/v2/project-router.ts | 77 +++- backend/src/server/routes/v3/index.ts | 2 - .../project-bot/project-bot-service.ts | 7 +- .../services/project-bot/project-bot-types.ts | 4 + .../src/services/project/project-service.ts | 401 +++++++++++++++++- backend/src/services/project/project-types.ts | 6 + backend/src/services/secret/secret-dal.ts | 34 +- .../src/services/secret/secret-version-dal.ts | 52 ++- .../utilities/secrets/encryptSecrets.ts | 2 - frontend/src/hooks/api/keys/queries.tsx | 4 +- frontend/src/hooks/api/users/mutation.tsx | 2 +- frontend/src/hooks/api/workspace/index.tsx | 3 +- frontend/src/hooks/api/workspace/queries.tsx | 17 +- frontend/src/hooks/api/workspace/types.ts | 2 +- frontend/src/layouts/AppLayout/AppLayout.tsx | 57 +-- .../MemberListTab/MemberListTab.tsx | 9 +- .../SecretOverviewPage/SecretOverviewPage.tsx | 43 +- 24 files changed, 810 insertions(+), 123 deletions(-) create mode 100644 backend/src/lib/secret/index.ts rename backend/src/server/routes/{v3/project-router.ts => v2/project-membership-router.ts} (55%) diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts index 9b4742255..93e5a2538 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts @@ -1,8 +1,14 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; -import { SecretApprovalRequestsSecretsSchema, TableName, TSecretTags } from "@app/db/schemas"; -import { DatabaseError } from "@app/lib/errors"; +import { + SecretApprovalRequestsSecretsSchema, + TableName, + TSecretApprovalRequestsSecrets, + TSecretApprovalRequestsSecretsUpdate, + TSecretTags +} from "@app/db/schemas"; +import { BadRequestError, DatabaseError } from "@app/lib/errors"; import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; export type TSecretApprovalRequestSecretDALFactory = ReturnType; @@ -11,6 +17,27 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => { const secretApprovalRequestSecretOrm = ormify(db, TableName.SecretApprovalRequestSecret); const secretApprovalRequestSecretTagOrm = ormify(db, TableName.SecretApprovalRequestSecretTag); + const bulkUpdateNoVersionIncrement = async ( + data: Array<{ filter: Partial; data: TSecretApprovalRequestsSecretsUpdate }>, + tx?: Knex + ) => { + try { + const secs = await Promise.all( + data.map(async ({ filter, data: updateData }) => { + const [doc] = await (tx || db)(TableName.SecretApprovalRequestSecret) + .where(filter) + .update(updateData) + .returning("*"); + if (!doc) throw new BadRequestError({ message: "Failed to update document" }); + return doc; + }) + ); + return secs; + } catch (error) { + throw new DatabaseError({ error, name: "bulk update secret" }); + } + }; + const findByRequestId = async (requestId: string, tx?: Knex) => { try { const doc = await (tx || db)({ @@ -190,6 +217,7 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => { return { ...secretApprovalRequestSecretOrm, findByRequestId, + bulkUpdateNoVersionIncrement, insertApprovalSecretTags: secretApprovalRequestSecretTagOrm.insertMany }; }; diff --git a/backend/src/lib/secret/index.ts b/backend/src/lib/secret/index.ts new file mode 100644 index 000000000..08cb0163d --- /dev/null +++ b/backend/src/lib/secret/index.ts @@ -0,0 +1,126 @@ +import { z } from "zod"; + +import { SecretKeyEncoding, TProjectKeys } from "@app/db/schemas"; + +import { decryptAsymmetric, decryptSymmetric } from "../crypto"; +import { decryptSymmetric128BitHexKeyUTF8, TDecryptSymmetricInput } from "../crypto/encryption"; + +export enum SecretDocType { + Secret = "secret", + SecretVersion = "secretVersion", + ApprovalSecret = "approvalSecret" +} + +const PartialSecretSchema = z.object({ + id: z.string(), + secretKeyCiphertext: z.string(), + secretKeyIV: z.string(), + secretKeyTag: z.string(), + + secretValueCiphertext: z.string(), + secretValueIV: z.string(), + secretValueTag: z.string(), + + secretCommentCiphertext: z.string().nullish(), + secretCommentIV: z.string().nullish(), + secretCommentTag: z.string().nullish(), + + docType: z.nativeEnum(SecretDocType), + + keyEncoding: z.string() +}); + +const PartialDecryptedSecretSchema = z.object({ + id: z.string(), + secretKey: z.string(), + secretValue: z.string(), + secretComment: z.string().optional(), + + docType: z.nativeEnum(SecretDocType) +}); + +export type TPartialSecret = z.infer; +export type TPartialDecryptedSecret = z.infer; + +const symmetricDecrypt = ({ + keyEncoding, + ciphertext, + tag, + iv, + key, + isApprovalSecret +}: TDecryptSymmetricInput & { keyEncoding: SecretKeyEncoding; isApprovalSecret: boolean }) => { + if (keyEncoding === SecretKeyEncoding.UTF8 || isApprovalSecret) { + const data = decryptSymmetric128BitHexKeyUTF8({ key, iv, tag, ciphertext }); + return data; + } + if (keyEncoding === SecretKeyEncoding.BASE64) { + const data = decryptSymmetric({ key, iv, tag, ciphertext }); + return data; + } + + throw new Error("Missing both encryption keys"); +}; + +export const decryptSecrets = ( + encryptedSecrets: TPartialSecret[], + privateKey: string, + latestKey: TProjectKeys & { + sender: { + publicKey: string; + }; + } +) => { + const key = decryptAsymmetric({ + ciphertext: latestKey.encryptedKey, + nonce: latestKey.nonce, + publicKey: latestKey.sender.publicKey, + privateKey + }); + + const secrets: TPartialDecryptedSecret[] = []; + + encryptedSecrets.forEach((encSecret) => { + const secretKey = symmetricDecrypt({ + ciphertext: encSecret.secretKeyCiphertext, + iv: encSecret.secretKeyIV, + tag: encSecret.secretKeyTag, + key, + keyEncoding: encSecret.keyEncoding as SecretKeyEncoding, + isApprovalSecret: encSecret.docType === SecretDocType.ApprovalSecret + }); + + const secretValue = symmetricDecrypt({ + ciphertext: encSecret.secretValueCiphertext, + iv: encSecret.secretValueIV, + tag: encSecret.secretValueTag, + key, + keyEncoding: encSecret.keyEncoding as SecretKeyEncoding, + isApprovalSecret: encSecret.docType === SecretDocType.ApprovalSecret + }); + + const secretComment = + encSecret.secretCommentCiphertext && encSecret.secretCommentIV && encSecret.secretCommentTag + ? symmetricDecrypt({ + ciphertext: encSecret.secretCommentCiphertext, + iv: encSecret.secretCommentIV, + tag: encSecret.secretCommentTag, + key, + keyEncoding: encSecret.keyEncoding as SecretKeyEncoding, + isApprovalSecret: encSecret.docType === SecretDocType.ApprovalSecret + }) + : ""; + + const decryptedSecret: TPartialDecryptedSecret = { + id: encSecret.id, + secretKey, + secretValue, + secretComment, + docType: encSecret.docType + }; + + secrets.push(decryptedSecret); + }); + + return secrets; +}; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index e793d2133..068142ab8 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -306,7 +306,12 @@ export const registerRoutes = async ( identityProjectDAL, identityOrgMembershipDAL, projectBotDAL, + secretDAL, + orgDAL, + secretApprovalRequestDAL, + secretApprovalSecretDAL: sarSecretDAL, projectKeyDAL, + secretVersionDAL, userDAL, projectEnvDAL, orgService, diff --git a/backend/src/server/routes/v1/bot-router.ts b/backend/src/server/routes/v1/bot-router.ts index ff451a17e..20acf95ec 100644 --- a/backend/src/server/routes/v1/bot-router.ts +++ b/backend/src/server/routes/v1/bot-router.ts @@ -33,7 +33,7 @@ export const registerProjectBotRouter = async (server: FastifyZodProvider) => { projectId: req.params.projectId }); - if (!project.e2ee) { + if (project.version === "v2") { throw new BadRequestError({ message: "Failed to find bot, project has E2EE disabled" }); } @@ -79,7 +79,7 @@ export const registerProjectBotRouter = async (server: FastifyZodProvider) => { handler: async (req) => { const project = await server.services.projectBot.findProjectByBotId(req.params.botId); - if (project?.e2ee === false) { + if (project?.version === "v2") { throw new BadRequestError({ message: "Failed to set bot active, project has E2EE disabled" }); } diff --git a/backend/src/server/routes/v1/index.ts b/backend/src/server/routes/v1/index.ts index 744ba6ebf..fbc68d974 100644 --- a/backend/src/server/routes/v1/index.ts +++ b/backend/src/server/routes/v1/index.ts @@ -48,6 +48,7 @@ export const registerV1Routes = async (server: FastifyZodProvider) => { await projectRouter.register(registerProjectMembershipRouter); await projectRouter.register(registerSecretTagRouter); }, + { prefix: "/workspace" } ); diff --git a/backend/src/server/routes/v2/index.ts b/backend/src/server/routes/v2/index.ts index 1f423c084..deebbc981 100644 --- a/backend/src/server/routes/v2/index.ts +++ b/backend/src/server/routes/v2/index.ts @@ -2,6 +2,7 @@ import { registerIdentityOrgRouter } from "./identity-org-router"; import { registerIdentityProjectRouter } from "./identity-project-router"; import { registerMfaRouter } from "./mfa-router"; import { registerOrgRouter } from "./organization-router"; +import { registerProjectMembershipRouter } from "./project-membership-router"; import { registerProjectRouter } from "./project-router"; import { registerServiceTokenRouter } from "./service-token-router"; import { registerUserRouter } from "./user-router"; @@ -21,6 +22,7 @@ export const registerV2Routes = async (server: FastifyZodProvider) => { async (projectServer) => { await projectServer.register(registerProjectRouter); await projectServer.register(registerIdentityProjectRouter); + await projectServer.register(registerProjectMembershipRouter); }, { prefix: "/workspace" } ); diff --git a/backend/src/server/routes/v3/project-router.ts b/backend/src/server/routes/v2/project-membership-router.ts similarity index 55% rename from backend/src/server/routes/v3/project-router.ts rename to backend/src/server/routes/v2/project-membership-router.ts index c1759f718..733147b00 100644 --- a/backend/src/server/routes/v3/project-router.ts +++ b/backend/src/server/routes/v2/project-membership-router.ts @@ -1,47 +1,10 @@ import { z } from "zod"; -import { ProjectMembershipsSchema, ProjectsSchema } from "@app/db/schemas"; +import { ProjectMembershipsSchema } from "@app/db/schemas"; import { EventType } from "@app/ee/services/audit-log/audit-log-types"; import { authRateLimit } from "@app/server/config/rateLimiter"; -const projectWithEnv = ProjectsSchema.merge( - z.object({ - _id: z.string(), - environments: z.object({ name: z.string(), slug: z.string(), id: z.string() }).array() - }) -); - -export const registerProjectRouter = async (server: FastifyZodProvider) => { - /* Create new project */ - server.route({ - method: "POST", - url: "/", - config: { - rateLimit: authRateLimit - }, - schema: { - body: z.object({ - projectName: z.string().trim(), - organizationId: z.string().trim() - }), - response: { - 200: z.object({ - project: projectWithEnv - }) - } - }, - handler: async (req) => { - const project = await server.services.project.createProject({ - actorId: req.permission.id, - actor: req.permission.type, - orgId: req.body.organizationId, - workspaceName: req.body.projectName - }); - - return { project }; - } - }); - +export const registerProjectMembershipRouter = async (server: FastifyZodProvider) => { server.route({ method: "POST", url: "/:projectId/memberships", diff --git a/backend/src/server/routes/v2/project-router.ts b/backend/src/server/routes/v2/project-router.ts index d38efdbb9..bc0282850 100644 --- a/backend/src/server/routes/v2/project-router.ts +++ b/backend/src/server/routes/v2/project-router.ts @@ -1,13 +1,22 @@ import { z } from "zod"; -import { ProjectKeysSchema } from "@app/db/schemas"; +import { ProjectKeysSchema, ProjectsSchema } from "@app/db/schemas"; import { EventType } from "@app/ee/services/audit-log/audit-log-types"; +import { authRateLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; +const projectWithEnv = ProjectsSchema.merge( + z.object({ + _id: z.string(), + environments: z.object({ name: z.string(), slug: z.string(), id: z.string() }).array() + }) +); + export const registerProjectRouter = async (server: FastifyZodProvider) => { + /* Get project key */ server.route({ - url: "/:workspaceId/encrypted-key", + url: "/:projectId/encrypted-key", method: "GET", schema: { description: "Return encrypted project key", @@ -17,7 +26,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { } ], params: z.object({ - workspaceId: z.string().trim() + projectId: z.string().trim() }), response: { 200: ProjectKeysSchema.merge( @@ -34,13 +43,13 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { const key = await server.services.projectKey.getLatestProjectKey({ actor: req.permission.type, actorId: req.permission.id, - projectId: req.params.workspaceId, - actorOrgId: req.permission.orgId + actorOrgId: req.permission.orgId, + projectId: req.params.projectId }); await server.services.auditLog.createAuditLog({ ...req.auditLogInfo, - projectId: req.params.workspaceId, + projectId: req.params.projectId, event: { type: EventType.GET_WORKSPACE_KEY, metadata: { @@ -52,4 +61,60 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { return key; } }); + + server.route({ + url: "/:projectId/upgrade", + method: "POST", + schema: { + params: z.object({ + projectId: z.string().trim() + }), + + body: z.object({ + userPrivateKey: z.string().trim() + }), + response: { + 200: z.object({}) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY]), + handler: async (req) => { + await server.services.project.upgradeProject({ + actorId: req.permission.id, + actor: req.permission.type, + projectId: req.params.projectId, + userPrivateKey: req.body.userPrivateKey + }); + } + }); + + /* Create new project */ + server.route({ + method: "POST", + url: "/", + config: { + rateLimit: authRateLimit + }, + schema: { + body: z.object({ + projectName: z.string().trim(), + organizationId: z.string().trim() + }), + response: { + 200: z.object({ + project: projectWithEnv + }) + } + }, + handler: async (req) => { + const project = await server.services.project.createProject({ + actorId: req.permission.id, + actor: req.permission.type, + orgId: req.body.organizationId, + workspaceName: req.body.projectName + }); + + return { project }; + } + }); }; diff --git a/backend/src/server/routes/v3/index.ts b/backend/src/server/routes/v3/index.ts index 21d57d359..6a061a635 100644 --- a/backend/src/server/routes/v3/index.ts +++ b/backend/src/server/routes/v3/index.ts @@ -1,5 +1,4 @@ import { registerLoginRouter } from "./login-router"; -import { registerProjectRouter } from "./project-router"; import { registerSecretBlindIndexRouter } from "./secret-blind-index-router"; import { registerSecretRouter } from "./secret-router"; import { registerSignupRouter } from "./signup-router"; @@ -11,5 +10,4 @@ export const registerV3Routes = async (server: FastifyZodProvider) => { await server.register(registerUserRouter, { prefix: "/users" }); await server.register(registerSecretRouter, { prefix: "/secrets" }); await server.register(registerSecretBlindIndexRouter, { prefix: "/workspaces" }); - await server.register(registerProjectRouter, { prefix: "/projects" }); }; diff --git a/backend/src/services/project-bot/project-bot-service.ts b/backend/src/services/project-bot/project-bot-service.ts index 476fba829..e95ab1610 100644 --- a/backend/src/services/project-bot/project-bot-service.ts +++ b/backend/src/services/project-bot/project-bot-service.ts @@ -50,6 +50,7 @@ export const projectBotServiceFactory = ({ projectBotDAL, permissionService }: T projectId, actorOrgId, privateKey, + botKey, publicKey }: TFindBotByProjectIdDTO) => { const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId, actorOrgId); @@ -73,7 +74,11 @@ export const projectBotServiceFactory = ({ projectBotDAL, permissionService }: T isActive: false, publicKey: keys.publicKey, algorithm, - keyEncoding: encoding + keyEncoding: encoding, + ...(botKey && { + encryptedProjectKey: botKey.encryptedKey, + encryptedProjectKeyNonce: botKey.nonce + }) }, tx ); diff --git a/backend/src/services/project-bot/project-bot-types.ts b/backend/src/services/project-bot/project-bot-types.ts index 4431c6686..47e3885c7 100644 --- a/backend/src/services/project-bot/project-bot-types.ts +++ b/backend/src/services/project-bot/project-bot-types.ts @@ -14,6 +14,10 @@ export type TSetActiveStateDTO = { export type TFindBotByProjectIdDTO = { privateKey?: string; publicKey?: string; + botKey?: { + nonce: string; + encryptedKey: string; + }; } & TProjectPermission; export type TGetPrivateKeyDTO = { diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index 26fb5056f..9b079576b 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -1,32 +1,46 @@ +/* eslint-disable no-console */ +/* eslint-disable no-await-in-loop */ import { ForbiddenError } from "@casl/ability"; import slugify from "@sindresorhus/slugify"; -import { ProjectMembershipRole } from "@app/db/schemas"; +import { ProjectMembershipRole, ProjectVersion, SecretKeyEncoding, TSecrets } from "@app/db/schemas"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; +import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal"; +import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; +import { RequestState } from "@app/ee/services/secret-approval-request/secret-approval-request-types"; import { isAtLeastAsPrivileged } from "@app/lib/casl"; import { getConfig } from "@app/lib/config/env"; import { createSecretBlindIndex } from "@app/lib/crypto"; -import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; +import { + decryptAsymmetric, + encryptSymmetric128BitHexKeyUTF8, + infisicalSymmetricDecrypt, + infisicalSymmetricEncypt +} from "@app/lib/crypto/encryption"; import { BadRequestError, ForbiddenRequestError } from "@app/lib/errors"; import { alphaNumericNanoId } from "@app/lib/nanoid"; -import { createWorkspaceKey, createWsMembers } from "@app/lib/project"; +import { createProjectKey, createWsMembers } from "@app/lib/project"; +import { decryptSecrets, SecretDocType, TPartialSecret } from "@app/lib/secret"; import { ActorType } from "../auth/auth-type"; import { TIdentityOrgDALFactory } from "../identity/identity-org-dal"; import { TIdentityProjectDALFactory } from "../identity-project/identity-project-dal"; +import { TOrgDALFactory } from "../org/org-dal"; import { TOrgServiceFactory } from "../org/org-service"; import { TProjectBotDALFactory } from "../project-bot/project-bot-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectKeyDALFactory } from "../project-key/project-key-dal"; import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; +import { TSecretDALFactory } from "../secret/secret-dal"; +import { TSecretVersionDALFactory } from "../secret/secret-version-dal"; import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-index-dal"; import { ROOT_FOLDER_NAME, TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TUserDALFactory } from "../user/user-dal"; import { TProjectDALFactory } from "./project-dal"; -import { TCreateProjectDTO, TDeleteProjectDTO, TGetProjectDTO } from "./project-types"; +import { TCreateProjectDTO, TDeleteProjectDTO, TGetProjectDTO, TUpgradeProjectDTO } from "./project-types"; export const DEFAULT_PROJECT_ENVS = [ { name: "Development", slug: "dev" }, @@ -37,16 +51,21 @@ export const DEFAULT_PROJECT_ENVS = [ type TProjectServiceFactoryDep = { projectDAL: TProjectDALFactory; userDAL: TUserDALFactory; - folderDAL: Pick; - projectEnvDAL: Pick; + folderDAL: TSecretFolderDALFactory; + projectEnvDAL: Pick; + secretVersionDAL: TSecretVersionDALFactory; identityOrgMembershipDAL: TIdentityOrgDALFactory; identityProjectDAL: TIdentityProjectDALFactory; - projectKeyDAL: Pick; - projectBotDAL: Pick; - projectMembershipDAL: Pick; - orgService: Pick; + projectKeyDAL: Pick; + projectBotDAL: Pick; + projectMembershipDAL: Pick; + orgDAL: TOrgDALFactory; + secretApprovalRequestDAL: TSecretApprovalRequestDALFactory; + secretApprovalSecretDAL: TSecretApprovalRequestSecretDALFactory; secretBlindIndexDAL: Pick; permissionService: TPermissionServiceFactory; + orgService: Pick; + secretDAL: TSecretDALFactory; licenseService: Pick; }; @@ -55,13 +74,18 @@ export type TProjectServiceFactory = ReturnType; export const projectServiceFactory = ({ projectDAL, projectKeyDAL, + secretApprovalRequestDAL, + secretApprovalSecretDAL, permissionService, userDAL, folderDAL, orgService, + orgDAL, identityProjectDAL, + secretVersionDAL, projectBotDAL, identityOrgMembershipDAL, + secretDAL, secretBlindIndexDAL, projectMembershipDAL, projectEnvDAL, @@ -99,7 +123,7 @@ export const projectServiceFactory = ({ name: workspaceName, orgId, slug: slugify(`${workspaceName}-${alphaNumericNanoId(4)}`), - e2ee: false + version: ProjectVersion.V2 }, tx ); @@ -136,7 +160,7 @@ export const projectServiceFactory = ({ ); // 3. Create a random key that we'll use as the project key. - const { key: encryptedProjectKey, iv: encryptedProjectKeyIv } = createWorkspaceKey({ + const { key: encryptedProjectKey, iv: encryptedProjectKeyIv } = createProjectKey({ publicKey: ghostUser.keys.publicKey, privateKey: ghostUser.keys.plainPrivateKey }); @@ -162,9 +186,12 @@ export const projectServiceFactory = ({ projectId: project.id, tag, iv, + encryptedProjectKey, + encryptedProjectKeyNonce: encryptedProjectKeyIv, encryptedPrivateKey: ciphertext, isActive: true, publicKey: ghostUser.keys.publicKey, + senderId: ghostUser.user.id, algorithm, keyEncoding: encoding }, @@ -321,6 +348,353 @@ export const projectServiceFactory = ({ return updatedProject; }; + const upgradeProject = async ({ projectId, actor, actorId, userPrivateKey }: TUpgradeProjectDTO) => { + const { permission, membership } = await permissionService.getProjectPermission(actor, actorId, projectId); + + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Delete, ProjectPermissionSub.Project); + + if (membership?.role !== ProjectMembershipRole.Admin) { + throw new ForbiddenRequestError({ + message: "User must be admin" + }); + } + + /* + 1. Get the existing project + 2. Get the existing project keys + 4. Get all the project envs & folders + 5. Get ALL secrets within the project + 6. Create a new ghost user + 7. Create a project membership for the ghost user + 8. Get the existing bot, and the existing project keys for the members of the project + 9. IF a bot already exists for the project, delete it! + 10. Delete all the existing project keys + 11. Create a project key for the ghost user + 12. Find the newly created ghost user's latest key + + + FOR EACH OF THE OLD PROJECT KEYS (loop): + 13. Find the user based on the key.receiverId. + 14. Find the org membership for the user. + 15. Create a new project key for the user. + + + 16. Encrypt the ghost user's private key + 17. Create a new bot, and set the public/private key of the bot, to the ghost user's public/private key. + 18. Add the workspace key to the bot + 19. Decrypt the secrets with the old project key + 20. Get the newly created bot's private key, and workspace key (we do it this way to test as many steps of the bot process as possible) + 21. Get the workspace key from the bot + + + FOR EACH DECRYPTED SECRET (loop): + 22. Re-encrypt the secret value, secret key, and secret comment with the NEW project key from the bot. + 23. Update the secret in the database with the new encrypted values. + + + 24. Transaction ends. If there were no errors. All changes are applied. + 25. API route returns 200 OK. + */ + + const project = await projectDAL.findOne({ id: projectId, version: ProjectVersion.V1 }); + const oldProjectKey = await projectKeyDAL.findLatestProjectKey(actorId, projectId); + if (!project || !oldProjectKey) { + throw new BadRequestError({ + message: "Project or project key not found" + }); + } + + const projectEnvs = await projectEnvDAL.find({ + projectId: project.id + }); + + console.log( + "projectEnvs", + projectEnvs.map((e) => e.name) + ); + + const projectFolders = await folderDAL.find({ + $in: { + envId: projectEnvs.map((env) => env.id) + } + }); + + // Get all the secrets within the project (as encrypted) + const secrets: TPartialSecret[] = []; + for (const folder of projectFolders) { + const folderSecrets = await secretDAL.find({ folderId: folder.id }); + + const folderSecretVersions = await secretVersionDAL.find({ + folderId: folder.id + }); + const approvalRequests = await secretApprovalRequestDAL.find({ + status: RequestState.Open, + folderId: folder.id + }); + const approvalSecrets = await secretApprovalSecretDAL.find({ + $in: { + requestId: approvalRequests.map((el) => el.id) + } + }); + + secrets.push(...folderSecrets.map((el) => ({ ...el, docType: SecretDocType.Secret }))); + secrets.push(...folderSecretVersions.map((el) => ({ ...el, docType: SecretDocType.SecretVersion }))); + secrets.push(...approvalSecrets.map((el) => ({ ...el, docType: SecretDocType.ApprovalSecret }))); + } + + const decryptedSecrets = decryptSecrets(secrets, userPrivateKey, oldProjectKey); + + if (secrets.length !== decryptedSecrets.length) { + throw new Error("Failed to decrypt some secret versions"); + } + + // Get the existing bot and the existing project keys for the members of the project + const existingBot = await projectBotDAL.findOne({ projectId: project.id }).catch(() => null); + const existingProjectKeys = await projectKeyDAL.find({ projectId: project.id }); + + // TRANSACTION START + await projectDAL.transaction(async (tx) => { + await projectDAL.updateById(project.id, { version: ProjectVersion.V2 }, tx); + + // Create a ghost user + const ghostUser = await orgService.addGhostUser(project.orgId, tx); + + // Create a project key + const { key: newEncryptedProjectKey, iv: newEncryptedProjectKeyIv } = createProjectKey({ + publicKey: ghostUser.keys.publicKey, + privateKey: ghostUser.keys.plainPrivateKey + }); + + console.log("Creating new project key for ghost user"); + // Create a new project key for the GHOST + await projectKeyDAL.create( + { + projectId: project.id, + receiverId: ghostUser.user.id, + encryptedKey: newEncryptedProjectKey, + nonce: newEncryptedProjectKeyIv, + senderId: ghostUser.user.id + }, + tx + ); + + // Create a membership for the ghost user + await projectMembershipDAL.create( + { + projectId: project.id, + userId: ghostUser.user.id, + role: ProjectMembershipRole.Admin + }, + tx + ); + + // If a bot already exists, delete it + if (existingBot) { + console.log("Deleting existing bot"); + await projectBotDAL.delete({ id: existingBot.id }, tx); + } + + console.log("Deleting old project keys"); + // Delete all the existing project keys + await projectKeyDAL.delete( + { + projectId: project.id, + $in: { + id: existingProjectKeys.map((key) => key.id) + } + }, + tx + ); + + console.log("Finding latest key for ghost user"); + const ghostUserLatestKey = await projectKeyDAL.findLatestProjectKey(ghostUser.user.id, project.id, tx); + + if (!ghostUserLatestKey) { + throw new Error("User latest key not found (V2 Upgrade)"); + } + + console.log("Creating new project keys for old members"); + + const newProjectMembers: { + encryptedKey: string; + nonce: string; + senderId: string; + receiverId: string; + projectId: string; + }[] = []; + + for (const key of existingProjectKeys) { + const user = await userDAL.findUserEncKeyByUserId(key.receiverId); + const [orgMembership] = await orgDAL.findMembership({ userId: key.receiverId, orgId: project.orgId }); + + if (!user || !orgMembership) { + throw new Error(`User with ID ${key.receiverId} was not found during upgrade, or user is not in org.`); + } + + const [newMember] = createWsMembers({ + decryptKey: ghostUserLatestKey, + userPrivateKey: ghostUser.keys.plainPrivateKey, + members: [ + { + userPublicKey: user.publicKey, + orgMembershipId: orgMembership.id, + projectMembershipRole: ProjectMembershipRole.Admin + } + ] + }); + + newProjectMembers.push({ + encryptedKey: newMember.workspaceEncryptedKey, + nonce: newMember.workspaceEncryptedNonce, + senderId: ghostUser.user.id, + receiverId: user.id, + projectId: project.id + }); + } + + // Create project keys for all the old members + await projectKeyDAL.insertMany(newProjectMembers, tx); + + // Encrypt the bot private key (which is the same as the ghost user) + const { iv, tag, ciphertext, encoding, algorithm } = infisicalSymmetricEncypt(ghostUser.keys.plainPrivateKey); + + // 5. Create a bot for the project + const newBot = await projectBotDAL.create( + { + name: "Infisical Bot (Ghost)", + projectId: project.id, + tag, + iv, + encryptedPrivateKey: ciphertext, + isActive: true, + publicKey: ghostUser.keys.publicKey, + senderId: ghostUser.user.id, + encryptedProjectKey: newEncryptedProjectKey, + encryptedProjectKeyNonce: newEncryptedProjectKeyIv, + algorithm, + keyEncoding: encoding + }, + tx + ); + + console.log("Updating secrets with new project key"); + console.log("Got decrypted secrets"); + + const botPrivateKey = infisicalSymmetricDecrypt({ + keyEncoding: newBot.keyEncoding as SecretKeyEncoding, + iv: newBot.iv, + tag: newBot.tag, + ciphertext: newBot.encryptedPrivateKey + }); + + const botKey = decryptAsymmetric({ + ciphertext: newBot.encryptedProjectKey!, + privateKey: botPrivateKey, + nonce: newBot.encryptedProjectKeyNonce!, + publicKey: ghostUser.keys.publicKey + }); + + type TPartialSecret = Pick< + TSecrets, + | "id" + | "secretKeyCiphertext" + | "secretKeyIV" + | "secretKeyTag" + | "secretValueCiphertext" + | "secretValueIV" + | "secretValueTag" + | "secretCommentCiphertext" + | "secretCommentIV" + | "secretCommentTag" + >; + + const updatedSecrets: TPartialSecret[] = []; + const updatedSecretVersions: TPartialSecret[] = []; + const updatedSecretApprovals: TPartialSecret[] = []; + for (const rawSecret of decryptedSecrets) { + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretKey, botKey); + const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretValue || "", botKey); + const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretComment || "", botKey); + + const payload = { + id: rawSecret.id, + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + secretCommentCiphertext: secretCommentEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag + } as const; + + if (rawSecret.docType === SecretDocType.Secret) { + updatedSecrets.push(payload); + } else if (rawSecret.docType === SecretDocType.SecretVersion) { + updatedSecretVersions.push(payload); + } else if (rawSecret.docType === SecretDocType.ApprovalSecret) { + updatedSecretApprovals.push(payload); + } else { + throw new Error("Unknown secret type"); + } + } + + const secretUpdates = await secretDAL.bulkUpdateNoVersionIncrement( + [ + ...updatedSecrets.map((secret) => ({ + filter: { id: secret.id }, + data: { + ...secret, + id: undefined + } + })) + ], + tx + ); + + const secretVersionUpdates = await secretVersionDAL.bulkUpdateNoVersionIncrement( + [ + ...updatedSecretVersions.map((version) => ({ + filter: { id: version.id }, + data: { + ...version, + id: undefined + } + })) + ], + tx + ); + + const secretApprovalUpdates = await secretApprovalSecretDAL.bulkUpdateNoVersionIncrement( + [ + ...updatedSecretApprovals.map((approval) => ({ + filter: { + id: approval.id + }, + data: { + ...approval, + id: undefined + } + })) + ], + tx + ); + + if (secretUpdates.length !== updatedSecrets.length) { + throw new Error("Failed to update some secrets"); + } + if (secretVersionUpdates.length !== updatedSecretVersions.length) { + throw new Error("Failed to update some secret versions"); + } + if (secretApprovalUpdates.length !== updatedSecretApprovals.length) { + throw new Error("Failed to update some secret approvals"); + } + + throw new Error("Transaction was successful"); + }); + }; + return { createProject, deleteProject, @@ -328,6 +702,7 @@ export const projectServiceFactory = ({ findProjectGhostUser, getAProject, toggleAutoCapitalization, - updateName + updateName, + upgradeProject }; }; diff --git a/backend/src/services/project/project-types.ts b/backend/src/services/project/project-types.ts index 2ffea1117..d4bbd9af1 100644 --- a/backend/src/services/project/project-types.ts +++ b/backend/src/services/project/project-types.ts @@ -1,3 +1,5 @@ +import { TProjectPermission } from "@app/lib/types"; + import { ActorType } from "../auth/auth-type"; export type TCreateProjectDTO = { @@ -21,3 +23,7 @@ export type TGetProjectDTO = { actorOrgId?: string; projectId: string; }; + +export type TUpgradeProjectDTO = { + userPrivateKey: string; +} & TProjectPermission; diff --git a/backend/src/services/secret/secret-dal.ts b/backend/src/services/secret/secret-dal.ts index c4b74eef8..b3b538168 100644 --- a/backend/src/services/secret/secret-dal.ts +++ b/backend/src/services/secret/secret-dal.ts @@ -22,7 +22,11 @@ export const secretDALFactory = (db: TDbClient) => { // the idea is to use postgres specific function // insert with id this will cause a conflict then merge the data - const bulkUpdate = async (data: Array<{ filter: Partial; data: TSecretsUpdate }>, tx?: Knex) => { + const bulkUpdate = async ( + data: Array<{ filter: Partial; data: TSecretsUpdate }>, + + tx?: Knex + ) => { try { const secs = await Promise.all( data.map(async ({ filter, data: updateData }) => { @@ -41,6 +45,24 @@ export const secretDALFactory = (db: TDbClient) => { } }; + const bulkUpdateNoVersionIncrement = async ( + data: Array<{ filter: Partial; data: TSecretsUpdate }>, + tx?: Knex + ) => { + try { + const secs = await Promise.all( + data.map(async ({ filter, data: updateData }) => { + const [doc] = await (tx || db)(TableName.Secret).where(filter).update(updateData).returning("*"); + if (!doc) throw new BadRequestError({ message: "Failed to update document" }); + return doc; + }) + ); + return secs; + } catch (error) { + throw new DatabaseError({ error, name: "bulk update secret" }); + } + }; + const deleteMany = async ( data: Array<{ blindIndex: string; type: SecretType }>, folderId: string, @@ -145,5 +167,13 @@ export const secretDALFactory = (db: TDbClient) => { } }; - return { ...secretOrm, update, bulkUpdate, deleteMany, findByFolderId, findByBlindIndexes }; + return { + ...secretOrm, + update, + bulkUpdate, + deleteMany, + bulkUpdateNoVersionIncrement, + findByFolderId, + findByBlindIndexes + }; }; diff --git a/backend/src/services/secret/secret-version-dal.ts b/backend/src/services/secret/secret-version-dal.ts index 7a6695e18..516a82a77 100644 --- a/backend/src/services/secret/secret-version-dal.ts +++ b/backend/src/services/secret/secret-version-dal.ts @@ -1,8 +1,8 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; -import { TableName, TSecretVersions } from "@app/db/schemas"; -import { DatabaseError } from "@app/lib/errors"; +import { TableName, TSecretVersions, TSecretVersionsUpdate } from "@app/db/schemas"; +import { BadRequestError, DatabaseError } from "@app/lib/errors"; import { ormify, selectAllTableCols } from "@app/lib/knex"; export type TSecretVersionDALFactory = ReturnType; @@ -36,6 +36,46 @@ export const secretVersionDALFactory = (db: TDbClient) => { } }; + const bulkUpdate = async ( + data: Array<{ filter: Partial; data: TSecretVersionsUpdate }>, + tx?: Knex + ) => { + try { + const secs = await Promise.all( + data.map(async ({ filter, data: updateData }) => { + const [doc] = await (tx || db)(TableName.SecretVersion) + .where(filter) + .update(updateData) + // .increment("version", 1) // TODO: Is this really needed? + .returning("*"); + if (!doc) throw new BadRequestError({ message: "Failed to update document" }); + return doc; + }) + ); + return secs; + } catch (error) { + throw new DatabaseError({ error, name: "bulk update secret" }); + } + }; + + const bulkUpdateNoVersionIncrement = async ( + data: Array<{ filter: Partial; data: TSecretVersionsUpdate }>, + tx?: Knex + ) => { + try { + const secs = await Promise.all( + data.map(async ({ filter, data: updateData }) => { + const [doc] = await (tx || db)(TableName.SecretVersion).where(filter).update(updateData).returning("*"); + if (!doc) throw new BadRequestError({ message: "Failed to update document" }); + return doc; + }) + ); + return secs; + } catch (error) { + throw new DatabaseError({ error, name: "bulk update secret" }); + } + }; + const findLatestVersionMany = async (folderId: string, secretIds: string[], tx?: Knex) => { try { const docs: Array = await (tx || db)(TableName.SecretVersion) @@ -59,5 +99,11 @@ export const secretVersionDALFactory = (db: TDbClient) => { } }; - return { ...secretVersionOrm, findLatestVersionMany, findLatestVersionByFolderId }; + return { + ...secretVersionOrm, + findLatestVersionMany, + bulkUpdate, + findLatestVersionByFolderId, + bulkUpdateNoVersionIncrement + }; }; diff --git a/frontend/src/components/utilities/secrets/encryptSecrets.ts b/frontend/src/components/utilities/secrets/encryptSecrets.ts index 610e2a6f1..943502cdf 100644 --- a/frontend/src/components/utilities/secrets/encryptSecrets.ts +++ b/frontend/src/components/utilities/secrets/encryptSecrets.ts @@ -56,7 +56,6 @@ const encryptSecrets = async ({ publicKey: wsKey.sender.publicKey, privateKey: PRIVATE_KEY }); - } else { // case: a (shared) key does not exist for the workspace randomBytes = crypto.randomBytes(16).toString("hex"); @@ -116,7 +115,6 @@ const encryptSecrets = async ({ return result; }); - } catch (error) { console.log("Error while encrypting secrets"); } diff --git a/frontend/src/hooks/api/keys/queries.tsx b/frontend/src/hooks/api/keys/queries.tsx index 3cf11e189..7e59c0840 100644 --- a/frontend/src/hooks/api/keys/queries.tsx +++ b/frontend/src/hooks/api/keys/queries.tsx @@ -8,9 +8,9 @@ const encKeyKeys = { getUserWorkspaceKey: (workspaceID: string) => ["workspace-key-pair", { workspaceID }] as const }; -export const fetchUserWsKey = async (workspaceID: string) => { +export const fetchUserWsKey = async (projectId: string) => { const { data } = await apiRequest.get( - `/api/v2/workspace/${workspaceID}/encrypted-key` + `/api/v2/workspace/${projectId}/encrypted-key` ); return data; diff --git a/frontend/src/hooks/api/users/mutation.tsx b/frontend/src/hooks/api/users/mutation.tsx index f7c0e1558..7d982cea1 100644 --- a/frontend/src/hooks/api/users/mutation.tsx +++ b/frontend/src/hooks/api/users/mutation.tsx @@ -51,7 +51,7 @@ export const useAddUserToWsNonE2EE = () => { return useMutation<{}, {}, AddUserToWsDTONonE2EE>({ mutationFn: async ({ projectId, emails }) => { - const { data } = await apiRequest.post(`/api/v3/projects/${projectId}/memberships`, { + const { data } = await apiRequest.post(`/api/v2/workspace/${projectId}/memberships`, { emails }); return data; diff --git a/frontend/src/hooks/api/workspace/index.tsx b/frontend/src/hooks/api/workspace/index.tsx index 63be0f7ad..1ad051bc0 100644 --- a/frontend/src/hooks/api/workspace/index.tsx +++ b/frontend/src/hooks/api/workspace/index.tsx @@ -21,5 +21,6 @@ export { useToggleAutoCapitalization, useUpdateIdentityWorkspaceRole, useUpdateUserWorkspaceRole, - useUpdateWsEnvironment + useUpdateWsEnvironment, + useUpgradeProject } from "./queries"; diff --git a/frontend/src/hooks/api/workspace/queries.tsx b/frontend/src/hooks/api/workspace/queries.tsx index 38879de9d..7ec85c442 100644 --- a/frontend/src/hooks/api/workspace/queries.tsx +++ b/frontend/src/hooks/api/workspace/queries.tsx @@ -61,6 +61,21 @@ export const fetchWorkspaceSecrets = async (workspaceId: string) => { return secrets; }; +export const useUpgradeProject = () => { + const queryClient = useQueryClient(); + + return useMutation<{}, {}, { projectId: string; privateKey: string }>({ + mutationFn: ({ projectId, privateKey }) => { + return apiRequest.post(`/api/v2/workspace/${projectId}/upgrade`, { + userPrivateKey: privateKey + }); + }, + onSuccess: () => { + queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace); + } + }); +}; + const fetchUserWorkspaces = async () => { const { data } = await apiRequest.get<{ workspaces: Workspace[] }>("/api/v1/workspace"); return data.workspaces; @@ -160,7 +175,7 @@ export const createWorkspace = ({ organizationId, projectName }: CreateWorkspaceDTO): Promise<{ data: { project: Workspace } }> => { - return apiRequest.post("/api/v3/projects", { projectName, organizationId }); + return apiRequest.post("/api/v2/workspace", { projectName, organizationId }); }; export const useCreateWorkspace = () => { diff --git a/frontend/src/hooks/api/workspace/types.ts b/frontend/src/hooks/api/workspace/types.ts index 240a9e858..5fd3898f8 100644 --- a/frontend/src/hooks/api/workspace/types.ts +++ b/frontend/src/hooks/api/workspace/types.ts @@ -3,7 +3,7 @@ export type Workspace = { id: string; name: string; orgId: string; - e2ee: boolean; + version: "v1" | "v2"; autoCapitalization: boolean; environments: WorkspaceEnv[]; slug: string; diff --git a/frontend/src/layouts/AppLayout/AppLayout.tsx b/frontend/src/layouts/AppLayout/AppLayout.tsx index 6f9d80e6b..1cda2c6d7 100644 --- a/frontend/src/layouts/AppLayout/AppLayout.tsx +++ b/frontend/src/layouts/AppLayout/AppLayout.tsx @@ -4,7 +4,6 @@ /* eslint-disable vars-on-top */ /* eslint-disable no-var */ /* eslint-disable func-names */ -import crypto from "crypto"; import { useEffect } from "react"; import { Controller, useForm } from "react-hook-form"; @@ -34,7 +33,6 @@ import * as yup from "yup"; import { useNotificationContext } from "@app/components/context/Notifications/NotificationProvider"; import { OrgPermissionCan } from "@app/components/permissions"; import { tempLocalStorage } from "@app/components/utilities/checks/tempLocalStorage"; -import { encryptAssymmetric } from "@app/components/utilities/cryptography/crypto"; import { Button, Checkbox, @@ -62,16 +60,14 @@ import { import { usePopUp } from "@app/hooks"; import { fetchOrgUsers, - useAddUserToWsE2EE, + useAddUserToWsNonE2EE, useCreateWorkspace, useGetOrgTrialUrl, useGetSecretApprovalRequestCount, useGetUserAction, useLogoutUser, - useRegisterUserAction, - useUploadWsKey + useRegisterUserAction } from "@app/hooks/api"; -import { fetchUserWsKey } from "@app/hooks/api/keys/queries"; import { CreateOrgModal } from "@app/views/Org/components"; interface LayoutProps { @@ -129,8 +125,8 @@ export const AppLayout = ({ children }: LayoutProps) => { : true; const createWs = useCreateWorkspace(); - const uploadWsKey = useUploadWsKey(); - const addWsUser = useAddUserToWsE2EE(); + const addUsersToProject = useAddUserToWsNonE2EE(); + const infisicalPlatformVersion = process.env.NEXT_PUBLIC_INFISICAL_PLATFORM_VERSION; const { popUp, handlePopUpOpen, handlePopUpClose, handlePopUpToggle } = usePopUp([ @@ -220,53 +216,32 @@ export const AppLayout = ({ children }: LayoutProps) => { const onCreateProject = async ({ name, addMembers }: TAddProjectFormData) => { // type check - if (!currentOrg?.id) return; + if (!currentOrg) return; + if (!user) return; try { const { data: { - project: { id: newWorkspaceId } + project: { id: newProjectId } } } = await createWs.mutateAsync({ - organizationId: currentOrg?.id, + organizationId: currentOrg.id, projectName: name }); - const randomBytes = crypto.randomBytes(16).toString("hex"); - const PRIVATE_KEY = String(localStorage.getItem("PRIVATE_KEY")); - const { ciphertext, nonce } = encryptAssymmetric({ - plaintext: randomBytes, - publicKey: user.publicKey, - privateKey: PRIVATE_KEY - }); - - await uploadWsKey.mutateAsync({ - encryptedKey: ciphertext, - nonce, - userId: user?.id, - workspaceId: newWorkspaceId - }); - if (addMembers) { - // not using hooks because need at this point only const orgUsers = await fetchOrgUsers(currentOrg.id); - const decryptKey = await fetchUserWsKey(newWorkspaceId); - await addWsUser.mutateAsync({ - workspaceId: newWorkspaceId, - decryptKey, - userPrivateKey: PRIVATE_KEY, - members: orgUsers - .filter( - ({ status, user: orgUser }) => status === "accepted" && user.email !== orgUser.email - ) - .map(({ user: orgUser, id: orgMembershipId }) => ({ - userPublicKey: orgUser.publicKey, - orgMembershipId - })) + + await addUsersToProject.mutateAsync({ + emails: orgUsers + .map((member) => member.user.email) + .filter((email) => email !== user.email), + projectId: newProjectId }); } + createNotification({ text: "Workspace created", type: "success" }); handlePopUpClose("addNewWs"); - router.push(`/project/${newWorkspaceId}/secrets/overview`); + router.push(`/project/${newProjectId}/secrets/overview`); } catch (err) { console.error(err); createNotification({ text: "Failed to create workspace", type: "error" }); diff --git a/frontend/src/views/Project/MembersPage/components/MemberListTab/MemberListTab.tsx b/frontend/src/views/Project/MembersPage/components/MemberListTab/MemberListTab.tsx index 838ccce19..222504a91 100644 --- a/frontend/src/views/Project/MembersPage/components/MemberListTab/MemberListTab.tsx +++ b/frontend/src/views/Project/MembersPage/components/MemberListTab/MemberListTab.tsx @@ -117,18 +117,23 @@ export const MemberListTab = () => { if (!orgUser) return; try { - if (currentWorkspace.e2ee) { + if (currentWorkspace.version === "v1") { await addUserToWorkspace({ workspaceId, userPrivateKey, decryptKey: wsKey, members: [{ orgMembershipId, userPublicKey: orgUser.user.publicKey }] }); - } else { + } else if (currentWorkspace.version === "v2") { await addUserToWorkspaceNonE2EE({ projectId: workspaceId, emails: [orgUser.user.email] }); + } else { + createNotification({ + text: "Failed to add user to project, unknown project type", + type: "error" + }); } createNotification({ text: "Successfully added user to the project", diff --git a/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx b/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx index 394914a00..b7af0e3a4 100644 --- a/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx +++ b/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import Link from "next/link"; import { useRouter } from "next/router"; @@ -14,6 +14,8 @@ import { useNotificationContext } from "@app/components/context/Notifications/No import NavHeader from "@app/components/navigation/NavHeader"; import { PermissionDeniedBanner } from "@app/components/permissions"; import { + Alert, + AlertDescription, Button, EmptyState, IconButton, @@ -37,7 +39,8 @@ import { useGetFoldersByEnv, useGetProjectSecretsAllEnv, useGetUserWsKey, - useUpdateSecretV3 + useUpdateSecretV3, + useUpgradeProject } from "@app/hooks/api"; import { FolderBreadCrumbs } from "./components/FolderBreadCrumbs"; @@ -104,6 +107,7 @@ export const SecretOverviewPage = () => { environments: userAvailableEnvs.map(({ slug }) => slug) }); + const upgradeProject = useUpgradeProject(); const { mutateAsync: createSecretV3 } = useCreateSecretV3(); const { mutateAsync: updateSecretV3 } = useUpdateSecretV3(); const { mutateAsync: deleteSecretV3 } = useDeleteSecretV3(); @@ -197,6 +201,24 @@ export const SecretOverviewPage = () => { } }; + const onUpgradeProject = useCallback(async () => { + const PRIVATE_KEY = localStorage.getItem("PRIVATE_KEY"); + + if (!PRIVATE_KEY) { + createNotification({ + type: "error", + text: "Private key not found" + }); + + return; + } + + await upgradeProject.mutateAsync({ + projectId: workspaceId, + privateKey: PRIVATE_KEY + }); + }, []); + const handleResetSearch = () => setSearchFilter(""); const handleFolderClick = (path: string) => { @@ -315,6 +337,23 @@ export const SecretOverviewPage = () => { .

+ + {currentWorkspace?.version === "v1" && ( +
+ + + Upgrade your project. More filler text More filler text More filler text More filler + text More filler text More filler text More filler text More filler text More filler + text More filler text More filler text More filler text{" "} + +
+ +
+ +
+ )}
From 4802a36473f23ae294e076a8680b9cc8150eb4ed Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 01:03:59 +0100 Subject: [PATCH 022/157] Finished migration --- .../migrations/20240202093209_ghost_user.ts | 2 + backend/src/db/schemas/models.ts | 18 + backend/src/db/schemas/projects.ts | 3 +- .../saml-config/saml-config-service.ts | 3 +- backend/src/queue/queue-service.ts | 21 +- backend/src/server/routes/index.ts | 25 +- .../src/server/routes/v2/project-router.ts | 27 +- .../src/services/auth/auth-signup-service.ts | 2 +- backend/src/services/project/project-dal.ts | 15 +- backend/src/services/project/project-queue.ts | 410 ++++++++++++++++++ .../src/services/project/project-service.ts | 348 ++------------- .../super-admin/super-admin-service.ts | 1 + .../UpgradeProjectAlert.tsx | 90 ++++ .../v2/UpgradeProjectAlert/index.tsx | 1 + frontend/src/hooks/api/workspace/index.tsx | 1 + frontend/src/hooks/api/workspace/queries.tsx | 17 + frontend/src/hooks/api/workspace/types.ts | 1 + .../SecretOverviewPage/SecretOverviewPage.tsx | 44 +- 18 files changed, 660 insertions(+), 369 deletions(-) create mode 100644 backend/src/services/project/project-queue.ts create mode 100644 frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx create mode 100644 frontend/src/components/v2/UpgradeProjectAlert/index.tsx diff --git a/backend/src/db/migrations/20240202093209_ghost_user.ts b/backend/src/db/migrations/20240202093209_ghost_user.ts index 72181cd83..ddfdbe812 100644 --- a/backend/src/db/migrations/20240202093209_ghost_user.ts +++ b/backend/src/db/migrations/20240202093209_ghost_user.ts @@ -15,6 +15,7 @@ export async function up(knex: Knex): Promise { if (!hasProjectVersionColumn) { await knex.schema.alterTable(TableName.Project, (t) => { t.string("version").defaultTo(ProjectVersion.V1).notNullable(); + t.text("upgradeStatus").nullable(); }); } } @@ -32,6 +33,7 @@ export async function down(knex: Knex): Promise { if (hasProjectVersionColumn) { await knex.schema.alterTable(TableName.Project, (t) => { t.dropColumn("version"); + t.dropColumn("upgradeStatus"); }); } } diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 4109c516c..426d3af3f 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -117,6 +117,24 @@ export enum ProjectVersion { V2 = "v2" } +// Case study: +/* + +Q: If the upgrade status is null and the project version is V1: +A: It would mean that the project has not been attempted to be upgraded + +Q: If the upgrade status is not null (FAILED or IN_PROGRESS), we a status + +Q: if the project status is null, and the project version is v2, we should display nothing cuz its upgraded! + +*/ + +export enum ProjectUpgradeStatus { + InProgress = "IN_PROGRESS", + Completed = "Project upgrade completed.", // Will be null if completed. So a completed status is not needed + Failed = "FAILED" +} + export enum IdentityAuthMethod { Univeral = "universal-auth" } diff --git a/backend/src/db/schemas/projects.ts b/backend/src/db/schemas/projects.ts index fceb18388..92be7a651 100644 --- a/backend/src/db/schemas/projects.ts +++ b/backend/src/db/schemas/projects.ts @@ -15,7 +15,8 @@ export const ProjectsSchema = z.object({ orgId: z.string().uuid(), createdAt: z.date(), updatedAt: z.date(), - version: z.string().default("v1") + version: z.string().default("v1"), + upgradeStatus: z.string().nullable().optional() }); export type TProjects = z.infer; diff --git a/backend/src/ee/services/saml-config/saml-config-service.ts b/backend/src/ee/services/saml-config/saml-config-service.ts index ad33051fb..824db8251 100644 --- a/backend/src/ee/services/saml-config/saml-config-service.ts +++ b/backend/src/ee/services/saml-config/saml-config-service.ts @@ -338,7 +338,8 @@ export const samlConfigServiceFactory = ({ email, firstName, lastName, - authMethods: [AuthMethod.EMAIL] + authMethods: [AuthMethod.EMAIL], + ghost: false }, tx ); diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index 58c829549..e11e0b590 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -1,6 +1,7 @@ import { Job, JobsOptions, Queue, QueueOptions, RepeatOptions, Worker, WorkerListener } from "bullmq"; import Redis from "ioredis"; +import { SecretKeyEncoding } from "@app/db/schemas"; import { TCreateAuditLogDTO } from "@app/ee/services/audit-log/audit-log-types"; import { TScanFullRepoEventPayload, @@ -15,7 +16,8 @@ export enum QueueName { IntegrationSync = "sync-integrations", SecretWebhook = "secret-webhook", SecretFullRepoScan = "secret-full-repo-scan", - SecretPushEventScan = "secret-push-event-scan" + SecretPushEventScan = "secret-push-event-scan", + UpgradeProjectToGhost = "upgrade-project-to-ghost" } export enum QueueJobs { @@ -25,7 +27,8 @@ export enum QueueJobs { AuditLogPrune = "audit-log-prune-job", SecWebhook = "secret-webhook-trigger", IntegrationSync = "secret-integration-pull", - SecretScan = "secret-scan" + SecretScan = "secret-scan", + UpgradeProjectToGhost = "upgrade-project-to-ghost-job" } export type TQueueJobTypes = { @@ -64,6 +67,20 @@ export type TQueueJobTypes = { payload: TScanFullRepoEventPayload; }; [QueueName.SecretPushEventScan]: { name: QueueJobs.SecretScan; payload: TScanPushEventPayload }; + + [QueueName.UpgradeProjectToGhost]: { + name: QueueJobs.UpgradeProjectToGhost; + payload: { + projectId: string; + startedByUserId: string; + encryptedPrivateKey: { + encryptedKey: string; + encryptedKeyIv: string; + encryptedKeyTag: string; + keyEncoding: SecretKeyEncoding; + }; + }; + }; }; export type TQueueServiceFactory = ReturnType; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 068142ab8..3952ea980 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -65,6 +65,7 @@ import { orgRoleDALFactory } from "@app/services/org/org-role-dal"; import { orgRoleServiceFactory } from "@app/services/org/org-role-service"; import { orgServiceFactory } from "@app/services/org/org-service"; import { projectDALFactory } from "@app/services/project/project-dal"; +import { projectQueueFactory } from "@app/services/project/project-queue"; import { projectServiceFactory } from "@app/services/project/project-service"; import { projectBotDALFactory } from "@app/services/project-bot/project-bot-dal"; import { projectBotServiceFactory } from "@app/services/project-bot/project-bot-service"; @@ -299,19 +300,33 @@ export const registerRoutes = async ( projectKeyDAL, projectMembershipDAL }); + + const projectQueueService = projectQueueFactory({ + queueService, + secretDAL, + folderDAL, + projectDAL, + orgDAL, + orgService, + projectEnvDAL, + userDAL, + secretVersionDAL, + projectKeyDAL, + projectBotDAL, + projectMembershipDAL, + secretApprovalRequestDAL, + secretApprovalSecretDAL: sarSecretDAL + }); + const projectService = projectServiceFactory({ permissionService, projectDAL, + projectQueue: projectQueueService, secretBlindIndexDAL, identityProjectDAL, identityOrgMembershipDAL, projectBotDAL, - secretDAL, - orgDAL, - secretApprovalRequestDAL, - secretApprovalSecretDAL: sarSecretDAL, projectKeyDAL, - secretVersionDAL, userDAL, projectEnvDAL, orgService, diff --git a/backend/src/server/routes/v2/project-router.ts b/backend/src/server/routes/v2/project-router.ts index bc0282850..fd7573c65 100644 --- a/backend/src/server/routes/v2/project-router.ts +++ b/backend/src/server/routes/v2/project-router.ts @@ -74,7 +74,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { userPrivateKey: z.string().trim() }), response: { - 200: z.object({}) + 200: z.void() } }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY]), @@ -88,6 +88,31 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { } }); + server.route({ + url: "/:projectId/upgrade/status", + method: "GET", + schema: { + params: z.object({ + projectId: z.string().trim() + }), + response: { + 200: z.object({ + status: z.string().nullable() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY]), + handler: async (req) => { + const status = await server.services.project.getProjectUpgradeStatus({ + projectId: req.params.projectId, + actor: req.permission.type, + actorId: req.permission.id + }); + + return { status }; + } + }); + /* Create new project */ server.route({ method: "POST", diff --git a/backend/src/services/auth/auth-signup-service.ts b/backend/src/services/auth/auth-signup-service.ts index 021ca7070..0c2f53e5f 100644 --- a/backend/src/services/auth/auth-signup-service.ts +++ b/backend/src/services/auth/auth-signup-service.ts @@ -50,7 +50,7 @@ export const authSignupServiceFactory = ({ throw new Error("Failed to send verification code for complete account"); } if (!user) { - user = await userDAL.create({ authMethods: [AuthMethod.EMAIL], email }); + user = await userDAL.create({ authMethods: [AuthMethod.EMAIL], email, ghost: false }); } if (!user) throw new Error("Failed to create user"); diff --git a/backend/src/services/project/project-dal.ts b/backend/src/services/project/project-dal.ts index 4d7c6072e..a9906f859 100644 --- a/backend/src/services/project/project-dal.ts +++ b/backend/src/services/project/project-dal.ts @@ -1,5 +1,5 @@ import { TDbClient } from "@app/db"; -import { ProjectsSchema, TableName } from "@app/db/schemas"; +import { ProjectsSchema, ProjectUpgradeStatus, TableName, TProjectsUpdate } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; @@ -66,6 +66,18 @@ export const projectDALFactory = (db: TDbClient) => { } }; + const setProjectUpgradeStatus = async (projectId: string, status: ProjectUpgradeStatus) => { + try { + const data: TProjectsUpdate = { + upgradeStatus: status + } as const; + + await db(TableName.Project).where({ id: projectId }).update(data); + } catch (error) { + throw new DatabaseError({ error, name: "Set project upgrade status" }); + } + }; + const findAllProjectsByIdentity = async (identityId: string) => { try { const workspaces = await db(TableName.IdentityProjectMembership) @@ -149,6 +161,7 @@ export const projectDALFactory = (db: TDbClient) => { return { ...projectOrm, findAllProjects, + setProjectUpgradeStatus, findAllProjectsByIdentity, findProjectGhostUser, findProjectById diff --git a/backend/src/services/project/project-queue.ts b/backend/src/services/project/project-queue.ts new file mode 100644 index 000000000..e8b15ddf9 --- /dev/null +++ b/backend/src/services/project/project-queue.ts @@ -0,0 +1,410 @@ +/* eslint-disable no-await-in-loop */ +import { + ProjectMembershipRole, + ProjectUpgradeStatus, + ProjectVersion, + SecretKeyEncoding, + TSecrets +} from "@app/db/schemas"; +import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal"; +import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; +import { RequestState } from "@app/ee/services/secret-approval-request/secret-approval-request-types"; +import { + decryptAsymmetric, + encryptSymmetric128BitHexKeyUTF8, + infisicalSymmetricDecrypt, + infisicalSymmetricEncypt +} from "@app/lib/crypto/encryption"; +import { BadRequestError } from "@app/lib/errors"; +import { logger } from "@app/lib/logger"; +import { createProjectKey, createWsMembers } from "@app/lib/project"; +import { decryptSecrets, SecretDocType, TPartialSecret } from "@app/lib/secret"; +import { QueueJobs, QueueName, TQueueJobTypes, TQueueServiceFactory } from "@app/queue"; + +import { TOrgDALFactory } from "../org/org-dal"; +import { TOrgServiceFactory } from "../org/org-service"; +import { TProjectBotDALFactory } from "../project-bot/project-bot-dal"; +import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; +import { TProjectKeyDALFactory } from "../project-key/project-key-dal"; +import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; +import { TSecretDALFactory } from "../secret/secret-dal"; +import { TSecretVersionDALFactory } from "../secret/secret-version-dal"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TUserDALFactory } from "../user/user-dal"; +import { TProjectDALFactory } from "./project-dal"; + +export type TProjectQueueFactory = ReturnType; + +type TProjectQueueFactoryDep = { + queueService: TQueueServiceFactory; + secretVersionDAL: Pick; + folderDAL: Pick; + secretDAL: Pick; + projectKeyDAL: Pick; + secretApprovalRequestDAL: Pick; + secretApprovalSecretDAL: Pick; + projectBotDAL: Pick; + orgService: Pick; + projectMembershipDAL: Pick; + userDAL: Pick; + + projectEnvDAL: Pick; + projectDAL: Pick; + orgDAL: Pick; +}; + +export const projectQueueFactory = ({ + queueService, + secretDAL, + folderDAL, + userDAL, + secretVersionDAL, + secretApprovalRequestDAL, + secretApprovalSecretDAL, + projectKeyDAL, + projectBotDAL, + projectEnvDAL, + orgDAL, + projectDAL, + orgService, + projectMembershipDAL +}: TProjectQueueFactoryDep) => { + const upgradeProject = async (dto: TQueueJobTypes["upgrade-project-to-ghost"]["payload"]) => { + await queueService.queue(QueueName.UpgradeProjectToGhost, QueueJobs.UpgradeProjectToGhost, dto, { + attempts: 1, + removeOnComplete: true, + removeOnFail: { + count: 5 // keep the most recent jobs + } + }); + }; + + queueService.start(QueueName.UpgradeProjectToGhost, async ({ data }) => { + try { + const [project] = await projectDAL.find({ + id: data.projectId, + version: ProjectVersion.V1, + $in: { + upgradeStatus: [ProjectUpgradeStatus.Failed, null] + } + }); + + const oldProjectKey = await projectKeyDAL.findLatestProjectKey(data.startedByUserId, data.projectId); + + if (!project || !oldProjectKey) { + throw new BadRequestError({ + message: "Project or project key not found" + }); + } + + await projectDAL.setProjectUpgradeStatus(data.projectId, ProjectUpgradeStatus.InProgress); + const userPrivateKey = infisicalSymmetricDecrypt({ + keyEncoding: data.encryptedPrivateKey.keyEncoding, + ciphertext: data.encryptedPrivateKey.encryptedKey, + iv: data.encryptedPrivateKey.encryptedKeyIv, + tag: data.encryptedPrivateKey.encryptedKeyTag + }); + + const projectEnvs = await projectEnvDAL.find({ + projectId: project.id + }); + + const projectFolders = await folderDAL.find({ + $in: { + envId: projectEnvs.map((env) => env.id) + } + }); + + // Get all the secrets within the project (as encrypted) + const secrets: TPartialSecret[] = []; + for (const folder of projectFolders) { + const folderSecrets = await secretDAL.find({ folderId: folder.id }); + + const folderSecretVersions = await secretVersionDAL.find( + { + folderId: folder.id + }, + // Only get the latest 100 secret versions for each folder. + { + limit: 100 + } + ); + const approvalRequests = await secretApprovalRequestDAL.find({ + status: RequestState.Open, + folderId: folder.id + }); + const approvalSecrets = await secretApprovalSecretDAL.find({ + $in: { + requestId: approvalRequests.map((el) => el.id) + } + }); + + secrets.push(...folderSecrets.map((el) => ({ ...el, docType: SecretDocType.Secret }))); + secrets.push(...folderSecretVersions.map((el) => ({ ...el, docType: SecretDocType.SecretVersion }))); + secrets.push(...approvalSecrets.map((el) => ({ ...el, docType: SecretDocType.ApprovalSecret }))); + } + + const decryptedSecrets = decryptSecrets(secrets, userPrivateKey, oldProjectKey); + + if (secrets.length !== decryptedSecrets.length) { + throw new Error("Failed to decrypt some secret versions"); + } + + // Get the existing bot and the existing project keys for the members of the project + const existingBot = await projectBotDAL.findOne({ projectId: project.id }).catch(() => null); + const existingProjectKeys = await projectKeyDAL.find({ projectId: project.id }); + + // TRANSACTION START + await projectDAL.transaction(async (tx) => { + await projectDAL.updateById(project.id, { version: ProjectVersion.V2 }, tx); + + // Create a ghost user + const ghostUser = await orgService.addGhostUser(project.orgId, tx); + + // Create a project key + const { key: newEncryptedProjectKey, iv: newEncryptedProjectKeyIv } = createProjectKey({ + publicKey: ghostUser.keys.publicKey, + privateKey: ghostUser.keys.plainPrivateKey + }); + + // Create a new project key for the GHOST + await projectKeyDAL.create( + { + projectId: project.id, + receiverId: ghostUser.user.id, + encryptedKey: newEncryptedProjectKey, + nonce: newEncryptedProjectKeyIv, + senderId: ghostUser.user.id + }, + tx + ); + + // Create a membership for the ghost user + await projectMembershipDAL.create( + { + projectId: project.id, + userId: ghostUser.user.id, + role: ProjectMembershipRole.Admin + }, + tx + ); + + // If a bot already exists, delete it + if (existingBot) { + await projectBotDAL.delete({ id: existingBot.id }, tx); + } + + // Delete all the existing project keys + await projectKeyDAL.delete( + { + projectId: project.id, + $in: { + id: existingProjectKeys.map((key) => key.id) + } + }, + tx + ); + + const ghostUserLatestKey = await projectKeyDAL.findLatestProjectKey(ghostUser.user.id, project.id, tx); + + if (!ghostUserLatestKey) { + throw new Error("User latest key not found (V2 Upgrade)"); + } + + const newProjectMembers: { + encryptedKey: string; + nonce: string; + senderId: string; + receiverId: string; + projectId: string; + }[] = []; + + for (const key of existingProjectKeys) { + const user = await userDAL.findUserEncKeyByUserId(key.receiverId); + const [orgMembership] = await orgDAL.findMembership({ userId: key.receiverId, orgId: project.orgId }); + + if (!user || !orgMembership) { + throw new Error(`User with ID ${key.receiverId} was not found during upgrade, or user is not in org.`); + } + + const [newMember] = createWsMembers({ + decryptKey: ghostUserLatestKey, + userPrivateKey: ghostUser.keys.plainPrivateKey, + members: [ + { + userPublicKey: user.publicKey, + orgMembershipId: orgMembership.id, + projectMembershipRole: ProjectMembershipRole.Admin + } + ] + }); + + newProjectMembers.push({ + encryptedKey: newMember.workspaceEncryptedKey, + nonce: newMember.workspaceEncryptedNonce, + senderId: ghostUser.user.id, + receiverId: user.id, + projectId: project.id + }); + } + + // Create project keys for all the old members + await projectKeyDAL.insertMany(newProjectMembers, tx); + + // Encrypt the bot private key (which is the same as the ghost user) + const { iv, tag, ciphertext, encoding, algorithm } = infisicalSymmetricEncypt(ghostUser.keys.plainPrivateKey); + + // 5. Create a bot for the project + const newBot = await projectBotDAL.create( + { + name: "Infisical Bot (Ghost)", + projectId: project.id, + tag, + iv, + encryptedPrivateKey: ciphertext, + isActive: true, + publicKey: ghostUser.keys.publicKey, + senderId: ghostUser.user.id, + encryptedProjectKey: newEncryptedProjectKey, + encryptedProjectKeyNonce: newEncryptedProjectKeyIv, + algorithm, + keyEncoding: encoding + }, + tx + ); + + const botPrivateKey = infisicalSymmetricDecrypt({ + keyEncoding: newBot.keyEncoding as SecretKeyEncoding, + iv: newBot.iv, + tag: newBot.tag, + ciphertext: newBot.encryptedPrivateKey + }); + + const botKey = decryptAsymmetric({ + ciphertext: newBot.encryptedProjectKey!, + privateKey: botPrivateKey, + nonce: newBot.encryptedProjectKeyNonce!, + publicKey: ghostUser.keys.publicKey + }); + + type TPartialSecret = Pick< + TSecrets, + | "id" + | "secretKeyCiphertext" + | "secretKeyIV" + | "secretKeyTag" + | "secretValueCiphertext" + | "secretValueIV" + | "secretValueTag" + | "secretCommentCiphertext" + | "secretCommentIV" + | "secretCommentTag" + >; + + const updatedSecrets: TPartialSecret[] = []; + const updatedSecretVersions: TPartialSecret[] = []; + const updatedSecretApprovals: TPartialSecret[] = []; + for (const rawSecret of decryptedSecrets) { + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretKey, botKey); + const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretValue || "", botKey); + const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretComment || "", botKey); + + const payload = { + id: rawSecret.id, + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + secretCommentCiphertext: secretCommentEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag + } as const; + + if (rawSecret.docType === SecretDocType.Secret) { + updatedSecrets.push(payload); + } else if (rawSecret.docType === SecretDocType.SecretVersion) { + updatedSecretVersions.push(payload); + } else if (rawSecret.docType === SecretDocType.ApprovalSecret) { + updatedSecretApprovals.push(payload); + } else { + throw new Error("Unknown secret type"); + } + } + + const secretUpdates = await secretDAL.bulkUpdateNoVersionIncrement( + [ + ...updatedSecrets.map((secret) => ({ + filter: { id: secret.id }, + data: { + ...secret, + id: undefined + } + })) + ], + tx + ); + + const secretVersionUpdates = await secretVersionDAL.bulkUpdateNoVersionIncrement( + [ + ...updatedSecretVersions.map((version) => ({ + filter: { id: version.id }, + data: { + ...version, + id: undefined + } + })) + ], + tx + ); + + const secretApprovalUpdates = await secretApprovalSecretDAL.bulkUpdateNoVersionIncrement( + [ + ...updatedSecretApprovals.map((approval) => ({ + filter: { + id: approval.id + }, + data: { + ...approval, + id: undefined + } + })) + ], + tx + ); + + if (secretUpdates.length !== updatedSecrets.length) { + throw new Error("Failed to update some secrets"); + } + if (secretVersionUpdates.length !== updatedSecretVersions.length) { + throw new Error("Failed to update some secret versions"); + } + if (secretApprovalUpdates.length !== updatedSecretApprovals.length) { + throw new Error("Failed to update some secret approvals"); + } + + throw new Error("Transaction was successful!"); + }); + } catch (err) { + const project = await projectDAL.findOne({ id: data.projectId, version: ProjectVersion.V1 }).catch(() => null); + + if (!project) { + logger.error("Failed to upgrade project, because no project was found", data); + } else { + await projectDAL.setProjectUpgradeStatus(data.projectId, ProjectUpgradeStatus.Failed); + logger.error("Failed to upgrade project", data, err); + } + + throw err; + } + }); + + queueService.listen(QueueName.UpgradeProjectToGhost, "failed", (job, err) => { + logger.error("Upgrade project failed", job?.data, err); + }); + + return { + upgradeProject + }; +}; diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index 9b079576b..8c763d0db 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -1,45 +1,36 @@ +/* eslint-disable @typescript-eslint/no-redeclare */ /* eslint-disable no-console */ /* eslint-disable no-await-in-loop */ import { ForbiddenError } from "@casl/ability"; import slugify from "@sindresorhus/slugify"; -import { ProjectMembershipRole, ProjectVersion, SecretKeyEncoding, TSecrets } from "@app/db/schemas"; +import { ProjectMembershipRole, ProjectVersion } from "@app/db/schemas"; import { TLicenseServiceFactory } from "@app/ee/services/license/license-service"; import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; -import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal"; -import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; -import { RequestState } from "@app/ee/services/secret-approval-request/secret-approval-request-types"; import { isAtLeastAsPrivileged } from "@app/lib/casl"; import { getConfig } from "@app/lib/config/env"; import { createSecretBlindIndex } from "@app/lib/crypto"; -import { - decryptAsymmetric, - encryptSymmetric128BitHexKeyUTF8, - infisicalSymmetricDecrypt, - infisicalSymmetricEncypt -} from "@app/lib/crypto/encryption"; +import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { BadRequestError, ForbiddenRequestError } from "@app/lib/errors"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { createProjectKey, createWsMembers } from "@app/lib/project"; -import { decryptSecrets, SecretDocType, TPartialSecret } from "@app/lib/secret"; +import { TProjectPermission } from "@app/lib/types"; import { ActorType } from "../auth/auth-type"; import { TIdentityOrgDALFactory } from "../identity/identity-org-dal"; import { TIdentityProjectDALFactory } from "../identity-project/identity-project-dal"; -import { TOrgDALFactory } from "../org/org-dal"; import { TOrgServiceFactory } from "../org/org-service"; import { TProjectBotDALFactory } from "../project-bot/project-bot-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectKeyDALFactory } from "../project-key/project-key-dal"; import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; -import { TSecretDALFactory } from "../secret/secret-dal"; -import { TSecretVersionDALFactory } from "../secret/secret-version-dal"; import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-index-dal"; import { ROOT_FOLDER_NAME, TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TUserDALFactory } from "../user/user-dal"; import { TProjectDALFactory } from "./project-dal"; +import { TProjectQueueFactory } from "./project-queue"; import { TCreateProjectDTO, TDeleteProjectDTO, TGetProjectDTO, TUpgradeProjectDTO } from "./project-types"; export const DEFAULT_PROJECT_ENVS = [ @@ -50,22 +41,18 @@ export const DEFAULT_PROJECT_ENVS = [ type TProjectServiceFactoryDep = { projectDAL: TProjectDALFactory; + projectQueue: TProjectQueueFactory; userDAL: TUserDALFactory; folderDAL: TSecretFolderDALFactory; projectEnvDAL: Pick; - secretVersionDAL: TSecretVersionDALFactory; identityOrgMembershipDAL: TIdentityOrgDALFactory; identityProjectDAL: TIdentityProjectDALFactory; projectKeyDAL: Pick; projectBotDAL: Pick; projectMembershipDAL: Pick; - orgDAL: TOrgDALFactory; - secretApprovalRequestDAL: TSecretApprovalRequestDALFactory; - secretApprovalSecretDAL: TSecretApprovalRequestSecretDALFactory; secretBlindIndexDAL: Pick; permissionService: TPermissionServiceFactory; orgService: Pick; - secretDAL: TSecretDALFactory; licenseService: Pick; }; @@ -73,19 +60,15 @@ export type TProjectServiceFactory = ReturnType; export const projectServiceFactory = ({ projectDAL, + projectQueue, projectKeyDAL, - secretApprovalRequestDAL, - secretApprovalSecretDAL, permissionService, userDAL, folderDAL, orgService, - orgDAL, identityProjectDAL, - secretVersionDAL, projectBotDAL, identityOrgMembershipDAL, - secretDAL, secretBlindIndexDAL, projectMembershipDAL, projectEnvDAL, @@ -396,303 +379,33 @@ export const projectServiceFactory = ({ 25. API route returns 200 OK. */ - const project = await projectDAL.findOne({ id: projectId, version: ProjectVersion.V1 }); - const oldProjectKey = await projectKeyDAL.findLatestProjectKey(actorId, projectId); - if (!project || !oldProjectKey) { + const encryptedPrivateKey = infisicalSymmetricEncypt(userPrivateKey); + + await projectQueue.upgradeProject({ + projectId, + startedByUserId: actorId, + encryptedPrivateKey: { + encryptedKey: encryptedPrivateKey.ciphertext, + encryptedKeyIv: encryptedPrivateKey.iv, + encryptedKeyTag: encryptedPrivateKey.tag, + keyEncoding: encryptedPrivateKey.encoding + } + }); + }; + + const getProjectUpgradeStatus = async ({ projectId, actor, actorId }: TProjectPermission) => { + const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId); + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Secrets); + + const project = await projectDAL.findProjectById(projectId); + + if (!project) { throw new BadRequestError({ - message: "Project or project key not found" + message: `Project with id ${projectId} not found` }); } - const projectEnvs = await projectEnvDAL.find({ - projectId: project.id - }); - - console.log( - "projectEnvs", - projectEnvs.map((e) => e.name) - ); - - const projectFolders = await folderDAL.find({ - $in: { - envId: projectEnvs.map((env) => env.id) - } - }); - - // Get all the secrets within the project (as encrypted) - const secrets: TPartialSecret[] = []; - for (const folder of projectFolders) { - const folderSecrets = await secretDAL.find({ folderId: folder.id }); - - const folderSecretVersions = await secretVersionDAL.find({ - folderId: folder.id - }); - const approvalRequests = await secretApprovalRequestDAL.find({ - status: RequestState.Open, - folderId: folder.id - }); - const approvalSecrets = await secretApprovalSecretDAL.find({ - $in: { - requestId: approvalRequests.map((el) => el.id) - } - }); - - secrets.push(...folderSecrets.map((el) => ({ ...el, docType: SecretDocType.Secret }))); - secrets.push(...folderSecretVersions.map((el) => ({ ...el, docType: SecretDocType.SecretVersion }))); - secrets.push(...approvalSecrets.map((el) => ({ ...el, docType: SecretDocType.ApprovalSecret }))); - } - - const decryptedSecrets = decryptSecrets(secrets, userPrivateKey, oldProjectKey); - - if (secrets.length !== decryptedSecrets.length) { - throw new Error("Failed to decrypt some secret versions"); - } - - // Get the existing bot and the existing project keys for the members of the project - const existingBot = await projectBotDAL.findOne({ projectId: project.id }).catch(() => null); - const existingProjectKeys = await projectKeyDAL.find({ projectId: project.id }); - - // TRANSACTION START - await projectDAL.transaction(async (tx) => { - await projectDAL.updateById(project.id, { version: ProjectVersion.V2 }, tx); - - // Create a ghost user - const ghostUser = await orgService.addGhostUser(project.orgId, tx); - - // Create a project key - const { key: newEncryptedProjectKey, iv: newEncryptedProjectKeyIv } = createProjectKey({ - publicKey: ghostUser.keys.publicKey, - privateKey: ghostUser.keys.plainPrivateKey - }); - - console.log("Creating new project key for ghost user"); - // Create a new project key for the GHOST - await projectKeyDAL.create( - { - projectId: project.id, - receiverId: ghostUser.user.id, - encryptedKey: newEncryptedProjectKey, - nonce: newEncryptedProjectKeyIv, - senderId: ghostUser.user.id - }, - tx - ); - - // Create a membership for the ghost user - await projectMembershipDAL.create( - { - projectId: project.id, - userId: ghostUser.user.id, - role: ProjectMembershipRole.Admin - }, - tx - ); - - // If a bot already exists, delete it - if (existingBot) { - console.log("Deleting existing bot"); - await projectBotDAL.delete({ id: existingBot.id }, tx); - } - - console.log("Deleting old project keys"); - // Delete all the existing project keys - await projectKeyDAL.delete( - { - projectId: project.id, - $in: { - id: existingProjectKeys.map((key) => key.id) - } - }, - tx - ); - - console.log("Finding latest key for ghost user"); - const ghostUserLatestKey = await projectKeyDAL.findLatestProjectKey(ghostUser.user.id, project.id, tx); - - if (!ghostUserLatestKey) { - throw new Error("User latest key not found (V2 Upgrade)"); - } - - console.log("Creating new project keys for old members"); - - const newProjectMembers: { - encryptedKey: string; - nonce: string; - senderId: string; - receiverId: string; - projectId: string; - }[] = []; - - for (const key of existingProjectKeys) { - const user = await userDAL.findUserEncKeyByUserId(key.receiverId); - const [orgMembership] = await orgDAL.findMembership({ userId: key.receiverId, orgId: project.orgId }); - - if (!user || !orgMembership) { - throw new Error(`User with ID ${key.receiverId} was not found during upgrade, or user is not in org.`); - } - - const [newMember] = createWsMembers({ - decryptKey: ghostUserLatestKey, - userPrivateKey: ghostUser.keys.plainPrivateKey, - members: [ - { - userPublicKey: user.publicKey, - orgMembershipId: orgMembership.id, - projectMembershipRole: ProjectMembershipRole.Admin - } - ] - }); - - newProjectMembers.push({ - encryptedKey: newMember.workspaceEncryptedKey, - nonce: newMember.workspaceEncryptedNonce, - senderId: ghostUser.user.id, - receiverId: user.id, - projectId: project.id - }); - } - - // Create project keys for all the old members - await projectKeyDAL.insertMany(newProjectMembers, tx); - - // Encrypt the bot private key (which is the same as the ghost user) - const { iv, tag, ciphertext, encoding, algorithm } = infisicalSymmetricEncypt(ghostUser.keys.plainPrivateKey); - - // 5. Create a bot for the project - const newBot = await projectBotDAL.create( - { - name: "Infisical Bot (Ghost)", - projectId: project.id, - tag, - iv, - encryptedPrivateKey: ciphertext, - isActive: true, - publicKey: ghostUser.keys.publicKey, - senderId: ghostUser.user.id, - encryptedProjectKey: newEncryptedProjectKey, - encryptedProjectKeyNonce: newEncryptedProjectKeyIv, - algorithm, - keyEncoding: encoding - }, - tx - ); - - console.log("Updating secrets with new project key"); - console.log("Got decrypted secrets"); - - const botPrivateKey = infisicalSymmetricDecrypt({ - keyEncoding: newBot.keyEncoding as SecretKeyEncoding, - iv: newBot.iv, - tag: newBot.tag, - ciphertext: newBot.encryptedPrivateKey - }); - - const botKey = decryptAsymmetric({ - ciphertext: newBot.encryptedProjectKey!, - privateKey: botPrivateKey, - nonce: newBot.encryptedProjectKeyNonce!, - publicKey: ghostUser.keys.publicKey - }); - - type TPartialSecret = Pick< - TSecrets, - | "id" - | "secretKeyCiphertext" - | "secretKeyIV" - | "secretKeyTag" - | "secretValueCiphertext" - | "secretValueIV" - | "secretValueTag" - | "secretCommentCiphertext" - | "secretCommentIV" - | "secretCommentTag" - >; - - const updatedSecrets: TPartialSecret[] = []; - const updatedSecretVersions: TPartialSecret[] = []; - const updatedSecretApprovals: TPartialSecret[] = []; - for (const rawSecret of decryptedSecrets) { - const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretKey, botKey); - const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretValue || "", botKey); - const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretComment || "", botKey); - - const payload = { - id: rawSecret.id, - secretKeyCiphertext: secretKeyEncrypted.ciphertext, - secretKeyIV: secretKeyEncrypted.iv, - secretKeyTag: secretKeyEncrypted.tag, - secretValueCiphertext: secretValueEncrypted.ciphertext, - secretValueIV: secretValueEncrypted.iv, - secretValueTag: secretValueEncrypted.tag, - secretCommentCiphertext: secretCommentEncrypted.ciphertext, - secretCommentIV: secretCommentEncrypted.iv, - secretCommentTag: secretCommentEncrypted.tag - } as const; - - if (rawSecret.docType === SecretDocType.Secret) { - updatedSecrets.push(payload); - } else if (rawSecret.docType === SecretDocType.SecretVersion) { - updatedSecretVersions.push(payload); - } else if (rawSecret.docType === SecretDocType.ApprovalSecret) { - updatedSecretApprovals.push(payload); - } else { - throw new Error("Unknown secret type"); - } - } - - const secretUpdates = await secretDAL.bulkUpdateNoVersionIncrement( - [ - ...updatedSecrets.map((secret) => ({ - filter: { id: secret.id }, - data: { - ...secret, - id: undefined - } - })) - ], - tx - ); - - const secretVersionUpdates = await secretVersionDAL.bulkUpdateNoVersionIncrement( - [ - ...updatedSecretVersions.map((version) => ({ - filter: { id: version.id }, - data: { - ...version, - id: undefined - } - })) - ], - tx - ); - - const secretApprovalUpdates = await secretApprovalSecretDAL.bulkUpdateNoVersionIncrement( - [ - ...updatedSecretApprovals.map((approval) => ({ - filter: { - id: approval.id - }, - data: { - ...approval, - id: undefined - } - })) - ], - tx - ); - - if (secretUpdates.length !== updatedSecrets.length) { - throw new Error("Failed to update some secrets"); - } - if (secretVersionUpdates.length !== updatedSecretVersions.length) { - throw new Error("Failed to update some secret versions"); - } - if (secretApprovalUpdates.length !== updatedSecretApprovals.length) { - throw new Error("Failed to update some secret approvals"); - } - - throw new Error("Transaction was successful"); - }); + return project.upgradeStatus || null; }; return { @@ -700,6 +413,7 @@ export const projectServiceFactory = ({ deleteProject, getProjects, findProjectGhostUser, + getProjectUpgradeStatus, getAProject, toggleAutoCapitalization, updateName, diff --git a/backend/src/services/super-admin/super-admin-service.ts b/backend/src/services/super-admin/super-admin-service.ts index 9cdc55c8a..faaef42d4 100644 --- a/backend/src/services/super-admin/super-admin-service.ts +++ b/backend/src/services/super-admin/super-admin-service.ts @@ -70,6 +70,7 @@ export const superAdminServiceFactory = ({ lastName, email, superAdmin: true, + ghost: false, isAccepted: true, authMethods: [AuthMethod.EMAIL] }, diff --git a/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx b/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx new file mode 100644 index 000000000..c0d3b4d59 --- /dev/null +++ b/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx @@ -0,0 +1,90 @@ +import { useCallback, useEffect, useState } from "react"; + +import { useNotificationContext } from "@app/components/context/Notifications/NotificationProvider"; +import { useGetUpgradeProjectStatus, useUpgradeProject } from "@app/hooks/api"; +import { Workspace } from "@app/hooks/api/types"; + +import { Alert } from "../Alert"; +import { Button } from "../Button"; + +export type UpgradeProjectAlertProps = { + project: Workspace; +}; + +export const UpgradeProjectAlert = ({ project }: UpgradeProjectAlertProps): JSX.Element | null => { + const { createNotification } = useNotificationContext(); + const upgradeProject = useUpgradeProject(); + const [currentStatus, setCurrentStatus] = useState(null); + const { + data: projectStatus, + refetch: getLatestProjectStatus, + isLoading: statusIsLoading + } = useGetUpgradeProjectStatus(project.id); + + const onUpgradeProject = useCallback(async () => { + const PRIVATE_KEY = localStorage.getItem("PRIVATE_KEY"); + + if (!PRIVATE_KEY) { + createNotification({ + type: "error", + text: "Private key not found" + }); + return; + } + + await upgradeProject.mutateAsync({ + projectId: project.id, + privateKey: PRIVATE_KEY + }); + + await getLatestProjectStatus(); + }, []); + + useEffect(() => { + if (project.version === "v1") { + getLatestProjectStatus(); + } + + if (projectStatus && projectStatus?.status !== null) { + if (projectStatus.status === "IN_PROGRESS") { + setCurrentStatus("Your upgrade is being processed."); + } else if (projectStatus.status === "FAILED") { + setCurrentStatus("Upgrade failed, please try again."); + } + } + + const interval = setInterval(() => { + if (project.version === "v1") { + getLatestProjectStatus(); + } + }, 5_000); + + return () => { + clearInterval(interval); + }; + }, [projectStatus]); + + const isLoading = + (upgradeProject.isLoading || + currentStatus !== null || + (currentStatus === null && statusIsLoading)) && + projectStatus?.status !== "FAILED"; + + if (project.version !== "v1") return null; + + return ( +
+ +
+ Upgrade your project version to continue receiving the latest improvements and patches. + {currentStatus &&

Status: {currentStatus}

} +
+
+ +
+ +
+ ); +}; diff --git a/frontend/src/components/v2/UpgradeProjectAlert/index.tsx b/frontend/src/components/v2/UpgradeProjectAlert/index.tsx new file mode 100644 index 000000000..ab67f86cf --- /dev/null +++ b/frontend/src/components/v2/UpgradeProjectAlert/index.tsx @@ -0,0 +1 @@ +export { UpgradeProjectAlert } from "./UpgradeProjectAlert"; diff --git a/frontend/src/hooks/api/workspace/index.tsx b/frontend/src/hooks/api/workspace/index.tsx index 1ad051bc0..e66af93a1 100644 --- a/frontend/src/hooks/api/workspace/index.tsx +++ b/frontend/src/hooks/api/workspace/index.tsx @@ -7,6 +7,7 @@ export { useDeleteUserFromWorkspace, useDeleteWorkspace, useDeleteWsEnvironment, + useGetUpgradeProjectStatus, useGetUserWorkspaceMemberships, useGetUserWorkspaces, useGetWorkspaceAuthorizations, diff --git a/frontend/src/hooks/api/workspace/queries.tsx b/frontend/src/hooks/api/workspace/queries.tsx index 7ec85c442..443cf45e9 100644 --- a/frontend/src/hooks/api/workspace/queries.tsx +++ b/frontend/src/hooks/api/workspace/queries.tsx @@ -24,6 +24,7 @@ export const workspaceKeys = { getWorkspaceSecrets: (workspaceId: string) => [{ workspaceId }, "workspace-secrets"] as const, getWorkspaceIndexStatus: (workspaceId: string) => [{ workspaceId }, "workspace-index-status"] as const, + getProjectUpgradeStatus: (workspaceId: string) => [{ workspaceId }, "workspace-upgrade-status"], getWorkspaceMemberships: (orgId: string) => [{ orgId }, "workspace-memberships"], getWorkspaceAuthorization: (workspaceId: string) => [{ workspaceId }, "workspace-authorizations"], getWorkspaceIntegrations: (workspaceId: string) => [{ workspaceId }, "workspace-integrations"], @@ -51,6 +52,14 @@ const fetchWorkspaceIndexStatus = async (workspaceId: string) => { return data; }; +const fetchProjectUpgradeStatus = async (projectId: string) => { + const { data } = await apiRequest.get<{ status: string }>( + `/api/v2/workspace/${projectId}/upgrade/status` + ); + + return data; +}; + export const fetchWorkspaceSecrets = async (workspaceId: string) => { const { data: { secrets } @@ -76,6 +85,14 @@ export const useUpgradeProject = () => { }); }; +export const useGetUpgradeProjectStatus = (projectId: string) => { + return useQuery({ + queryKey: workspaceKeys.getProjectUpgradeStatus(projectId), + queryFn: () => fetchProjectUpgradeStatus(projectId), + enabled: true + }); +}; + const fetchUserWorkspaces = async () => { const { data } = await apiRequest.get<{ workspaces: Workspace[] }>("/api/v1/workspace"); return data.workspaces; diff --git a/frontend/src/hooks/api/workspace/types.ts b/frontend/src/hooks/api/workspace/types.ts index 5fd3898f8..a888b7c34 100644 --- a/frontend/src/hooks/api/workspace/types.ts +++ b/frontend/src/hooks/api/workspace/types.ts @@ -4,6 +4,7 @@ export type Workspace = { name: string; orgId: string; version: "v1" | "v2"; + upgradeStatus: string | null; autoCapitalization: boolean; environments: WorkspaceEnv[]; slug: string; diff --git a/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx b/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx index b7af0e3a4..7ea00831c 100644 --- a/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx +++ b/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useRef, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import Link from "next/link"; import { useRouter } from "next/router"; @@ -14,8 +14,6 @@ import { useNotificationContext } from "@app/components/context/Notifications/No import NavHeader from "@app/components/navigation/NavHeader"; import { PermissionDeniedBanner } from "@app/components/permissions"; import { - Alert, - AlertDescription, Button, EmptyState, IconButton, @@ -31,6 +29,7 @@ import { Tooltip, Tr } from "@app/components/v2"; +import { UpgradeProjectAlert } from "@app/components/v2/UpgradeProjectAlert"; import { useOrganization, useWorkspace } from "@app/context"; import { useCreateFolder, @@ -39,8 +38,7 @@ import { useGetFoldersByEnv, useGetProjectSecretsAllEnv, useGetUserWsKey, - useUpdateSecretV3, - useUpgradeProject + useUpdateSecretV3 } from "@app/hooks/api"; import { FolderBreadCrumbs } from "./components/FolderBreadCrumbs"; @@ -107,7 +105,6 @@ export const SecretOverviewPage = () => { environments: userAvailableEnvs.map(({ slug }) => slug) }); - const upgradeProject = useUpgradeProject(); const { mutateAsync: createSecretV3 } = useCreateSecretV3(); const { mutateAsync: updateSecretV3 } = useUpdateSecretV3(); const { mutateAsync: deleteSecretV3 } = useDeleteSecretV3(); @@ -201,24 +198,6 @@ export const SecretOverviewPage = () => { } }; - const onUpgradeProject = useCallback(async () => { - const PRIVATE_KEY = localStorage.getItem("PRIVATE_KEY"); - - if (!PRIVATE_KEY) { - createNotification({ - type: "error", - text: "Private key not found" - }); - - return; - } - - await upgradeProject.mutateAsync({ - projectId: workspaceId, - privateKey: PRIVATE_KEY - }); - }, []); - const handleResetSearch = () => setSearchFilter(""); const handleFolderClick = (path: string) => { @@ -338,22 +317,7 @@ export const SecretOverviewPage = () => {

- {currentWorkspace?.version === "v1" && ( -
- - - Upgrade your project. More filler text More filler text More filler text More filler - text More filler text More filler text More filler text More filler text More filler - text More filler text More filler text More filler text{" "} - -
- -
- -
- )} + {currentWorkspace?.version === "v1" && }
From 5364480ca23f0622c9b3337b277dbd4501810bbc Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 01:04:32 +0100 Subject: [PATCH 023/157] Update models.ts --- backend/src/db/schemas/models.ts | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 426d3af3f..083340d9d 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -117,21 +117,9 @@ export enum ProjectVersion { V2 = "v2" } -// Case study: -/* - -Q: If the upgrade status is null and the project version is V1: -A: It would mean that the project has not been attempted to be upgraded - -Q: If the upgrade status is not null (FAILED or IN_PROGRESS), we a status - -Q: if the project status is null, and the project version is v2, we should display nothing cuz its upgraded! - -*/ - export enum ProjectUpgradeStatus { InProgress = "IN_PROGRESS", - Completed = "Project upgrade completed.", // Will be null if completed. So a completed status is not needed + // Completed = "Project upgrade completed.", // Will be null if completed. So a completed status is not needed Failed = "FAILED" } From 4aa5822ae2c4527ca1f3015a135e466da17bbcc5 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 14:15:58 +0100 Subject: [PATCH 024/157] Don't show ghost users --- .../src/services/project-membership/project-membership-dal.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/services/project-membership/project-membership-dal.ts b/backend/src/services/project-membership/project-membership-dal.ts index b07043719..cd7af9040 100644 --- a/backend/src/services/project-membership/project-membership-dal.ts +++ b/backend/src/services/project-membership/project-membership-dal.ts @@ -30,8 +30,8 @@ export const projectMembershipDALFactory = (db: TDbClient) => { db.ref("firstName").withSchema(TableName.Users), db.ref("lastName").withSchema(TableName.Users), db.ref("id").withSchema(TableName.Users).as("userId") - ); - // .where({ ghost: false }); + ) + .where({ ghost: false }); return members.map(({ email, firstName, lastName, publicKey, ghost, ...data }) => ({ ...data, user: { email, firstName, lastName, id: data.userId, publicKey, ghost } From ec3cf0208c7f85de647fb826961694a476d74c2e Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 14:16:12 +0100 Subject: [PATCH 025/157] Add transaction support --- backend/src/services/project/project-dal.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/src/services/project/project-dal.ts b/backend/src/services/project/project-dal.ts index a9906f859..c85d67ee1 100644 --- a/backend/src/services/project/project-dal.ts +++ b/backend/src/services/project/project-dal.ts @@ -1,3 +1,5 @@ +import { Knex } from "knex"; + import { TDbClient } from "@app/db"; import { ProjectsSchema, ProjectUpgradeStatus, TableName, TProjectsUpdate } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; @@ -66,13 +68,13 @@ export const projectDALFactory = (db: TDbClient) => { } }; - const setProjectUpgradeStatus = async (projectId: string, status: ProjectUpgradeStatus) => { + const setProjectUpgradeStatus = async (projectId: string, status: ProjectUpgradeStatus | null, tx?: Knex) => { try { const data: TProjectsUpdate = { upgradeStatus: status } as const; - await db(TableName.Project).where({ id: projectId }).update(data); + await (tx || db)(TableName.Project).where({ id: projectId }).update(data); } catch (error) { throw new DatabaseError({ error, name: "Set project upgrade status" }); } From 819a9b8d2753ea043b66945fa6881d165218cb60 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 14:16:27 +0100 Subject: [PATCH 026/157] Finish upgrade queue --- backend/src/services/project/project-queue.ts | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/backend/src/services/project/project-queue.ts b/backend/src/services/project/project-queue.ts index e8b15ddf9..03a075d7c 100644 --- a/backend/src/services/project/project-queue.ts +++ b/backend/src/services/project/project-queue.ts @@ -15,7 +15,6 @@ import { infisicalSymmetricDecrypt, infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; -import { BadRequestError } from "@app/lib/errors"; import { logger } from "@app/lib/logger"; import { createProjectKey, createWsMembers } from "@app/lib/project"; import { decryptSecrets, SecretDocType, TPartialSecret } from "@app/lib/secret"; @@ -83,21 +82,21 @@ export const projectQueueFactory = ({ try { const [project] = await projectDAL.find({ id: data.projectId, - version: ProjectVersion.V1, - $in: { - upgradeStatus: [ProjectUpgradeStatus.Failed, null] - } + version: ProjectVersion.V1 }); const oldProjectKey = await projectKeyDAL.findLatestProjectKey(data.startedByUserId, data.projectId); if (!project || !oldProjectKey) { - throw new BadRequestError({ - message: "Project or project key not found" - }); + throw new Error("Project or project key not found"); } - await projectDAL.setProjectUpgradeStatus(data.projectId, ProjectUpgradeStatus.InProgress); + if (project.upgradeStatus !== ProjectUpgradeStatus.Failed && project.upgradeStatus !== null) { + throw new Error("Project upgrade status is not valid"); + } + + await projectDAL.setProjectUpgradeStatus(data.projectId, ProjectUpgradeStatus.InProgress); // Set the status to in progress. This is important to prevent multiple upgrades at the same time. + const userPrivateKey = infisicalSymmetricDecrypt({ keyEncoding: data.encryptedPrivateKey.keyEncoding, ciphertext: data.encryptedPrivateKey.encryptedKey, @@ -384,10 +383,18 @@ export const projectQueueFactory = ({ throw new Error("Failed to update some secret approvals"); } - throw new Error("Transaction was successful!"); + await projectDAL.setProjectUpgradeStatus(data.projectId, null, tx); + + // throw new Error("Transaction was successful!"); }); } catch (err) { - const project = await projectDAL.findOne({ id: data.projectId, version: ProjectVersion.V1 }).catch(() => null); + console.log(err); + const [project] = await projectDAL + .find({ + id: data.projectId, + version: ProjectVersion.V1 + }) + .catch(() => [null]); if (!project) { logger.error("Failed to upgrade project, because no project was found", data); From 8e13eb6077afad01078d7fdcc7d144a98082da5a Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 14:21:21 +0100 Subject: [PATCH 027/157] Update project-service.ts --- .../src/services/project/project-service.ts | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index 8c763d0db..f3bbff491 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -251,7 +251,6 @@ export const projectServiceFactory = ({ } // Get the role permission for the identity - // IS THIS CORRECT? const { permission: rolePermission, role: customRole } = await permissionService.getOrgPermissionByRole( ProjectMembershipRole.Admin, orgId @@ -342,43 +341,6 @@ export const projectServiceFactory = ({ }); } - /* - 1. Get the existing project - 2. Get the existing project keys - 4. Get all the project envs & folders - 5. Get ALL secrets within the project - 6. Create a new ghost user - 7. Create a project membership for the ghost user - 8. Get the existing bot, and the existing project keys for the members of the project - 9. IF a bot already exists for the project, delete it! - 10. Delete all the existing project keys - 11. Create a project key for the ghost user - 12. Find the newly created ghost user's latest key - - - FOR EACH OF THE OLD PROJECT KEYS (loop): - 13. Find the user based on the key.receiverId. - 14. Find the org membership for the user. - 15. Create a new project key for the user. - - - 16. Encrypt the ghost user's private key - 17. Create a new bot, and set the public/private key of the bot, to the ghost user's public/private key. - 18. Add the workspace key to the bot - 19. Decrypt the secrets with the old project key - 20. Get the newly created bot's private key, and workspace key (we do it this way to test as many steps of the bot process as possible) - 21. Get the workspace key from the bot - - - FOR EACH DECRYPTED SECRET (loop): - 22. Re-encrypt the secret value, secret key, and secret comment with the NEW project key from the bot. - 23. Update the secret in the database with the new encrypted values. - - - 24. Transaction ends. If there were no errors. All changes are applied. - 25. API route returns 200 OK. - */ - const encryptedPrivateKey = infisicalSymmetricEncypt(userPrivateKey); await projectQueue.upgradeProject({ From 6dcab6646cac2f80f8704ec934e816d29944a9c3 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 14:21:50 +0100 Subject: [PATCH 028/157] Make it impossible to log in with the ghost user --- backend/src/services/user/user-dal.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/services/user/user-dal.ts b/backend/src/services/user/user-dal.ts index 7b54e7c10..086ad5931 100644 --- a/backend/src/services/user/user-dal.ts +++ b/backend/src/services/user/user-dal.ts @@ -23,7 +23,7 @@ export const userDALFactory = (db: TDbClient) => { const findUserEncKeyByEmail = async (email: string) => { try { return await db(TableName.Users) - .where({ email }) + .where({ email, ghost: false }) .join(TableName.UserEncryptionKey, `${TableName.Users}.id`, `${TableName.UserEncryptionKey}.userId`) .first(); } catch (error) { From 69cba4e6c70025fef53dd2dbb021eb6f7cba9dcc Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 14:22:18 +0100 Subject: [PATCH 029/157] Make alert only visible to admins and add reloading on completion --- .../UpgradeProjectAlert.tsx | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx b/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx index c0d3b4d59..3aac7852e 100644 --- a/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx +++ b/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx @@ -1,6 +1,8 @@ import { useCallback, useEffect, useState } from "react"; +import { useRouter } from "next/router"; import { useNotificationContext } from "@app/components/context/Notifications/NotificationProvider"; +import { useProjectPermission } from "@app/context"; import { useGetUpgradeProjectStatus, useUpgradeProject } from "@app/hooks/api"; import { Workspace } from "@app/hooks/api/types"; @@ -13,6 +15,8 @@ export type UpgradeProjectAlertProps = { export const UpgradeProjectAlert = ({ project }: UpgradeProjectAlertProps): JSX.Element | null => { const { createNotification } = useNotificationContext(); + const router = useRouter(); + const { membership } = useProjectPermission(); const upgradeProject = useUpgradeProject(); const [currentStatus, setCurrentStatus] = useState(null); const { @@ -41,26 +45,36 @@ export const UpgradeProjectAlert = ({ project }: UpgradeProjectAlertProps): JSX. }, []); useEffect(() => { - if (project.version === "v1") { - getLatestProjectStatus(); - } + let interval: NodeJS.Timeout | null = null; - if (projectStatus && projectStatus?.status !== null) { - if (projectStatus.status === "IN_PROGRESS") { - setCurrentStatus("Your upgrade is being processed."); - } else if (projectStatus.status === "FAILED") { - setCurrentStatus("Upgrade failed, please try again."); - } - } - - const interval = setInterval(() => { + if (membership.role === "admin") { if (project.version === "v1") { getLatestProjectStatus(); } - }, 5_000); + + if (projectStatus && projectStatus?.status !== null) { + if (projectStatus.status === "IN_PROGRESS") { + setCurrentStatus("Your upgrade is being processed."); + } else if (projectStatus.status === "FAILED") { + setCurrentStatus("Upgrade failed, please try again."); + } + } + + if (currentStatus !== null && projectStatus?.status === null) { + router.reload(); + } + + interval = setInterval(() => { + if (project.version === "v1") { + getLatestProjectStatus(); + } + }, 5_000); + } return () => { - clearInterval(interval); + if (interval) { + clearInterval(interval); + } }; }, [projectStatus]); @@ -71,6 +85,7 @@ export const UpgradeProjectAlert = ({ project }: UpgradeProjectAlertProps): JSX. projectStatus?.status !== "FAILED"; if (project.version !== "v1") return null; + if (membership.role !== "admin") return null; return (
From 8f3de3cc90dfd5c9e6069d4719992eb6313bbdf1 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 14:22:58 +0100 Subject: [PATCH 030/157] Small UI fixes on approvals page (color and capitalization and spelling) --- .../components/SecretApprovalRequestChanges.tsx | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx index 6b8fb6f80..a767a5b20 100644 --- a/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx +++ b/frontend/src/views/SecretApprovalPage/components/SecretApprovalRequest/components/SecretApprovalRequestChanges.tsx @@ -9,6 +9,7 @@ import { faXmarkCircle } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { cx } from "cva"; import { useNotificationContext } from "@app/components/context/Notifications/NotificationProvider"; import { Button, ContentLoader, EmptyState, IconButton, Tooltip } from "@app/components/v2"; @@ -170,9 +171,18 @@ export const SecretApprovalRequestChanges = ({ -
+
- {secretApprovalRequestDetails.status} + + {secretApprovalRequestDetails.status === "close" + ? "closed" + : secretApprovalRequestDetails.status} +
From 5156971d75f43a63d5416147b960e22955e15e66 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 14:33:46 +0100 Subject: [PATCH 031/157] Add feature flag for showing upgrade project modal --- .../components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx b/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx index 3aac7852e..9dd8b7602 100644 --- a/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx +++ b/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx @@ -84,6 +84,7 @@ export const UpgradeProjectAlert = ({ project }: UpgradeProjectAlertProps): JSX. (currentStatus === null && statusIsLoading)) && projectStatus?.status !== "FAILED"; + if (process.env.NEXT_PUBLIC_SHOW_UPGRADE_PROJECT !== "true") return null; if (project.version !== "v1") return null; if (membership.role !== "admin") return null; From 075f457bd12508cf9b60156ae5cf110bb7045ef4 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 18:15:27 +0100 Subject: [PATCH 032/157] Update models.ts --- backend/src/db/schemas/models.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 083340d9d..b15912a57 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -119,7 +119,7 @@ export enum ProjectVersion { export enum ProjectUpgradeStatus { InProgress = "IN_PROGRESS", - // Completed = "Project upgrade completed.", // Will be null if completed. So a completed status is not needed + // Completed -> Will be null if completed. So a completed status is not needed Failed = "FAILED" } From a3a1a0007d9b88cac1eac5b51605d1bf202b8cf7 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 18:15:44 +0100 Subject: [PATCH 033/157] Add auth methods --- backend/src/server/routes/v1/project-router.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/server/routes/v1/project-router.ts b/backend/src/server/routes/v1/project-router.ts index 873a2e78c..dab8aaa2c 100644 --- a/backend/src/server/routes/v1/project-router.ts +++ b/backend/src/server/routes/v1/project-router.ts @@ -171,7 +171,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { }) } }, - onRequest: verifyAuth([AuthMode.JWT]), + onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { const workspace = await server.services.project.deleteProject({ actorId: req.permission.id, From 858b49d766c5c230a792a531d4bd69408530422a Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 18:19:07 +0100 Subject: [PATCH 034/157] Add delete memberships by email --- .../routes/v2/project-membership-router.ts | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/backend/src/server/routes/v2/project-membership-router.ts b/backend/src/server/routes/v2/project-membership-router.ts index 733147b00..333242142 100644 --- a/backend/src/server/routes/v2/project-membership-router.ts +++ b/backend/src/server/routes/v2/project-membership-router.ts @@ -3,6 +3,8 @@ import { z } from "zod"; import { ProjectMembershipsSchema } from "@app/db/schemas"; import { EventType } from "@app/ee/services/audit-log/audit-log-types"; import { authRateLimit } from "@app/server/config/rateLimiter"; +import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; +import { AuthMode } from "@app/services/auth/auth-type"; export const registerProjectMembershipRouter = async (server: FastifyZodProvider) => { server.route({ @@ -24,6 +26,7 @@ export const registerProjectMembershipRouter = async (server: FastifyZodProvider }) } }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { const memberships = await server.services.projectMembership.addUsersToProjectNonE2EE({ projectId: req.params.projectId, @@ -48,4 +51,52 @@ export const registerProjectMembershipRouter = async (server: FastifyZodProvider return { memberships }; } }); + + server.route({ + method: "DELETE", + url: "/:projectId/memberships", + config: { + rateLimit: authRateLimit + }, + schema: { + params: z.object({ + projectId: z.string() + }), + + body: z.object({ + emails: z.string().email().array() + }), + response: { + 200: z.object({ + memberships: ProjectMembershipsSchema.array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + const memberships = await server.services.projectMembership.deleteProjectMemberships({ + actorId: req.permission.id, + actor: req.permission.type, + actorOrgId: req.permission.orgId, + projectId: req.params.projectId, + emails: req.body.emails + }); + + for (const membership of memberships) { + // eslint-disable-next-line no-await-in-loop + await server.services.auditLog.createAuditLog({ + ...req.auditLogInfo, + projectId: req.params.projectId, + event: { + type: EventType.REMOVE_WORKSPACE_MEMBER, + metadata: { + userId: membership.userId, + email: "" + } + } + }); + } + return { memberships }; + } + }); }; From 37645ba126f9f977acd63d3b546b312bbab167ad Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 18:19:17 +0100 Subject: [PATCH 035/157] Update project-router.ts --- backend/src/server/routes/v2/project-router.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/src/server/routes/v2/project-router.ts b/backend/src/server/routes/v2/project-router.ts index fd7573c65..20fff74da 100644 --- a/backend/src/server/routes/v2/project-router.ts +++ b/backend/src/server/routes/v2/project-router.ts @@ -62,6 +62,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { } }); + /* Start upgrade of a project */ server.route({ url: "/:projectId/upgrade", method: "POST", @@ -88,6 +89,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { } }); + /* Get upgrade status of project */ server.route({ url: "/:projectId/upgrade/status", method: "GET", @@ -131,6 +133,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { }) } }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { const project = await server.services.project.createProject({ actorId: req.permission.id, From 45327f10b1c75310a122c9bdc93dfbf3c90728f3 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 18:19:36 +0100 Subject: [PATCH 036/157] Update project-bot-service.ts --- .../services/project-bot/project-bot-service.ts | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/backend/src/services/project-bot/project-bot-service.ts b/backend/src/services/project-bot/project-bot-service.ts index e95ab1610..2d11b12da 100644 --- a/backend/src/services/project-bot/project-bot-service.ts +++ b/backend/src/services/project-bot/project-bot-service.ts @@ -1,5 +1,4 @@ import { ForbiddenError } from "@casl/ability"; -import { Knex } from "knex"; import { SecretKeyEncoding } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; @@ -106,16 +105,12 @@ export const projectBotServiceFactory = ({ projectBotDAL, permissionService }: T if (!botKey?.nonce || !botKey?.encryptedKey) { throw new BadRequestError({ message: "Failed to set bot active - missing bot key" }); } - const doc = await projectBotDAL.updateById( - botId, - { - isActive: true, - encryptedProjectKey: botKey.encryptedKey, - encryptedProjectKeyNonce: botKey.nonce, - senderId: actorId - }, - tx - ); + const doc = await projectBotDAL.updateById(botId, { + isActive: true, + encryptedProjectKey: botKey.encryptedKey, + encryptedProjectKeyNonce: botKey.nonce, + senderId: actorId + }); if (!doc) throw new BadRequestError({ message: "Failed to update bot active state" }); return doc; } From 5409cffe3332dc4558e37ca32cf3125bd61f9c0d Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 18:19:43 +0100 Subject: [PATCH 037/157] Update project-key-service.ts --- backend/src/services/project-key/project-key-service.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/src/services/project-key/project-key-service.ts b/backend/src/services/project-key/project-key-service.ts index 81b589bd0..fa77760a4 100644 --- a/backend/src/services/project-key/project-key-service.ts +++ b/backend/src/services/project-key/project-key-service.ts @@ -1,5 +1,4 @@ import { ForbiddenError } from "@casl/ability"; -import { Knex } from "knex"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; @@ -44,7 +43,7 @@ export const projectKeyServiceFactory = ({ name: "Upload project keys" }); - await projectKeyDAL.create({ projectId, receiverId, encryptedKey, nonce, senderId: actorId }, tx); + await projectKeyDAL.create({ projectId, receiverId, encryptedKey, nonce, senderId: actorId }); }; const getLatestProjectKey = async ({ actorId, projectId, actor, actorOrgId }: TGetLatestProjectKeyDTO) => { From 1b82a157cc30b416aa3812301335b005f3a9cca4 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 18:19:54 +0100 Subject: [PATCH 038/157] Find membership by email --- .../project-membership-dal.ts | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/backend/src/services/project-membership/project-membership-dal.ts b/backend/src/services/project-membership/project-membership-dal.ts index cd7af9040..14a899164 100644 --- a/backend/src/services/project-membership/project-membership-dal.ts +++ b/backend/src/services/project-membership/project-membership-dal.ts @@ -56,5 +56,31 @@ export const projectMembershipDALFactory = (db: TDbClient) => { } }; - return { ...projectMemberOrm, findAllProjectMembers, findProjectGhostUser }; + const findMembershipsByEmail = async (projectId: string, emails: string[]) => { + try { + const members = await db(TableName.ProjectMembership) + .where({ projectId }) + .join(TableName.Users, `${TableName.ProjectMembership}.userId`, `${TableName.Users}.id`) + .join( + TableName.UserEncryptionKey, + `${TableName.UserEncryptionKey}.userId`, + `${TableName.Users}.id` + ) + .select( + selectAllTableCols(TableName.ProjectMembership), + db.ref("id").withSchema(TableName.Users).as("userId"), + db.ref("email").withSchema(TableName.Users) + ) + .whereIn("email", emails) + .where({ ghost: false }); + return members.map(({ userId, email, ...data }) => ({ + ...data, + user: { id: userId, email } + })); + } catch (error) { + throw new DatabaseError({ error, name: "Find members by email" }); + } + }; + + return { ...projectMemberOrm, findAllProjectMembers, findProjectGhostUser, findMembershipsByEmail }; }; From 4cca82c3c8a5e2eaa4d7a01c4f4fe3f3b85bd06e Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 18:20:17 +0100 Subject: [PATCH 039/157] New service for deleting memberships by email --- .../project-membership-service.ts | 68 +++++++++++++++++-- 1 file changed, 62 insertions(+), 6 deletions(-) diff --git a/backend/src/services/project-membership/project-membership-service.ts b/backend/src/services/project-membership/project-membership-service.ts index e95401d11..33fbee902 100644 --- a/backend/src/services/project-membership/project-membership-service.ts +++ b/backend/src/services/project-membership/project-membership-service.ts @@ -18,6 +18,7 @@ import { BadRequestError } from "@app/lib/errors"; import { groupBy } from "@app/lib/fn"; import { createWsMembers } from "@app/lib/project"; +import { ActorType } from "../auth/auth-type"; import { TOrgDALFactory } from "../org/org-dal"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectBotDALFactory } from "../project-bot/project-bot-dal"; @@ -29,7 +30,8 @@ import { TProjectMembershipDALFactory } from "./project-membership-dal"; import { TAddUsersToWorkspaceDTO, TAddUsersToWorkspaceNonE2EEDTO, - TDeleteProjectMembershipDTO, + TDeleteProjectMembershipOldDTO, + TDeleteProjectMembershipsDTO, TGetProjectMembershipDTO, TInviteUserToProjectDTO, TUpdateProjectMembershipDTO @@ -355,21 +357,22 @@ export const projectMembershipServiceFactory = ({ return membership; }; + // This is old and should be removed later. Its not used anywhere, but it is exposed in our API. So to avoid breaking changes, we are keeping it for now. const deleteProjectMembership = async ({ actorId, actor, actorOrgId, projectId, membershipId - }: TDeleteProjectMembershipDTO) => { + }: TDeleteProjectMembershipOldDTO) => { const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId, actorOrgId); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Delete, ProjectPermissionSub.Member); - const membershipUser = await userDAL.findUserByProjectMembershipId(membershipId); + const member = await userDAL.findUserByProjectMembershipId(membershipId); - if (membershipUser?.ghost) { + if (member?.ghost) { throw new BadRequestError({ - message: "Cannot delete ghost", + message: "Unauthorized member delete", name: "Delete project membership" }); } @@ -382,12 +385,65 @@ export const projectMembershipServiceFactory = ({ return membership; }; + const deleteProjectMemberships = async ({ + actorId, + actor, + actorOrgId, + projectId, + emails + }: TDeleteProjectMembershipsDTO) => { + const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId, actorOrgId); + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Delete, ProjectPermissionSub.Member); + + const projectMembers = await projectMembershipDAL.findMembershipsByEmail(projectId, emails); + + if (projectMembers.length !== emails.length) { + throw new BadRequestError({ + message: "Some users are not part of project", + name: "Delete project membership" + }); + } + + if (actor === ActorType.USER && projectMembers.some(({ user }) => user.id === actorId)) { + throw new BadRequestError({ + message: "Cannot remove yourself from project", + name: "Delete project membership" + }); + } + + const memberships = await projectMembershipDAL.transaction(async (tx) => { + const deletedMemberships = await projectMembershipDAL.delete( + { + projectId, + $in: { + id: projectMembers.map(({ id }) => id) + } + }, + tx + ); + + await projectKeyDAL.delete( + { + projectId, + $in: { + receiverId: projectMembers.map(({ user }) => user.id).filter(Boolean) + } + }, + tx + ); + + return deletedMemberships; + }); + return memberships; + }; + return { getProjectMemberships, inviteUserToProject, updateProjectMembership, addUsersToProjectNonE2EE, - deleteProjectMembership, + deleteProjectMemberships, + deleteProjectMembership, // TODO: Remove this addUsersToProject }; }; From 26576b6bcd73d0890a5f476c21a5c382e47ff60d Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 18:20:24 +0100 Subject: [PATCH 040/157] Update project-membership-types.ts --- .../services/project-membership/project-membership-types.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/src/services/project-membership/project-membership-types.ts b/backend/src/services/project-membership/project-membership-types.ts index 4928ad522..634b51d77 100644 --- a/backend/src/services/project-membership/project-membership-types.ts +++ b/backend/src/services/project-membership/project-membership-types.ts @@ -12,10 +12,14 @@ export type TUpdateProjectMembershipDTO = { role: string; } & TProjectPermission; -export type TDeleteProjectMembershipDTO = { +export type TDeleteProjectMembershipOldDTO = { membershipId: string; } & TProjectPermission; +export type TDeleteProjectMembershipsDTO = { + emails: string[]; +} & TProjectPermission; + export type TAddUsersToWorkspaceDTO = { sendEmails?: boolean; members: { From f2dcc83a56cf13ba900b8f8fd5d8d9ed03acb515 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 18:20:37 +0100 Subject: [PATCH 041/157] New doc routes --- .../endpoints/workspaces/create-workspace.mdx | 4 ++++ .../endpoints/workspaces/delete-workspace.mdx | 8 ++++++++ .../endpoints/workspaces/invite-member-to-workspace.mdx | 4 ++++ .../endpoints/workspaces/remove-member-from-workspace.mdx | 4 ++++ 4 files changed, 20 insertions(+) create mode 100644 docs/api-reference/endpoints/workspaces/create-workspace.mdx create mode 100644 docs/api-reference/endpoints/workspaces/delete-workspace.mdx create mode 100644 docs/api-reference/endpoints/workspaces/invite-member-to-workspace.mdx create mode 100644 docs/api-reference/endpoints/workspaces/remove-member-from-workspace.mdx diff --git a/docs/api-reference/endpoints/workspaces/create-workspace.mdx b/docs/api-reference/endpoints/workspaces/create-workspace.mdx new file mode 100644 index 000000000..d0518df2d --- /dev/null +++ b/docs/api-reference/endpoints/workspaces/create-workspace.mdx @@ -0,0 +1,4 @@ +--- +title: "Create workspace" +openapi: "POST /api/v2/workspace" +--- \ No newline at end of file diff --git a/docs/api-reference/endpoints/workspaces/delete-workspace.mdx b/docs/api-reference/endpoints/workspaces/delete-workspace.mdx new file mode 100644 index 000000000..5037ca20d --- /dev/null +++ b/docs/api-reference/endpoints/workspaces/delete-workspace.mdx @@ -0,0 +1,8 @@ +--- +title: "Delete workspace" +openapi: "DELETE /api/v1/workspace/{workspaceId}" +--- + + + This operation is irreversible. All data associated with the project will be deleted. Please use with caution. + \ No newline at end of file diff --git a/docs/api-reference/endpoints/workspaces/invite-member-to-workspace.mdx b/docs/api-reference/endpoints/workspaces/invite-member-to-workspace.mdx new file mode 100644 index 000000000..a09c586ca --- /dev/null +++ b/docs/api-reference/endpoints/workspaces/invite-member-to-workspace.mdx @@ -0,0 +1,4 @@ +--- +title: "Invite Member" +openapi: "POST /api/v2/workspace/{projectId}/memberships" +--- \ No newline at end of file diff --git a/docs/api-reference/endpoints/workspaces/remove-member-from-workspace.mdx b/docs/api-reference/endpoints/workspaces/remove-member-from-workspace.mdx new file mode 100644 index 000000000..8b781a4e8 --- /dev/null +++ b/docs/api-reference/endpoints/workspaces/remove-member-from-workspace.mdx @@ -0,0 +1,4 @@ +--- +title: "Remove Member" +openapi: "DELETE /api/v2/workspace/{projectId}/memberships" +--- \ No newline at end of file From d3d429db37a1b54792eec38121fc99e51cf897cb Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 18:20:46 +0100 Subject: [PATCH 042/157] Delete membership by email --- frontend/src/components/basic/table/ProjectUsersTable.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/basic/table/ProjectUsersTable.tsx b/frontend/src/components/basic/table/ProjectUsersTable.tsx index 9f109b931..8ad06375b 100644 --- a/frontend/src/components/basic/table/ProjectUsersTable.tsx +++ b/frontend/src/components/basic/table/ProjectUsersTable.tsx @@ -59,8 +59,8 @@ const ProjectUsersTable = ({ userData, changeData, myUser, filter, isUserListLoa const workspaceId = router.query.id as string; // Delete the row in the table (e.g. a user) // #TODO: Add a pop-up that warns you that the user is going to be deleted. - const handleDelete = async (membershipId: string) => { - await deleteUserFromWorkspaceMutateAsync({ membershipId, workspaceId }); + const handleDelete = async (email: string) => { + await deleteUserFromWorkspaceMutateAsync({ emails: [email], workspaceId }); }; const handleRoleUpdate = async (index: number, e: string) => { @@ -368,7 +368,7 @@ const ProjectUsersTable = ({ userData, changeData, myUser, filter, isUserListLoa myRole !== "member" ? (
-
); }; From fb2a21321432f921ae34559b38143177022041e5 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Mon, 19 Feb 2024 22:01:32 +0100 Subject: [PATCH 082/157] Refactor migration to work with conflict/merge update logic --- .../secret-approval-request-secret-dal.ts | 39 ++- backend/src/lib/secret/index.ts | 269 +++++++++++++----- backend/src/services/org/org-service.ts | 3 +- backend/src/services/project/project-queue.ts | 203 ++++++++----- backend/src/services/secret/secret-dal.ts | 35 ++- .../src/services/secret/secret-version-dal.ts | 35 ++- 6 files changed, 396 insertions(+), 188 deletions(-) diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts index 93e5a2538..ff150fe67 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts @@ -5,7 +5,6 @@ import { SecretApprovalRequestsSecretsSchema, TableName, TSecretApprovalRequestsSecrets, - TSecretApprovalRequestsSecretsUpdate, TSecretTags } from "@app/db/schemas"; import { BadRequestError, DatabaseError } from "@app/lib/errors"; @@ -17,22 +16,32 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => { const secretApprovalRequestSecretOrm = ormify(db, TableName.SecretApprovalRequestSecret); const secretApprovalRequestSecretTagOrm = ormify(db, TableName.SecretApprovalRequestSecretTag); - const bulkUpdateNoVersionIncrement = async ( - data: Array<{ filter: Partial; data: TSecretApprovalRequestsSecretsUpdate }>, - tx?: Knex - ) => { + const bulkUpdateNoVersionIncrement = async (data: TSecretApprovalRequestsSecrets[], tx?: Knex) => { try { - const secs = await Promise.all( - data.map(async ({ filter, data: updateData }) => { - const [doc] = await (tx || db)(TableName.SecretApprovalRequestSecret) - .where(filter) - .update(updateData) - .returning("*"); - if (!doc) throw new BadRequestError({ message: "Failed to update document" }); - return doc; - }) + const existingApprovalSecrets = await secretApprovalRequestSecretOrm.find( + { + $in: { + id: data.map((el) => el.id) + } + }, + { tx } ); - return secs; + + if (existingApprovalSecrets.length !== data.length) { + throw new BadRequestError({ message: "Some of the secret approvals do not exist" }); + } + + const updatedApprovalSecrets = await (tx || db)(TableName.SecretApprovalRequestSecret) + .insert(data) + .onConflict("id") // this will cause a conflict then merge the data + .merge() // Merge the data with the existing data + .returning("*"); + + if (!updatedApprovalSecrets || updatedApprovalSecrets.length === 0) { + throw new BadRequestError({ message: "Failed to bulk update secret approvals" }); + } + + return updatedApprovalSecrets; } catch (error) { throw new DatabaseError({ error, name: "bulk update secret" }); } diff --git a/backend/src/lib/secret/index.ts b/backend/src/lib/secret/index.ts index 4292c7945..036b49970 100644 --- a/backend/src/lib/secret/index.ts +++ b/backend/src/lib/secret/index.ts @@ -1,46 +1,45 @@ import crypto from "crypto"; import { z } from "zod"; -import { TProjectKeys } from "@app/db/schemas"; -import { logger } from "@app/lib/logger"; +import { + SecretApprovalRequestsSecretsSchema, + SecretsSchema, + SecretVersionsSchema, + TProjectKeys, + TSecretApprovalRequestsSecrets, + TSecrets, + TSecretVersions +} from "@app/db/schemas"; import { decryptAsymmetric } from "../crypto"; -export enum SecretDocType { - Secret = "secret", - SecretVersion = "secretVersion", - ApprovalSecret = "approvalSecret" -} - -export interface TPartialSecret { - id: string; - secretKeyCiphertext: string; - secretKeyIV: string; - secretKeyTag: string; - - secretValueCiphertext: string; - secretValueIV: string; - secretValueTag: string; - - secretCommentCiphertext?: string | null; - secretCommentIV?: string | null; - secretCommentTag?: string | null; - - docType: SecretDocType; - keyEncoding: string; -} - -const PartialDecryptedSecretSchema = z.object({ +const DecryptedValuesSchema = z.object({ id: z.string(), secretKey: z.string(), secretValue: z.string(), - secretComment: z.string().optional(), - - docType: z.nativeEnum(SecretDocType) + secretComment: z.string().optional() }); -export type TPartialDecryptedSecret = z.infer; -const decryptSecret = ({ +const DecryptedSecretSchema = z.object({ + decrypted: DecryptedValuesSchema, + original: SecretsSchema +}); + +const DecryptedSecretVersionsSchema = z.object({ + decrypted: DecryptedValuesSchema, + original: SecretVersionsSchema +}); + +export const DecryptedSecretApprovalsSchema = z.object({ + decrypted: DecryptedValuesSchema, + original: SecretApprovalRequestsSecretsSchema +}); + +export type DecryptedSecret = z.infer; +export type DecryptedSecretVersions = z.infer; +export type DecryptedSecretApprovals = z.infer; + +const decryptCipher = ({ ciphertext, iv, tag, @@ -60,8 +59,62 @@ const decryptSecret = ({ return cleartext; }; +const getDecryptedValues = ({ + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretValueCiphertext, + secretValueIV, + secretValueTag, + + secretCommentCiphertext, + secretCommentIV, + secretCommentTag, + key +}: { + secretKeyCiphertext: string; + secretKeyIV: string; + secretKeyTag: string; + secretValueCiphertext: string; + secretValueIV: string; + secretValueTag: string; + secretCommentCiphertext?: string | null; + secretCommentIV?: string | null; + secretCommentTag?: string | null; + key: string | Buffer; +}) => { + const secretKey = decryptCipher({ + ciphertext: secretKeyCiphertext, + iv: secretKeyIV, + tag: secretKeyTag, + key + }); + + const secretValue = decryptCipher({ + ciphertext: secretValueCiphertext, + iv: secretValueIV, + tag: secretValueTag, + key + }); + + const secretComment = + secretCommentCiphertext && secretCommentIV && secretCommentTag + ? decryptCipher({ + ciphertext: secretCommentCiphertext, + iv: secretCommentIV, + tag: secretCommentTag, + key + }) + : ""; + + return { + secretKey, + secretValue, + secretComment + }; +}; export const decryptSecrets = ( - encryptedSecrets: TPartialSecret[], + encryptedSecrets: TSecrets[], privateKey: string, latestKey: TProjectKeys & { sender: { @@ -76,47 +129,123 @@ export const decryptSecrets = ( privateKey }); - const decryptedSecrets: TPartialDecryptedSecret[] = []; + const decryptedSecrets: DecryptedSecret[] = []; encryptedSecrets.forEach((encSecret) => { - try { - const secretKey = decryptSecret({ - ciphertext: encSecret.secretKeyCiphertext, - iv: encSecret.secretKeyIV, - tag: encSecret.secretKeyTag, - key - }); + const decrypted = getDecryptedValues({ + secretKeyCiphertext: encSecret.secretKeyCiphertext, + secretKeyIV: encSecret.secretKeyIV, + secretKeyTag: encSecret.secretKeyTag, + secretValueCiphertext: encSecret.secretValueCiphertext, + secretValueIV: encSecret.secretValueIV, + secretValueTag: encSecret.secretValueTag, + secretCommentCiphertext: encSecret.secretCommentCiphertext, + secretCommentIV: encSecret.secretCommentIV, + secretCommentTag: encSecret.secretCommentTag, + key + }); - const secretValue = decryptSecret({ - ciphertext: encSecret.secretValueCiphertext, - iv: encSecret.secretValueIV, - tag: encSecret.secretValueTag, - key - }); + const decryptedSecret: DecryptedSecret = { + decrypted: { + ...decrypted, + id: encSecret.id + }, + original: encSecret + }; - const secretComment = - encSecret.secretCommentCiphertext && encSecret.secretCommentIV && encSecret.secretCommentTag - ? decryptSecret({ - ciphertext: encSecret.secretCommentCiphertext, - iv: encSecret.secretCommentIV, - tag: encSecret.secretCommentTag, - key - }) - : ""; - - const decryptedSecret: TPartialDecryptedSecret = { - id: encSecret.id, - secretKey, - secretValue, - secretComment, - docType: encSecret.docType - }; - - decryptedSecrets.push(PartialDecryptedSecretSchema.parse(decryptedSecret)); - } catch (err) { - // This is ok, because we check that the decrypted secrets array length is the same as the encrypted secrets input array length. - logger.error(`[${encSecret.id}] - failed to decrypt`, err); - } + decryptedSecrets.push(DecryptedSecretSchema.parse(decryptedSecret)); + }); + + return decryptedSecrets; +}; + +export const decryptSecretVersions = ( + encryptedSecretVersions: TSecretVersions[], + privateKey: string, + latestKey: TProjectKeys & { + sender: { + publicKey: string; + }; + } +) => { + const key = decryptAsymmetric({ + ciphertext: latestKey.encryptedKey, + nonce: latestKey.nonce, + publicKey: latestKey.sender.publicKey, + privateKey + }); + + const decryptedSecrets: DecryptedSecretVersions[] = []; + + encryptedSecretVersions.forEach((encSecret) => { + const decrypted = getDecryptedValues({ + secretKeyCiphertext: encSecret.secretKeyCiphertext, + secretKeyIV: encSecret.secretKeyIV, + secretKeyTag: encSecret.secretKeyTag, + secretValueCiphertext: encSecret.secretValueCiphertext, + secretValueIV: encSecret.secretValueIV, + secretValueTag: encSecret.secretValueTag, + secretCommentCiphertext: encSecret.secretCommentCiphertext, + secretCommentIV: encSecret.secretCommentIV, + secretCommentTag: encSecret.secretCommentTag, + key + }); + + const decryptedSecret: DecryptedSecretVersions = { + decrypted: { + ...decrypted, + id: encSecret.id + }, + original: encSecret + }; + + decryptedSecrets.push(DecryptedSecretVersionsSchema.parse(decryptedSecret)); + }); + + return decryptedSecrets; +}; + +export const decryptSecretApprovals = ( + encryptedSecretApprovals: TSecretApprovalRequestsSecrets[], + privateKey: string, + latestKey: TProjectKeys & { + sender: { + publicKey: string; + }; + } +) => { + const key = decryptAsymmetric({ + ciphertext: latestKey.encryptedKey, + nonce: latestKey.nonce, + publicKey: latestKey.sender.publicKey, + privateKey + }); + + const decryptedSecrets: DecryptedSecretApprovals[] = []; + + encryptedSecretApprovals.forEach((encSecret) => { + const decrypted = getDecryptedValues({ + secretKeyCiphertext: encSecret.secretKeyCiphertext, + secretKeyIV: encSecret.secretKeyIV, + secretKeyTag: encSecret.secretKeyTag, + secretValueCiphertext: encSecret.secretValueCiphertext, + secretValueIV: encSecret.secretValueIV, + secretValueTag: encSecret.secretValueTag, + secretCommentCiphertext: encSecret.secretCommentCiphertext, + secretCommentIV: encSecret.secretCommentIV, + secretCommentTag: encSecret.secretCommentTag, + key + }); + + const decryptedSecret: DecryptedSecretApprovals = { + decrypted: { + ...decrypted, + id: encSecret.id + }, + original: encSecret + }; + + decryptedSecrets.push(DecryptedSecretApprovalsSchema.parse(decryptedSecret)); }); return decryptedSecrets; diff --git a/backend/src/services/org/org-service.ts b/backend/src/services/org/org-service.ts index 7a6e4c795..f499d2309 100644 --- a/backend/src/services/org/org-service.ts +++ b/backend/src/services/org/org-service.ts @@ -3,7 +3,6 @@ import slugify from "@sindresorhus/slugify"; import crypto from "crypto"; import jwt from "jsonwebtoken"; import { Knex } from "knex"; -import { nanoid } from "nanoid"; import { OrgMembershipRole, OrgMembershipStatus } from "@app/db/schemas"; import { TProjects } from "@app/db/schemas/projects"; @@ -133,7 +132,7 @@ export const orgServiceFactory = ({ }; const addGhostUser = async (orgId: string, tx?: Knex) => { - const email = `ghost-${nanoid(16)}-${orgId}@infisical.com`; // We add a nanoid because the email is unique. And we have to create a new ghost user each time, so we can have access to the private key. + const email = `ghost-${alphaNumericNanoId(16)}-${orgId}@infisical.com`; // We add a nanoid because the email is unique. And we have to create a new ghost user each time, so we can have access to the private key. const password = crypto.randomBytes(128).toString("hex"); const user = await userDAL.create( diff --git a/backend/src/services/project/project-queue.ts b/backend/src/services/project/project-queue.ts index 3f521a1e6..33aaaa361 100644 --- a/backend/src/services/project/project-queue.ts +++ b/backend/src/services/project/project-queue.ts @@ -3,8 +3,13 @@ import { ProjectMembershipRole, ProjectUpgradeStatus, ProjectVersion, + SecretApprovalRequestsSecretsSchema, SecretKeyEncoding, - TSecrets + SecretsSchema, + SecretVersionsSchema, + TSecretApprovalRequestsSecrets, + TSecrets, + TSecretVersions } from "@app/db/schemas"; import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal"; import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; @@ -16,7 +21,7 @@ import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { logger } from "@app/lib/logger"; -import { decryptSecrets, SecretDocType, TPartialSecret } from "@app/lib/secret"; +import { decryptSecretApprovals, decryptSecrets, decryptSecretVersions } from "@app/lib/secret"; import { QueueJobs, QueueName, TQueueJobTypes, TQueueServiceFactory } from "@app/queue"; import { TOrgDALFactory } from "../org/org-dal"; @@ -115,7 +120,9 @@ export const projectQueueFactory = ({ }); // Get all the secrets within the project (as encrypted) - const secrets: TPartialSecret[] = []; + const secrets: TSecrets[] = []; + const secretVersions: TSecretVersions[] = []; + const approvalSecrets: TSecretApprovalRequestsSecrets[] = []; for (const folder of projectFolders) { const folderSecrets = await secretDAL.find({ folderId: folder.id }); @@ -132,22 +139,30 @@ export const projectQueueFactory = ({ status: RequestState.Open, folderId: folder.id }); - const approvalSecrets = await secretApprovalSecretDAL.find({ + const secretApprovals = await secretApprovalSecretDAL.find({ $in: { requestId: approvalRequests.map((el) => el.id) } }); - secrets.push(...folderSecrets.map((el) => ({ ...el, docType: SecretDocType.Secret }))); - secrets.push(...folderSecretVersions.map((el) => ({ ...el, docType: SecretDocType.SecretVersion }))); - secrets.push(...approvalSecrets.map((el) => ({ ...el, docType: SecretDocType.ApprovalSecret }))); + secrets.push(...folderSecrets); + secretVersions.push(...folderSecretVersions); + approvalSecrets.push(...secretApprovals); } const decryptedSecrets = decryptSecrets(secrets, userPrivateKey, oldProjectKey); + const decryptedSecretVersions = decryptSecretVersions(secretVersions, userPrivateKey, oldProjectKey); + const decryptedApprovalSecrets = decryptSecretApprovals(approvalSecrets, userPrivateKey, oldProjectKey); if (secrets.length !== decryptedSecrets.length) { throw new Error("Failed to decrypt some secret versions"); } + if (secretVersions.length !== decryptedSecretVersions.length) { + throw new Error("Failed to decrypt some secret versions"); + } + if (approvalSecrets.length !== decryptedApprovalSecrets.length) { + throw new Error("Failed to decrypt some secret approvals"); + } // Get the existing bot and the existing project keys for the members of the project const existingBot = await projectBotDAL.findOne({ projectId: project.id }).catch(() => null); @@ -286,107 +301,137 @@ export const projectQueueFactory = ({ publicKey: ghostUser.keys.publicKey }); - type TPartialSecret = Pick< - TSecrets, - | "id" - | "secretKeyCiphertext" - | "secretKeyIV" - | "secretKeyTag" - | "secretValueCiphertext" - | "secretValueIV" - | "secretValueTag" - | "secretCommentCiphertext" - | "secretCommentIV" - | "secretCommentTag" - >; - - const updatedSecrets: TPartialSecret[] = []; - const updatedSecretVersions: TPartialSecret[] = []; - const updatedSecretApprovals: TPartialSecret[] = []; + const updatedSecrets: TSecrets[] = []; + const updatedSecretVersions: TSecretVersions[] = []; + const updatedSecretApprovals: TSecretApprovalRequestsSecrets[] = []; for (const rawSecret of decryptedSecrets) { - const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretKey, botKey); - const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretValue || "", botKey); - const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.secretComment || "", botKey); + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.decrypted.secretKey, botKey); + const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.decrypted.secretValue || "", botKey); + const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8( + rawSecret.decrypted.secretComment || "", + botKey + ); + + const payload: TSecrets = { + ...rawSecret.original, - const payload = { - id: rawSecret.id, secretKeyCiphertext: secretKeyEncrypted.ciphertext, secretKeyIV: secretKeyEncrypted.iv, secretKeyTag: secretKeyEncrypted.tag, + secretValueCiphertext: secretValueEncrypted.ciphertext, secretValueIV: secretValueEncrypted.iv, secretValueTag: secretValueEncrypted.tag, + secretCommentCiphertext: secretCommentEncrypted.ciphertext, secretCommentIV: secretCommentEncrypted.iv, secretCommentTag: secretCommentEncrypted.tag } as const; - if (rawSecret.docType === SecretDocType.Secret) { - updatedSecrets.push(payload); - } else if (rawSecret.docType === SecretDocType.SecretVersion) { - updatedSecretVersions.push(payload); - } else if (rawSecret.docType === SecretDocType.ApprovalSecret) { - updatedSecretApprovals.push(payload); - } else { - throw new Error("Unknown secret type"); + if (!SecretsSchema.safeParse(payload).success) { + throw new Error(`Invalid secret payload: ${JSON.stringify(payload)}`); } + + updatedSecrets.push(payload); } - const secretUpdates = await secretDAL.bulkUpdateNoVersionIncrement( - [ - ...updatedSecrets.map((secret) => ({ - filter: { id: secret.id }, - data: { - ...secret, - id: undefined - } - })) - ], - tx - ); + for (const rawSecretVersion of decryptedSecretVersions) { + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecretVersion.decrypted.secretKey, botKey); + const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8( + rawSecretVersion.decrypted.secretValue || "", + botKey + ); + const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8( + rawSecretVersion.decrypted.secretComment || "", + botKey + ); - const secretVersionUpdates = await secretVersionDAL.bulkUpdateNoVersionIncrement( - [ - ...updatedSecretVersions.map((version) => ({ - filter: { id: version.id }, - data: { - ...version, - id: undefined - } - })) - ], - tx - ); + const payload: TSecretVersions = { + ...rawSecretVersion.original, - const secretApprovalUpdates = await secretApprovalSecretDAL.bulkUpdateNoVersionIncrement( - [ - ...updatedSecretApprovals.map((approval) => ({ - filter: { - id: approval.id - }, - data: { - ...approval, - id: undefined - } - })) - ], - tx - ); + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, - if (secretUpdates.length !== updatedSecrets.length) { + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + + secretCommentCiphertext: secretCommentEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag + } as const; + + if (!SecretVersionsSchema.safeParse(payload).success) { + throw new Error(`Invalid secret version payload: ${JSON.stringify(payload)}`); + } + + updatedSecretVersions.push(payload); + } + + for (const rawSecretApproval of decryptedApprovalSecrets) { + const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecretApproval.decrypted.secretKey, botKey); + const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8( + rawSecretApproval.decrypted.secretValue || "", + botKey + ); + const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8( + rawSecretApproval.decrypted.secretComment || "", + botKey + ); + + const payload: TSecretApprovalRequestsSecrets = { + ...rawSecretApproval.original, + + secretKeyCiphertext: secretKeyEncrypted.ciphertext, + secretKeyIV: secretKeyEncrypted.iv, + secretKeyTag: secretKeyEncrypted.tag, + + secretValueCiphertext: secretValueEncrypted.ciphertext, + secretValueIV: secretValueEncrypted.iv, + secretValueTag: secretValueEncrypted.tag, + + secretCommentCiphertext: secretCommentEncrypted.ciphertext, + secretCommentIV: secretCommentEncrypted.iv, + secretCommentTag: secretCommentEncrypted.tag + } as const; + + if (!SecretApprovalRequestsSecretsSchema.safeParse(payload).success) { + throw new Error(`Invalid secret approval payload: ${JSON.stringify(payload)}`); + } + + updatedSecretApprovals.push(payload); + } + + if (updatedSecrets.length !== secrets.length) { throw new Error("Failed to update some secrets"); } - if (secretVersionUpdates.length !== updatedSecretVersions.length) { + if (updatedSecretVersions.length !== secretVersions.length) { throw new Error("Failed to update some secret versions"); } - if (secretApprovalUpdates.length !== updatedSecretApprovals.length) { + if (updatedSecretApprovals.length !== approvalSecrets.length) { throw new Error("Failed to update some secret approvals"); } + const secretUpdates = await secretDAL.bulkUpdateNoVersionIncrement(updatedSecrets, tx); + const secretVersionUpdates = await secretVersionDAL.bulkUpdateNoVersionIncrement(updatedSecretVersions, tx); + const secretApprovalUpdates = await secretApprovalSecretDAL.bulkUpdateNoVersionIncrement( + updatedSecretApprovals, + tx + ); + + if ( + secretUpdates.length !== updatedSecrets.length || + secretVersionUpdates.length !== updatedSecretVersions.length || + secretApprovalUpdates.length !== updatedSecretApprovals.length + ) { + throw new Error("Parts of the upgrade failed. Some secrets were not updated"); + } + await projectDAL.setProjectUpgradeStatus(data.projectId, null, tx); // await new Promise((resolve) => setTimeout(resolve, 15_000)); - throw new Error("Transaction was successful!"); + // throw new Error("Transaction was successful!"); }); } catch (err) { const [project] = await projectDAL diff --git a/backend/src/services/secret/secret-dal.ts b/backend/src/services/secret/secret-dal.ts index b3b538168..9c880938c 100644 --- a/backend/src/services/secret/secret-dal.ts +++ b/backend/src/services/secret/secret-dal.ts @@ -45,19 +45,32 @@ export const secretDALFactory = (db: TDbClient) => { } }; - const bulkUpdateNoVersionIncrement = async ( - data: Array<{ filter: Partial; data: TSecretsUpdate }>, - tx?: Knex - ) => { + const bulkUpdateNoVersionIncrement = async (data: TSecrets[], tx?: Knex) => { try { - const secs = await Promise.all( - data.map(async ({ filter, data: updateData }) => { - const [doc] = await (tx || db)(TableName.Secret).where(filter).update(updateData).returning("*"); - if (!doc) throw new BadRequestError({ message: "Failed to update document" }); - return doc; - }) + const existingSecrets = await secretOrm.find( + { + $in: { + id: data.map((el) => el.id) + } + }, + { tx } ); - return secs; + + if (existingSecrets.length !== data.length) { + throw new BadRequestError({ message: "Some of the secrets do not exist" }); + } + + const updatedSecrets = await (tx || db)(TableName.Secret) + .insert(data) + .onConflict("id") // this will cause a conflict then merge the data + .merge() // Merge the data with the existing data + .returning("*"); + + if (!updatedSecrets || updatedSecrets.length === 0) { + throw new BadRequestError({ message: "Failed to bulk update secret approvals" }); + } + + return updatedSecrets; } catch (error) { throw new DatabaseError({ error, name: "bulk update secret" }); } diff --git a/backend/src/services/secret/secret-version-dal.ts b/backend/src/services/secret/secret-version-dal.ts index 275150f46..c9d9cff91 100644 --- a/backend/src/services/secret/secret-version-dal.ts +++ b/backend/src/services/secret/secret-version-dal.ts @@ -58,19 +58,32 @@ export const secretVersionDALFactory = (db: TDbClient) => { } }; - const bulkUpdateNoVersionIncrement = async ( - data: Array<{ filter: Partial; data: TSecretVersionsUpdate }>, - tx?: Knex - ) => { + const bulkUpdateNoVersionIncrement = async (data: TSecretVersions[], tx?: Knex) => { try { - const secs = await Promise.all( - data.map(async ({ filter, data: updateData }) => { - const [doc] = await (tx || db)(TableName.SecretVersion).where(filter).update(updateData).returning("*"); - if (!doc) throw new BadRequestError({ message: "Failed to update document" }); - return doc; - }) + const existingSecretVersions = await secretVersionOrm.find( + { + $in: { + id: data.map((el) => el.id) + } + }, + { tx } ); - return secs; + + if (existingSecretVersions.length !== data.length) { + throw new BadRequestError({ message: "Some of the secret versions do not exist" }); + } + + const updatedSecretVersions = await (tx || db)(TableName.SecretVersion) + .insert(data) + .onConflict("id") // this will cause a conflict then merge the data + .merge() // Merge the data with the existing data + .returning("*"); + + if (!updatedSecretVersions || updatedSecretVersions.length === 0) { + throw new BadRequestError({ message: "Failed to bulk update secret versions" }); + } + + return updatedSecretVersions; } catch (error) { throw new DatabaseError({ error, name: "bulk update secret" }); } From 66e57d5d11f6227afb0fd7c95fb816204fc072cd Mon Sep 17 00:00:00 2001 From: Maidul Islam Date: Mon, 19 Feb 2024 19:01:36 -0500 Subject: [PATCH 083/157] correct error log --- backend/src/services/project/project-queue.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/services/project/project-queue.ts b/backend/src/services/project/project-queue.ts index 33aaaa361..bea0074d2 100644 --- a/backend/src/services/project/project-queue.ts +++ b/backend/src/services/project/project-queue.ts @@ -445,7 +445,7 @@ export const projectQueueFactory = ({ logger.error("Failed to upgrade project, because no project was found", data); } else { await projectDAL.setProjectUpgradeStatus(data.projectId, ProjectUpgradeStatus.Failed); - logger.error("Failed to upgrade project", data, err); + logger.error(err, "Failed to upgrade project"); } throw err; @@ -453,7 +453,7 @@ export const projectQueueFactory = ({ }); queueService.listen(QueueName.UpgradeProjectToGhost, "failed", (job, err) => { - logger.error("Upgrade project failed", job?.data, err); + logger.error(err, "Upgrade project failed", job?.data); }); return { From 75813deb81dd52c3a12a2df44aa9b6698f4c3c49 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 20 Feb 2024 02:34:30 +0100 Subject: [PATCH 084/157] Fixed integrations & bulk update issue --- .../secret-approval-request-secret-dal.ts | 6 +- backend/src/lib/secret/index.ts | 262 ++++++++++-------- backend/src/server/routes/index.ts | 1 + .../integration-auth/integration-auth-dal.ts | 29 +- backend/src/services/project/project-queue.ts | 65 ++++- backend/src/services/secret/secret-dal.ts | 6 +- .../src/services/secret/secret-version-dal.ts | 6 +- 7 files changed, 249 insertions(+), 126 deletions(-) diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts index ff150fe67..736cd253e 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-secret-dal.ts @@ -31,16 +31,14 @@ export const secretApprovalRequestSecretDALFactory = (db: TDbClient) => { throw new BadRequestError({ message: "Some of the secret approvals do not exist" }); } + if (data.length === 0) return []; + const updatedApprovalSecrets = await (tx || db)(TableName.SecretApprovalRequestSecret) .insert(data) .onConflict("id") // this will cause a conflict then merge the data .merge() // Merge the data with the existing data .returning("*"); - if (!updatedApprovalSecrets || updatedApprovalSecrets.length === 0) { - throw new BadRequestError({ message: "Failed to bulk update secret approvals" }); - } - return updatedApprovalSecrets; } catch (error) { throw new DatabaseError({ error, name: "bulk update secret" }); diff --git a/backend/src/lib/secret/index.ts b/backend/src/lib/secret/index.ts index 036b49970..dd23c318a 100644 --- a/backend/src/lib/secret/index.ts +++ b/backend/src/lib/secret/index.ts @@ -2,9 +2,11 @@ import crypto from "crypto"; import { z } from "zod"; import { + IntegrationAuthsSchema, SecretApprovalRequestsSecretsSchema, SecretsSchema, SecretVersionsSchema, + TIntegrationAuths, TProjectKeys, TSecretApprovalRequestsSecrets, TSecrets, @@ -25,6 +27,16 @@ const DecryptedSecretSchema = z.object({ original: SecretsSchema }); +const DecryptedIntegrationAuthsSchema = z.object({ + decrypted: z.object({ + id: z.string(), + access: z.string(), + accessId: z.string(), + refresh: z.string() + }), + original: IntegrationAuthsSchema +}); + const DecryptedSecretVersionsSchema = z.object({ decrypted: DecryptedValuesSchema, original: SecretVersionsSchema @@ -38,6 +50,13 @@ export const DecryptedSecretApprovalsSchema = z.object({ export type DecryptedSecret = z.infer; export type DecryptedSecretVersions = z.infer; export type DecryptedSecretApprovals = z.infer; +export type DecryptedIntegrationAuths = z.infer; + +type TLatestKey = TProjectKeys & { + sender: { + publicKey: string; + }; +}; const decryptCipher = ({ ciphertext, @@ -59,69 +78,20 @@ const decryptCipher = ({ return cleartext; }; -const getDecryptedValues = ({ - secretKeyCiphertext, - secretKeyIV, - secretKeyTag, - secretValueCiphertext, - secretValueIV, - secretValueTag, +const getDecryptedValues = (data: Array<{ ciphertext: string; iv: string; tag: string }>, key: string | Buffer) => { + const results = []; - secretCommentCiphertext, - secretCommentIV, - secretCommentTag, - key -}: { - secretKeyCiphertext: string; - secretKeyIV: string; - secretKeyTag: string; - secretValueCiphertext: string; - secretValueIV: string; - secretValueTag: string; - secretCommentCiphertext?: string | null; - secretCommentIV?: string | null; - secretCommentTag?: string | null; - key: string | Buffer; -}) => { - const secretKey = decryptCipher({ - ciphertext: secretKeyCiphertext, - iv: secretKeyIV, - tag: secretKeyTag, - key - }); - - const secretValue = decryptCipher({ - ciphertext: secretValueCiphertext, - iv: secretValueIV, - tag: secretValueTag, - key - }); - - const secretComment = - secretCommentCiphertext && secretCommentIV && secretCommentTag - ? decryptCipher({ - ciphertext: secretCommentCiphertext, - iv: secretCommentIV, - tag: secretCommentTag, - key - }) - : ""; - - return { - secretKey, - secretValue, - secretComment - }; -}; -export const decryptSecrets = ( - encryptedSecrets: TSecrets[], - privateKey: string, - latestKey: TProjectKeys & { - sender: { - publicKey: string; - }; + for (const { ciphertext, iv, tag } of data) { + if (!ciphertext || !iv || !tag) { + results.push(""); + } else { + results.push(decryptCipher({ ciphertext, iv, tag, key })); + } } -) => { + + return results; +}; +export const decryptSecrets = (encryptedSecrets: TSecrets[], privateKey: string, latestKey: TLatestKey) => { const key = decryptAsymmetric({ ciphertext: latestKey.encryptedKey, nonce: latestKey.nonce, @@ -132,22 +102,32 @@ export const decryptSecrets = ( const decryptedSecrets: DecryptedSecret[] = []; encryptedSecrets.forEach((encSecret) => { - const decrypted = getDecryptedValues({ - secretKeyCiphertext: encSecret.secretKeyCiphertext, - secretKeyIV: encSecret.secretKeyIV, - secretKeyTag: encSecret.secretKeyTag, - secretValueCiphertext: encSecret.secretValueCiphertext, - secretValueIV: encSecret.secretValueIV, - secretValueTag: encSecret.secretValueTag, - secretCommentCiphertext: encSecret.secretCommentCiphertext, - secretCommentIV: encSecret.secretCommentIV, - secretCommentTag: encSecret.secretCommentTag, + const [secretKey, secretValue, secretComment] = getDecryptedValues( + [ + { + ciphertext: encSecret.secretKeyCiphertext, + iv: encSecret.secretKeyIV, + tag: encSecret.secretKeyTag + }, + { + ciphertext: encSecret.secretValueCiphertext, + iv: encSecret.secretValueIV, + tag: encSecret.secretValueTag + }, + { + ciphertext: encSecret.secretCommentCiphertext || "", + iv: encSecret.secretCommentIV || "", + tag: encSecret.secretCommentTag || "" + } + ], key - }); + ); const decryptedSecret: DecryptedSecret = { decrypted: { - ...decrypted, + secretKey, + secretValue, + secretComment, id: encSecret.id }, original: encSecret @@ -162,11 +142,7 @@ export const decryptSecrets = ( export const decryptSecretVersions = ( encryptedSecretVersions: TSecretVersions[], privateKey: string, - latestKey: TProjectKeys & { - sender: { - publicKey: string; - }; - } + latestKey: TLatestKey ) => { const key = decryptAsymmetric({ ciphertext: latestKey.encryptedKey, @@ -178,22 +154,32 @@ export const decryptSecretVersions = ( const decryptedSecrets: DecryptedSecretVersions[] = []; encryptedSecretVersions.forEach((encSecret) => { - const decrypted = getDecryptedValues({ - secretKeyCiphertext: encSecret.secretKeyCiphertext, - secretKeyIV: encSecret.secretKeyIV, - secretKeyTag: encSecret.secretKeyTag, - secretValueCiphertext: encSecret.secretValueCiphertext, - secretValueIV: encSecret.secretValueIV, - secretValueTag: encSecret.secretValueTag, - secretCommentCiphertext: encSecret.secretCommentCiphertext, - secretCommentIV: encSecret.secretCommentIV, - secretCommentTag: encSecret.secretCommentTag, + const [secretKey, secretValue, secretComment] = getDecryptedValues( + [ + { + ciphertext: encSecret.secretKeyCiphertext, + iv: encSecret.secretKeyIV, + tag: encSecret.secretKeyTag + }, + { + ciphertext: encSecret.secretValueCiphertext, + iv: encSecret.secretValueIV, + tag: encSecret.secretValueTag + }, + { + ciphertext: encSecret.secretCommentCiphertext || "", + iv: encSecret.secretCommentIV || "", + tag: encSecret.secretCommentTag || "" + } + ], key - }); + ); const decryptedSecret: DecryptedSecretVersions = { decrypted: { - ...decrypted, + secretKey, + secretValue, + secretComment, id: encSecret.id }, original: encSecret @@ -208,11 +194,7 @@ export const decryptSecretVersions = ( export const decryptSecretApprovals = ( encryptedSecretApprovals: TSecretApprovalRequestsSecrets[], privateKey: string, - latestKey: TProjectKeys & { - sender: { - publicKey: string; - }; - } + latestKey: TLatestKey ) => { const key = decryptAsymmetric({ ciphertext: latestKey.encryptedKey, @@ -223,26 +205,36 @@ export const decryptSecretApprovals = ( const decryptedSecrets: DecryptedSecretApprovals[] = []; - encryptedSecretApprovals.forEach((encSecret) => { - const decrypted = getDecryptedValues({ - secretKeyCiphertext: encSecret.secretKeyCiphertext, - secretKeyIV: encSecret.secretKeyIV, - secretKeyTag: encSecret.secretKeyTag, - secretValueCiphertext: encSecret.secretValueCiphertext, - secretValueIV: encSecret.secretValueIV, - secretValueTag: encSecret.secretValueTag, - secretCommentCiphertext: encSecret.secretCommentCiphertext, - secretCommentIV: encSecret.secretCommentIV, - secretCommentTag: encSecret.secretCommentTag, + encryptedSecretApprovals.forEach((encApproval) => { + const [secretKey, secretValue, secretComment] = getDecryptedValues( + [ + { + ciphertext: encApproval.secretKeyCiphertext, + iv: encApproval.secretKeyIV, + tag: encApproval.secretKeyTag + }, + { + ciphertext: encApproval.secretValueCiphertext, + iv: encApproval.secretValueIV, + tag: encApproval.secretValueTag + }, + { + ciphertext: encApproval.secretCommentCiphertext || "", + iv: encApproval.secretCommentIV || "", + tag: encApproval.secretCommentTag || "" + } + ], key - }); + ); const decryptedSecret: DecryptedSecretApprovals = { decrypted: { - ...decrypted, - id: encSecret.id + secretKey, + secretValue, + secretComment, + id: encApproval.id }, - original: encSecret + original: encApproval }; decryptedSecrets.push(DecryptedSecretApprovalsSchema.parse(decryptedSecret)); @@ -250,3 +242,53 @@ export const decryptSecretApprovals = ( return decryptedSecrets; }; + +export const decryptIntegrationAuths = ( + encryptedIntegrationAuths: TIntegrationAuths[], + privateKey: string, + latestKey: TLatestKey +) => { + const key = decryptAsymmetric({ + ciphertext: latestKey.encryptedKey, + nonce: latestKey.nonce, + publicKey: latestKey.sender.publicKey, + privateKey + }); + + const decryptedIntegrationAuths: DecryptedIntegrationAuths[] = []; + + encryptedIntegrationAuths.forEach((encAuth) => { + const [access, accessId, refresh] = getDecryptedValues( + [ + { + ciphertext: encAuth.accessCiphertext || "", + iv: encAuth.accessIV || "", + tag: encAuth.accessTag || "" + }, + { + ciphertext: encAuth.accessIdCiphertext || "", + iv: encAuth.accessIdIV || "", + tag: encAuth.accessIdTag || "" + }, + { + ciphertext: encAuth.refreshCiphertext || "", + iv: encAuth.refreshIV || "", + tag: encAuth.refreshTag || "" + } + ], + key + ); + + decryptedIntegrationAuths.push({ + decrypted: { + id: encAuth.id, + access, + accessId, + refresh + }, + original: encAuth + }); + }); + + return decryptedIntegrationAuths; +}; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index fd1b45878..9444aff16 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -307,6 +307,7 @@ export const registerRoutes = async ( folderDAL, projectDAL, orgDAL, + integrationAuthDAL, orgService, projectEnvDAL, userDAL, diff --git a/backend/src/services/integration-auth/integration-auth-dal.ts b/backend/src/services/integration-auth/integration-auth-dal.ts index f2d9c9ea2..d32cd1579 100644 --- a/backend/src/services/integration-auth/integration-auth-dal.ts +++ b/backend/src/services/integration-auth/integration-auth-dal.ts @@ -1,10 +1,35 @@ +import { Knex } from "knex"; + import { TDbClient } from "@app/db"; -import { TableName } from "@app/db/schemas"; +import { TableName, TIntegrationAuths, TIntegrationAuthsUpdate } from "@app/db/schemas"; +import { BadRequestError, DatabaseError } from "@app/lib/errors"; import { ormify } from "@app/lib/knex"; export type TIntegrationAuthDALFactory = ReturnType; export const integrationAuthDALFactory = (db: TDbClient) => { const integrationAuthOrm = ormify(db, TableName.IntegrationAuth); - return integrationAuthOrm; + + const bulkUpdate = async ( + data: Array<{ filter: Partial; data: TIntegrationAuthsUpdate }>, + tx?: Knex + ) => { + try { + const integrationAuths = await Promise.all( + data.map(async ({ filter, data: updateData }) => { + const [doc] = await (tx || db)(TableName.IntegrationAuth).where(filter).update(updateData).returning("*"); + if (!doc) throw new BadRequestError({ message: "Failed to update document" }); + return doc; + }) + ); + return integrationAuths; + } catch (error) { + throw new DatabaseError({ error, name: "bulk update secret" }); + } + }; + + return { + ...integrationAuthOrm, + bulkUpdate + }; }; diff --git a/backend/src/services/project/project-queue.ts b/backend/src/services/project/project-queue.ts index bea0074d2..8582c8289 100644 --- a/backend/src/services/project/project-queue.ts +++ b/backend/src/services/project/project-queue.ts @@ -1,5 +1,6 @@ /* eslint-disable no-await-in-loop */ import { + IntegrationAuthsSchema, ProjectMembershipRole, ProjectUpgradeStatus, ProjectVersion, @@ -7,6 +8,7 @@ import { SecretKeyEncoding, SecretsSchema, SecretVersionsSchema, + TIntegrationAuths, TSecretApprovalRequestsSecrets, TSecrets, TSecretVersions @@ -21,9 +23,15 @@ import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { logger } from "@app/lib/logger"; -import { decryptSecretApprovals, decryptSecrets, decryptSecretVersions } from "@app/lib/secret"; +import { + decryptIntegrationAuths, + decryptSecretApprovals, + decryptSecrets, + decryptSecretVersions +} from "@app/lib/secret"; import { QueueJobs, QueueName, TQueueJobTypes, TQueueServiceFactory } from "@app/queue"; +import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal"; import { TOrgDALFactory } from "../org/org-dal"; import { TOrgServiceFactory } from "../org/org-service"; import { TProjectBotDALFactory } from "../project-bot/project-bot-dal"; @@ -50,6 +58,7 @@ type TProjectQueueFactoryDep = { projectBotDAL: Pick; orgService: Pick; projectMembershipDAL: Pick; + integrationAuthDAL: TIntegrationAuthDALFactory; userDAL: Pick; projectEnvDAL: Pick; @@ -63,6 +72,7 @@ export const projectQueueFactory = ({ folderDAL, userDAL, secretVersionDAL, + integrationAuthDAL, secretApprovalRequestDAL, secretApprovalSecretDAL, projectKeyDAL, @@ -150,9 +160,14 @@ export const projectQueueFactory = ({ approvalSecrets.push(...secretApprovals); } + const projectIntegrationAuths = await integrationAuthDAL.find({ + projectId: project.id + }); + const decryptedSecrets = decryptSecrets(secrets, userPrivateKey, oldProjectKey); const decryptedSecretVersions = decryptSecretVersions(secretVersions, userPrivateKey, oldProjectKey); const decryptedApprovalSecrets = decryptSecretApprovals(approvalSecrets, userPrivateKey, oldProjectKey); + const decryptedIntegrationAuths = decryptIntegrationAuths(projectIntegrationAuths, userPrivateKey, oldProjectKey); if (secrets.length !== decryptedSecrets.length) { throw new Error("Failed to decrypt some secret versions"); @@ -304,6 +319,7 @@ export const projectQueueFactory = ({ const updatedSecrets: TSecrets[] = []; const updatedSecretVersions: TSecretVersions[] = []; const updatedSecretApprovals: TSecretApprovalRequestsSecrets[] = []; + const updatedIntegrationAuths: TIntegrationAuths[] = []; for (const rawSecret of decryptedSecrets) { const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.decrypted.secretKey, botKey); const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(rawSecret.decrypted.secretValue || "", botKey); @@ -348,6 +364,7 @@ export const projectQueueFactory = ({ const payload: TSecretVersions = { ...rawSecretVersion.original, + keyEncoding: SecretKeyEncoding.UTF8, secretKeyCiphertext: secretKeyEncrypted.ciphertext, secretKeyIV: secretKeyEncrypted.iv, @@ -382,6 +399,7 @@ export const projectQueueFactory = ({ const payload: TSecretApprovalRequestsSecrets = { ...rawSecretApproval.original, + keyEncoding: SecretKeyEncoding.UTF8, secretKeyCiphertext: secretKeyEncrypted.ciphertext, secretKeyIV: secretKeyEncrypted.iv, @@ -403,6 +421,35 @@ export const projectQueueFactory = ({ updatedSecretApprovals.push(payload); } + for (const integrationAuth of decryptedIntegrationAuths) { + const access = encryptSymmetric128BitHexKeyUTF8(integrationAuth.decrypted.access, botKey); + const accessId = encryptSymmetric128BitHexKeyUTF8(integrationAuth.decrypted.accessId, botKey); + const refresh = encryptSymmetric128BitHexKeyUTF8(integrationAuth.decrypted.refresh, botKey); + + const payload: TIntegrationAuths = { + ...integrationAuth.original, + keyEncoding: SecretKeyEncoding.UTF8, + + accessCiphertext: access.ciphertext, + accessIV: access.iv, + accessTag: access.tag, + + accessIdCiphertext: accessId.ciphertext, + accessIdIV: accessId.iv, + accessIdTag: accessId.tag, + + refreshCiphertext: refresh.ciphertext, + refreshIV: refresh.iv, + refreshTag: refresh.tag + } as const; + + if (!IntegrationAuthsSchema.safeParse(payload).success) { + throw new Error(`Invalid integration auth payload: ${JSON.stringify(payload)}`); + } + + updatedIntegrationAuths.push(payload); + } + if (updatedSecrets.length !== secrets.length) { throw new Error("Failed to update some secrets"); } @@ -412,6 +459,9 @@ export const projectQueueFactory = ({ if (updatedSecretApprovals.length !== approvalSecrets.length) { throw new Error("Failed to update some secret approvals"); } + if (updatedIntegrationAuths.length !== projectIntegrationAuths.length) { + throw new Error("Failed to update some integration auths"); + } const secretUpdates = await secretDAL.bulkUpdateNoVersionIncrement(updatedSecrets, tx); const secretVersionUpdates = await secretVersionDAL.bulkUpdateNoVersionIncrement(updatedSecretVersions, tx); @@ -419,11 +469,22 @@ export const projectQueueFactory = ({ updatedSecretApprovals, tx ); + const integrationAuthUpdates = await integrationAuthDAL.bulkUpdate( + updatedIntegrationAuths.map((el) => ({ + filter: { id: el.id }, + data: { + ...el, + id: undefined + } + })), + tx + ); if ( secretUpdates.length !== updatedSecrets.length || secretVersionUpdates.length !== updatedSecretVersions.length || - secretApprovalUpdates.length !== updatedSecretApprovals.length + secretApprovalUpdates.length !== updatedSecretApprovals.length || + integrationAuthUpdates.length !== updatedIntegrationAuths.length ) { throw new Error("Parts of the upgrade failed. Some secrets were not updated"); } diff --git a/backend/src/services/secret/secret-dal.ts b/backend/src/services/secret/secret-dal.ts index 9c880938c..11cd522ca 100644 --- a/backend/src/services/secret/secret-dal.ts +++ b/backend/src/services/secret/secret-dal.ts @@ -60,16 +60,14 @@ export const secretDALFactory = (db: TDbClient) => { throw new BadRequestError({ message: "Some of the secrets do not exist" }); } + if (data.length === 0) return []; + const updatedSecrets = await (tx || db)(TableName.Secret) .insert(data) .onConflict("id") // this will cause a conflict then merge the data .merge() // Merge the data with the existing data .returning("*"); - if (!updatedSecrets || updatedSecrets.length === 0) { - throw new BadRequestError({ message: "Failed to bulk update secret approvals" }); - } - return updatedSecrets; } catch (error) { throw new DatabaseError({ error, name: "bulk update secret" }); diff --git a/backend/src/services/secret/secret-version-dal.ts b/backend/src/services/secret/secret-version-dal.ts index c9d9cff91..758352ed2 100644 --- a/backend/src/services/secret/secret-version-dal.ts +++ b/backend/src/services/secret/secret-version-dal.ts @@ -73,16 +73,14 @@ export const secretVersionDALFactory = (db: TDbClient) => { throw new BadRequestError({ message: "Some of the secret versions do not exist" }); } + if (data.length === 0) return []; + const updatedSecretVersions = await (tx || db)(TableName.SecretVersion) .insert(data) .onConflict("id") // this will cause a conflict then merge the data .merge() // Merge the data with the existing data .returning("*"); - if (!updatedSecretVersions || updatedSecretVersions.length === 0) { - throw new BadRequestError({ message: "Failed to bulk update secret versions" }); - } - return updatedSecretVersions; } catch (error) { throw new DatabaseError({ error, name: "bulk update secret" }); From 0f3ec51d1499eaa166414da4723df63063b64cf0 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 20 Feb 2024 02:46:33 +0100 Subject: [PATCH 085/157] Update project-queue.ts --- backend/src/services/project/project-queue.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/services/project/project-queue.ts b/backend/src/services/project/project-queue.ts index 8582c8289..a63f9aea8 100644 --- a/backend/src/services/project/project-queue.ts +++ b/backend/src/services/project/project-queue.ts @@ -330,6 +330,7 @@ export const projectQueueFactory = ({ const payload: TSecrets = { ...rawSecret.original, + keyEncoding: SecretKeyEncoding.UTF8, secretKeyCiphertext: secretKeyEncrypted.ciphertext, secretKeyIV: secretKeyEncrypted.iv, From 28287b8ed4e73841f39755452a0d6351eb4e84a5 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 20 Feb 2024 03:23:28 +0100 Subject: [PATCH 086/157] Update project-queue.ts --- backend/src/services/project/project-queue.ts | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/backend/src/services/project/project-queue.ts b/backend/src/services/project/project-queue.ts index a63f9aea8..4f6ec9b41 100644 --- a/backend/src/services/project/project-queue.ts +++ b/backend/src/services/project/project-queue.ts @@ -49,7 +49,7 @@ export type TProjectQueueFactory = ReturnType; type TProjectQueueFactoryDep = { queueService: TQueueServiceFactory; - secretVersionDAL: Pick; + secretVersionDAL: Pick; folderDAL: Pick; secretDAL: Pick; projectKeyDAL: Pick; @@ -133,6 +133,7 @@ export const projectQueueFactory = ({ const secrets: TSecrets[] = []; const secretVersions: TSecretVersions[] = []; const approvalSecrets: TSecretApprovalRequestsSecrets[] = []; + const folderSecretVersionIdsToDelete: string[] = []; for (const folder of projectFolders) { const folderSecrets = await secretDAL.find({ folderId: folder.id }); @@ -142,9 +143,21 @@ export const projectQueueFactory = ({ }, // Only get the latest 100 secret versions for each folder. { - limit: 100 + limit: 100, + sort: [["createdAt", "desc"]] } ); + + const deletedSecretVersions = await secretVersionDAL.find( + { + folderId: folder.id + }, + { + offset: 100 + } + ); + folderSecretVersionIdsToDelete.push(...deletedSecretVersions.map((el) => el.id)); + const approvalRequests = await secretApprovalRequestDAL.find({ status: RequestState.Open, folderId: folder.id @@ -481,6 +494,16 @@ export const projectQueueFactory = ({ tx ); + // Delete all secret versions that are no longer needed. We only store the latest 100 versions for each secret. + await secretVersionDAL.delete( + { + $in: { + id: folderSecretVersionIdsToDelete + } + }, + tx + ); + if ( secretUpdates.length !== updatedSecrets.length || secretVersionUpdates.length !== updatedSecretVersions.length || From f35f10558bed8cdfc8c7c54eb35b152d154c9ea7 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 02:49:57 +0100 Subject: [PATCH 087/157] Get last 700 secret versions instead of 100 --- backend/src/services/project/project-queue.ts | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/backend/src/services/project/project-queue.ts b/backend/src/services/project/project-queue.ts index 4f6ec9b41..8e924af1a 100644 --- a/backend/src/services/project/project-queue.ts +++ b/backend/src/services/project/project-queue.ts @@ -130,10 +130,14 @@ export const projectQueueFactory = ({ }); // Get all the secrets within the project (as encrypted) + const projectIntegrationAuths = await integrationAuthDAL.find({ + projectId: project.id + }); const secrets: TSecrets[] = []; const secretVersions: TSecretVersions[] = []; const approvalSecrets: TSecretApprovalRequestsSecrets[] = []; const folderSecretVersionIdsToDelete: string[] = []; + for (const folder of projectFolders) { const folderSecrets = await secretDAL.find({ folderId: folder.id }); @@ -141,9 +145,9 @@ export const projectQueueFactory = ({ { folderId: folder.id }, - // Only get the latest 100 secret versions for each folder. + // Only get the latest 700 secret versions for each folder. { - limit: 100, + limit: 700, sort: [["createdAt", "desc"]] } ); @@ -153,7 +157,8 @@ export const projectQueueFactory = ({ folderId: folder.id }, { - offset: 100 + // Get all the secret versions that are not the latest 700 + offset: 700 } ); folderSecretVersionIdsToDelete.push(...deletedSecretVersions.map((el) => el.id)); @@ -173,25 +178,11 @@ export const projectQueueFactory = ({ approvalSecrets.push(...secretApprovals); } - const projectIntegrationAuths = await integrationAuthDAL.find({ - projectId: project.id - }); - const decryptedSecrets = decryptSecrets(secrets, userPrivateKey, oldProjectKey); const decryptedSecretVersions = decryptSecretVersions(secretVersions, userPrivateKey, oldProjectKey); const decryptedApprovalSecrets = decryptSecretApprovals(approvalSecrets, userPrivateKey, oldProjectKey); const decryptedIntegrationAuths = decryptIntegrationAuths(projectIntegrationAuths, userPrivateKey, oldProjectKey); - if (secrets.length !== decryptedSecrets.length) { - throw new Error("Failed to decrypt some secret versions"); - } - if (secretVersions.length !== decryptedSecretVersions.length) { - throw new Error("Failed to decrypt some secret versions"); - } - if (approvalSecrets.length !== decryptedApprovalSecrets.length) { - throw new Error("Failed to decrypt some secret approvals"); - } - // Get the existing bot and the existing project keys for the members of the project const existingBot = await projectBotDAL.findOne({ projectId: project.id }).catch(() => null); const existingProjectKeys = await projectKeyDAL.find({ projectId: project.id }); From 3ea529d525b96897426bc3b0a8eb254a2f69cb4c Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 02:57:04 +0100 Subject: [PATCH 088/157] Update srp.ts --- backend/src/lib/crypto/srp.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/src/lib/crypto/srp.ts b/backend/src/lib/crypto/srp.ts index cdf862609..6f4588e3f 100644 --- a/backend/src/lib/crypto/srp.ts +++ b/backend/src/lib/crypto/srp.ts @@ -33,7 +33,9 @@ export const srpCheckClientProof = async ( return server.checkClientProof(clientProof); }; -// FOR GHOST USER STUFF +// Ghost user related: +// This functionality is intended for ghost user logic. This happens on the frontend when a user is being created. +// We replicate the same functionality on the backend when creating a ghost user. export const generateUserSrpKeys = async (email: string, password: string) => { const pair = nacl.box.keyPair(); const secretKeyUint8Array = pair.secretKey; From ec6ec8813e944bb5ee33391bd62b709c521120cb Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 03:04:23 +0100 Subject: [PATCH 089/157] Moved --- backend/src/lib/crypto/index.ts | 6 ++++++ .../index.ts => crypto/secret-encryption.ts} | 17 ++++++++--------- backend/src/services/project/project-queue.ts | 12 ++++++------ 3 files changed, 20 insertions(+), 15 deletions(-) rename backend/src/lib/{secret/index.ts => crypto/secret-encryption.ts} (93%) diff --git a/backend/src/lib/crypto/index.ts b/backend/src/lib/crypto/index.ts index 0fe05bc01..9d7f4e886 100644 --- a/backend/src/lib/crypto/index.ts +++ b/backend/src/lib/crypto/index.ts @@ -11,4 +11,10 @@ export { encryptSymmetric128BitHexKeyUTF8, generateAsymmetricKeyPair } from "./encryption"; +export { + decryptIntegrationAuths, + decryptSecretApprovals, + decryptSecrets, + decryptSecretVersions +} from "./secret-encryption"; export { generateSrpServerKey, srpCheckClientProof } from "./srp"; diff --git a/backend/src/lib/secret/index.ts b/backend/src/lib/crypto/secret-encryption.ts similarity index 93% rename from backend/src/lib/secret/index.ts rename to backend/src/lib/crypto/secret-encryption.ts index dd23c318a..2e0492560 100644 --- a/backend/src/lib/secret/index.ts +++ b/backend/src/lib/crypto/secret-encryption.ts @@ -11,9 +11,8 @@ import { TSecretApprovalRequestsSecrets, TSecrets, TSecretVersions -} from "@app/db/schemas"; - -import { decryptAsymmetric } from "../crypto"; +} from "../../db/schemas"; +import { decryptAsymmetric } from "./encryption"; const DecryptedValuesSchema = z.object({ id: z.string(), @@ -42,15 +41,15 @@ const DecryptedSecretVersionsSchema = z.object({ original: SecretVersionsSchema }); -export const DecryptedSecretApprovalsSchema = z.object({ +const DecryptedSecretApprovalsSchema = z.object({ decrypted: DecryptedValuesSchema, original: SecretApprovalRequestsSecretsSchema }); -export type DecryptedSecret = z.infer; -export type DecryptedSecretVersions = z.infer; -export type DecryptedSecretApprovals = z.infer; -export type DecryptedIntegrationAuths = z.infer; +type DecryptedSecret = z.infer; +type DecryptedSecretVersions = z.infer; +type DecryptedSecretApprovals = z.infer; +type DecryptedIntegrationAuths = z.infer; type TLatestKey = TProjectKeys & { sender: { @@ -79,7 +78,7 @@ const decryptCipher = ({ }; const getDecryptedValues = (data: Array<{ ciphertext: string; iv: string; tag: string }>, key: string | Buffer) => { - const results = []; + const results: string[] = []; for (const { ciphertext, iv, tag } of data) { if (!ciphertext || !iv || !tag) { diff --git a/backend/src/services/project/project-queue.ts b/backend/src/services/project/project-queue.ts index 8e924af1a..5620a89c4 100644 --- a/backend/src/services/project/project-queue.ts +++ b/backend/src/services/project/project-queue.ts @@ -16,6 +16,12 @@ import { import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal"; import { TSecretApprovalRequestSecretDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-secret-dal"; import { RequestState } from "@app/ee/services/secret-approval-request/secret-approval-request-types"; +import { + decryptIntegrationAuths, + decryptSecretApprovals, + decryptSecrets, + decryptSecretVersions +} from "@app/lib/crypto"; import { decryptAsymmetric, encryptSymmetric128BitHexKeyUTF8, @@ -23,12 +29,6 @@ import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption"; import { logger } from "@app/lib/logger"; -import { - decryptIntegrationAuths, - decryptSecretApprovals, - decryptSecrets, - decryptSecretVersions -} from "@app/lib/secret"; import { QueueJobs, QueueName, TQueueJobTypes, TQueueServiceFactory } from "@app/queue"; import { TIntegrationAuthDALFactory } from "../integration-auth/integration-auth-dal"; From ed353d3263f9de3bdae30ec6f79c2ea4a32efafa Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 04:09:46 +0100 Subject: [PATCH 090/157] Extra --- backend/src/services/org/org-service.ts | 2 +- .../project-membership-service.ts | 22 +++++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/backend/src/services/org/org-service.ts b/backend/src/services/org/org-service.ts index f499d2309..f39ac5366 100644 --- a/backend/src/services/org/org-service.ts +++ b/backend/src/services/org/org-service.ts @@ -132,7 +132,7 @@ export const orgServiceFactory = ({ }; const addGhostUser = async (orgId: string, tx?: Knex) => { - const email = `ghost-${alphaNumericNanoId(16)}-${orgId}@infisical.com`; // We add a nanoid because the email is unique. And we have to create a new ghost user each time, so we can have access to the private key. + const email = `sudo-${alphaNumericNanoId(16)}-${orgId}@infisical.com`; // We add a nanoid because the email is unique. And we have to create a new ghost user each time, so we can have access to the private key. const password = crypto.randomBytes(128).toString("hex"); const user = await userDAL.create( diff --git a/backend/src/services/project-membership/project-membership-service.ts b/backend/src/services/project-membership/project-membership-service.ts index 35deb9b2c..fbaa0bb00 100644 --- a/backend/src/services/project-membership/project-membership-service.ts +++ b/backend/src/services/project-membership/project-membership-service.ts @@ -4,6 +4,7 @@ import { ForbiddenError } from "@casl/ability"; import { OrgMembershipStatus, ProjectMembershipRole, + ProjectVersion, SecretKeyEncoding, TableName, TProjectMemberships, @@ -226,6 +227,10 @@ export const projectMembershipServiceFactory = ({ const project = await projectDAL.findById(projectId); if (!project) throw new BadRequestError({ message: "Project not found" }); + if (project.version === ProjectVersion.V1) { + throw new BadRequestError({ message: "Please upgrade your project on your dashboard" }); + } + const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Create, ProjectPermissionSub.Member); @@ -243,7 +248,7 @@ export const projectMembershipServiceFactory = ({ if (!ghostUser) { throw new BadRequestError({ - message: "Failed to find top-level user" + message: "Failed to find sudo user" }); } @@ -251,7 +256,7 @@ export const projectMembershipServiceFactory = ({ if (!ghostUserLatestKey) { throw new BadRequestError({ - message: "Failed to find top-level latest key" + message: "Failed to find sudo user latest key" }); } @@ -405,6 +410,19 @@ export const projectMembershipServiceFactory = ({ const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId, actorOrgId); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Delete, ProjectPermissionSub.Member); + const project = await projectDAL.findById(projectId); + + if (!project) { + throw new BadRequestError({ + message: "Project not found", + name: "Delete project membership" + }); + } + + if (project.version === ProjectVersion.V1) { + throw new BadRequestError({ message: "Please upgrade your project on your dashboard" }); + } + const projectMembers = await projectMembershipDAL.findMembershipsByEmail(projectId, emails); if (projectMembers.length !== emails.length) { From f178220c5a7a52fdc729eb4a1ab8180bc908cd4f Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 08:43:26 +0100 Subject: [PATCH 091/157] Documentation --- docs/mint.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/mint.json b/docs/mint.json index 6bc728405..ccdead084 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -388,6 +388,8 @@ "pages": [ "api-reference/endpoints/workspaces/create-workspace", "api-reference/endpoints/workspaces/delete-workspace", + "api-reference/endpoints/workspaces/get-workspace", + "api-reference/endpoints/workspaces/update-workspace", "api-reference/endpoints/workspaces/invite-member-to-workspace", "api-reference/endpoints/workspaces/remove-member-from-workspace", "api-reference/endpoints/workspaces/memberships", From 04b4e80dd15b99a920ffb678c12cc8d3ccfecbf6 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 08:44:19 +0100 Subject: [PATCH 092/157] Documentation --- docs/api-reference/endpoints/workspaces/create-workspace.mdx | 2 +- docs/api-reference/endpoints/workspaces/delete-workspace.mdx | 2 +- docs/api-reference/endpoints/workspaces/get-workspace.mdx | 4 ++++ docs/api-reference/endpoints/workspaces/update-workspace.mdx | 4 ++++ 4 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 docs/api-reference/endpoints/workspaces/get-workspace.mdx create mode 100644 docs/api-reference/endpoints/workspaces/update-workspace.mdx diff --git a/docs/api-reference/endpoints/workspaces/create-workspace.mdx b/docs/api-reference/endpoints/workspaces/create-workspace.mdx index d0518df2d..ec3d09b81 100644 --- a/docs/api-reference/endpoints/workspaces/create-workspace.mdx +++ b/docs/api-reference/endpoints/workspaces/create-workspace.mdx @@ -1,4 +1,4 @@ --- -title: "Create workspace" +title: "Create Workspace" openapi: "POST /api/v2/workspace" --- \ No newline at end of file diff --git a/docs/api-reference/endpoints/workspaces/delete-workspace.mdx b/docs/api-reference/endpoints/workspaces/delete-workspace.mdx index 5037ca20d..ca6cb8914 100644 --- a/docs/api-reference/endpoints/workspaces/delete-workspace.mdx +++ b/docs/api-reference/endpoints/workspaces/delete-workspace.mdx @@ -1,5 +1,5 @@ --- -title: "Delete workspace" +title: "Delete Workspace" openapi: "DELETE /api/v1/workspace/{workspaceId}" --- diff --git a/docs/api-reference/endpoints/workspaces/get-workspace.mdx b/docs/api-reference/endpoints/workspaces/get-workspace.mdx new file mode 100644 index 000000000..2d27d9455 --- /dev/null +++ b/docs/api-reference/endpoints/workspaces/get-workspace.mdx @@ -0,0 +1,4 @@ +--- +title: "Get Workspace" +openapi: "GET /api/v1/workspace/{workspaceId}" +--- \ No newline at end of file diff --git a/docs/api-reference/endpoints/workspaces/update-workspace.mdx b/docs/api-reference/endpoints/workspaces/update-workspace.mdx new file mode 100644 index 000000000..81a6ec8ea --- /dev/null +++ b/docs/api-reference/endpoints/workspaces/update-workspace.mdx @@ -0,0 +1,4 @@ +--- +title: "Update Workspace" +openapi: "PATCH /api/v1/workspace/{workspaceId}" +--- \ No newline at end of file From c16bf2afdb30cf08387f9faf028cf8d1b545c142 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 08:45:05 +0100 Subject: [PATCH 093/157] Removed unused invite-signup endpoint (finally) --- .../src/server/routes/v1/project-router.ts | 85 ++++++++----------- frontend/src/hooks/api/workspace/index.tsx | 1 - frontend/src/hooks/api/workspace/queries.tsx | 22 ----- 3 files changed, 37 insertions(+), 71 deletions(-) diff --git a/backend/src/server/routes/v1/project-router.ts b/backend/src/server/routes/v1/project-router.ts index 03101c762..0a5f8d85e 100644 --- a/backend/src/server/routes/v1/project-router.ts +++ b/backend/src/server/routes/v1/project-router.ts @@ -2,13 +2,11 @@ import { z } from "zod"; import { IntegrationsSchema, - ProjectKeysSchema, ProjectMembershipsSchema, ProjectsSchema, UserEncryptionKeysSchema, UsersSchema } from "@app/db/schemas"; -import { EventType } from "@app/ee/services/audit-log/audit-log-types"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; @@ -119,7 +117,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { }) } }, - onRequest: verifyAuth([AuthMode.JWT]), + onRequest: verifyAuth([AuthMode.JWT, AuthMode.SERVICE_TOKEN, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { const workspace = await server.services.project.getAProject({ actorId: req.permission.id, @@ -171,7 +169,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { }) } }, - onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY, AuthMode.IDENTITY_ACCESS_TOKEN]), + onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]), handler: async (req) => { const workspace = await server.services.project.deleteProject({ actorId: req.permission.id, @@ -216,6 +214,41 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { } }); + server.route({ + url: "/:workspaceId", + method: "PATCH", + schema: { + params: z.object({ + workspaceId: z.string().trim() + }), + body: z.object({ + name: z.string().trim().optional(), + autoCapitalization: z.boolean().optional() + }), + response: { + 200: z.object({ + workspace: ProjectsSchema + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const workspace = await server.services.project.updateProject({ + actorId: req.permission.id, + actor: req.permission.type, + actorOrgId: req.permission.orgId, + projectId: req.params.workspaceId, + update: { + name: req.body.name, + autoCapitalization: req.body.autoCapitalization + } + }); + return { + workspace + }; + } + }); + server.route({ url: "/:workspaceId/auto-capitalization", method: "POST", @@ -249,50 +282,6 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { } }); - server.route({ - url: "/:workspaceId/invite-signup", - method: "POST", - schema: { - params: z.object({ - workspaceId: z.string().trim() - }), - body: z.object({ - email: z.string().trim() - }), - response: { - 200: z.object({ - invitee: UsersSchema, - latestKey: ProjectKeysSchema.optional() - }) - } - }, - onRequest: verifyAuth([AuthMode.JWT]), - handler: async (req) => { - const { invitees, latestKey } = await server.services.projectMembership.inviteUserToProject({ - actorId: req.permission.id, - actor: req.permission.type, - actorOrgId: req.permission.orgId, - projectId: req.params.workspaceId, - emails: [req.body.email] - }); - - const invitee = invitees[0]; - - await server.services.auditLog.createAuditLog({ - ...req.auditLogInfo, - projectId: req.params.workspaceId, - event: { - type: EventType.ADD_WORKSPACE_MEMBER, - metadata: { - userId: invitee.id, - email: invitee.email - } - } - }); - return { invitee, latestKey }; - } - }); - server.route({ url: "/:workspaceId/integrations", method: "GET", diff --git a/frontend/src/hooks/api/workspace/index.tsx b/frontend/src/hooks/api/workspace/index.tsx index e66af93a1..b96223077 100644 --- a/frontend/src/hooks/api/workspace/index.tsx +++ b/frontend/src/hooks/api/workspace/index.tsx @@ -1,6 +1,5 @@ export { useAddIdentityToWorkspace, - useAddUserToWorkspace, useCreateWorkspace, useCreateWsEnvironment, useDeleteIdentityFromWorkspace, diff --git a/frontend/src/hooks/api/workspace/queries.tsx b/frontend/src/hooks/api/workspace/queries.tsx index bcee978d3..d970cab54 100644 --- a/frontend/src/hooks/api/workspace/queries.tsx +++ b/frontend/src/hooks/api/workspace/queries.tsx @@ -319,28 +319,6 @@ export const useGetWorkspaceUsers = (workspaceId: string) => { }); }; -export const useAddUserToWorkspace = () => { - const queryClient = useQueryClient(); - - return useMutation({ - mutationFn: async ({ email, workspaceId }: { email: string; workspaceId: string }) => { - const { - data: { invitees, latestKey } - } = await apiRequest.post(`/api/v1/workspace/${workspaceId}/invite-signup`, { - emails: [email] - }); - - return { - invitees, - latestKey - }; - }, - onSuccess: (_, dto) => { - queryClient.invalidateQueries(workspaceKeys.getWorkspaceUsers(dto.workspaceId)); - } - }); -}; - export const useDeleteUserFromWorkspace = () => { const queryClient = useQueryClient(); From 0931a17af5b10bacb2ff05d8b50f580a5c7f8d7a Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 08:45:43 +0100 Subject: [PATCH 094/157] Convert check to a standalone DAL operation --- .../secret-approval-request-service.ts | 10 +--- backend/src/services/project/project-dal.ts | 15 ++++-- .../secret-import/secret-import-service.ts | 10 +--- backend/src/services/secret/secret-service.ts | 50 +++---------------- 4 files changed, 22 insertions(+), 63 deletions(-) diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 9e2bb8477..9a91b745d 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -48,7 +48,7 @@ type TSecretApprovalRequestServiceFactoryDep = { secretBlindIndexDAL: Pick; snapshotService: Pick; secretVersionDAL: Pick; - projectDAL: Pick; + projectDAL: Pick; secretService: Pick< TSecretServiceFactory, | "fnSecretBulkInsert" @@ -437,13 +437,7 @@ export const secretApprovalRequestServiceFactory = ({ subject(ProjectPermissionSub.Secrets, { environment, secretPath }) ); - const isProjectBeingUpgraded = await projectDAL.isProjectBeingUpgraded(projectId); - - if (isProjectBeingUpgraded) { - throw new BadRequestError({ - message: "Project is currently being upgraded, and secrets cannot be written. Please try again" - }); - } + await projectDAL.checkProjectUpgradeStatus(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "GenSecretApproval" }); diff --git a/backend/src/services/project/project-dal.ts b/backend/src/services/project/project-dal.ts index ecd60dc41..7d0826e12 100644 --- a/backend/src/services/project/project-dal.ts +++ b/backend/src/services/project/project-dal.ts @@ -2,7 +2,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; import { ProjectsSchema, ProjectUpgradeStatus, ProjectVersion, TableName, TProjectsUpdate } from "@app/db/schemas"; -import { DatabaseError } from "@app/lib/errors"; +import { BadRequestError, DatabaseError } from "@app/lib/errors"; import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; export type TProjectDALFactory = ReturnType; @@ -160,9 +160,16 @@ export const projectDALFactory = (db: TDbClient) => { } }; - const isProjectBeingUpgraded = async (projectId: string) => { + const checkProjectUpgradeStatus = async (projectId: string) => { const project = await projectOrm.findById(projectId); - return project.upgradeStatus === ProjectUpgradeStatus.InProgress && project.version === ProjectVersion.V1; + const upgradeInProgress = + project.upgradeStatus === ProjectUpgradeStatus.InProgress && project.version === ProjectVersion.V1; + + if (upgradeInProgress) { + throw new BadRequestError({ + message: "Project is currently being upgraded, and secrets cannot be written. Please try again" + }); + } }; return { @@ -172,6 +179,6 @@ export const projectDALFactory = (db: TDbClient) => { findAllProjectsByIdentity, findProjectGhostUser, findProjectById, - isProjectBeingUpgraded + checkProjectUpgradeStatus }; }; diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index 33d89f014..1beae9be6 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -22,7 +22,7 @@ type TSecretImportServiceFactoryDep = { secretImportDAL: TSecretImportDALFactory; folderDAL: TSecretFolderDALFactory; secretDAL: Pick; - projectDAL: Pick; + projectDAL: Pick; projectEnvDAL: TProjectEnvDALFactory; permissionService: Pick; }; @@ -65,13 +65,7 @@ export const secretImportServiceFactory = ({ }) ); - const isProjectBeingUpgraded = await projectDAL.isProjectBeingUpgraded(projectId); - - if (isProjectBeingUpgraded) { - throw new BadRequestError({ - message: "Project is currently being upgraded, and secrets cannot be written. Please try again" - }); - } + await projectDAL.checkProjectUpgradeStatus(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create import" }); diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index e8af1ca66..96d851a63 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -50,7 +50,7 @@ type TSecretServiceFactoryDep = { secretTagDAL: TSecretTagDALFactory; secretVersionDAL: TSecretVersionDALFactory; folderDAL: Pick; - projectDAL: Pick; + projectDAL: Pick; secretBlindIndexDAL: TSecretBlindIndexDALFactory; permissionService: Pick; snapshotService: Pick; @@ -284,13 +284,7 @@ export const secretServiceFactory = ({ subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) ); - const isProjectBeingUpgraded = await projectDAL.isProjectBeingUpgraded(projectId); - - if (isProjectBeingUpgraded) { - throw new BadRequestError({ - message: "Project is currently being upgraded, and secrets cannot be written. Please try again" - }); - } + await projectDAL.checkProjectUpgradeStatus(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); @@ -370,13 +364,7 @@ export const secretServiceFactory = ({ subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) ); - const isProjectBeingUpgraded = await projectDAL.isProjectBeingUpgraded(projectId); - - if (isProjectBeingUpgraded) { - throw new BadRequestError({ - message: "Project is currently being upgraded, and secrets cannot be written. Please try again" - }); - } + await projectDAL.checkProjectUpgradeStatus(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); @@ -478,13 +466,7 @@ export const secretServiceFactory = ({ subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) ); - const isProjectBeingUpgraded = await projectDAL.isProjectBeingUpgraded(projectId); - - if (isProjectBeingUpgraded) { - throw new BadRequestError({ - message: "Project is currently being upgraded, and secrets cannot be written. Please try again" - }); - } + await projectDAL.checkProjectUpgradeStatus(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); @@ -677,13 +659,7 @@ export const secretServiceFactory = ({ subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) ); - const isProjectBeingUpgraded = await projectDAL.isProjectBeingUpgraded(projectId); - - if (isProjectBeingUpgraded) { - throw new BadRequestError({ - message: "Project is currently being upgraded, and secrets cannot be written. Please try again" - }); - } + await projectDAL.checkProjectUpgradeStatus(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); @@ -740,13 +716,7 @@ export const secretServiceFactory = ({ subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) ); - const isProjectBeingUpgraded = await projectDAL.isProjectBeingUpgraded(projectId); - - if (isProjectBeingUpgraded) { - throw new BadRequestError({ - message: "Project is currently being upgraded, and secrets cannot be written. Please try again" - }); - } + await projectDAL.checkProjectUpgradeStatus(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); @@ -819,13 +789,7 @@ export const secretServiceFactory = ({ subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) ); - const isProjectBeingUpgraded = await projectDAL.isProjectBeingUpgraded(projectId); - - if (isProjectBeingUpgraded) { - throw new BadRequestError({ - message: "Project is currently being upgraded, and secrets cannot be written. Please try again" - }); - } + await projectDAL.checkProjectUpgradeStatus(projectId); const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); From a0e25b8ea2cdac893491490da24ee9e2ea9b2061 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 08:45:58 +0100 Subject: [PATCH 095/157] Describe --- backend/src/server/routes/v2/project-membership-router.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/server/routes/v2/project-membership-router.ts b/backend/src/server/routes/v2/project-membership-router.ts index 94a3423a6..417fcb44c 100644 --- a/backend/src/server/routes/v2/project-membership-router.ts +++ b/backend/src/server/routes/v2/project-membership-router.ts @@ -11,10 +11,10 @@ export const registerProjectMembershipRouter = async (server: FastifyZodProvider url: "/:projectId/memberships", schema: { params: z.object({ - projectId: z.string() + projectId: z.string().describe("The ID of the project.") }), body: z.object({ - emails: z.string().email().array() + emails: z.string().email().array().describe("Emails of the users to add to the project.") }), response: { 200: z.object({ @@ -53,11 +53,11 @@ export const registerProjectMembershipRouter = async (server: FastifyZodProvider url: "/:projectId/memberships", schema: { params: z.object({ - projectId: z.string() + projectId: z.string().describe("The ID of the project.") }), body: z.object({ - emails: z.string().email().array() + emails: z.string().email().array().describe("Emails of the users to remove from the project.") }), response: { 200: z.object({ From ae3606c9fb8d821000cdc190dcd62ec685819b0c Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 08:46:59 +0100 Subject: [PATCH 096/157] Optional slug on create project --- backend/src/server/routes/v2/project-router.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/backend/src/server/routes/v2/project-router.ts b/backend/src/server/routes/v2/project-router.ts index 20fff74da..f9919a974 100644 --- a/backend/src/server/routes/v2/project-router.ts +++ b/backend/src/server/routes/v2/project-router.ts @@ -1,3 +1,4 @@ +import slugify from "@sindresorhus/slugify"; import { z } from "zod"; import { ProjectKeysSchema, ProjectsSchema } from "@app/db/schemas"; @@ -125,6 +126,14 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { schema: { body: z.object({ projectName: z.string().trim(), + slug: z + .string() + .min(5) + .max(36) + .refine((v) => slugify(v) === v, { + message: "Slug must be a valid slug" + }) + .optional(), organizationId: z.string().trim() }), response: { @@ -139,7 +148,8 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { actorId: req.permission.id, actor: req.permission.type, orgId: req.body.organizationId, - workspaceName: req.body.projectName + workspaceName: req.body.projectName, + slug: req.body.slug }); return { project }; From 9c11226b7175ba25b0dfa4d03544e68529d8ef73 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 08:48:05 +0100 Subject: [PATCH 097/157] Renaming --- .../project-membership/project-membership-service.ts | 4 ++-- backend/src/services/project/project-fns.ts | 6 +++--- backend/src/services/project/project-queue.ts | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/src/services/project-membership/project-membership-service.ts b/backend/src/services/project-membership/project-membership-service.ts index fbaa0bb00..e96247c97 100644 --- a/backend/src/services/project-membership/project-membership-service.ts +++ b/backend/src/services/project-membership/project-membership-service.ts @@ -21,7 +21,7 @@ import { groupBy } from "@app/lib/fn"; import { ActorType } from "../auth/auth-type"; import { TOrgDALFactory } from "../org/org-dal"; import { TProjectDALFactory } from "../project/project-dal"; -import { createWsMembers } from "../project/project-fns"; +import { assignWorkspaceKeysToMembers } from "../project/project-fns"; import { TProjectBotDALFactory } from "../project-bot/project-bot-dal"; import { TProjectKeyDALFactory } from "../project-key/project-key-dal"; import { TProjectRoleDALFactory } from "../project-role/project-role-dal"; @@ -275,7 +275,7 @@ export const projectMembershipServiceFactory = ({ ciphertext: bot.encryptedPrivateKey }); - const newWsMembers = createWsMembers({ + const newWsMembers = assignWorkspaceKeysToMembers({ decryptKey: ghostUserLatestKey, userPrivateKey: botPrivateKey, members: orgMembers.map((membership) => ({ diff --git a/backend/src/services/project/project-fns.ts b/backend/src/services/project/project-fns.ts index eb6fe3b11..0f1f5a08d 100644 --- a/backend/src/services/project/project-fns.ts +++ b/backend/src/services/project/project-fns.ts @@ -4,8 +4,8 @@ import { decryptAsymmetric, encryptAsymmetric } from "@app/lib/crypto"; import { AddUserToWsDTO } from "./project-types"; -export const createWsMembers = ({ members, decryptKey, userPrivateKey }: AddUserToWsDTO) => { - const key = decryptAsymmetric({ +export const assignWorkspaceKeysToMembers = ({ members, decryptKey, userPrivateKey }: AddUserToWsDTO) => { + const plaintextProjectKey = decryptAsymmetric({ ciphertext: decryptKey.encryptedKey, nonce: decryptKey.nonce, publicKey: decryptKey.sender.publicKey, @@ -14,7 +14,7 @@ export const createWsMembers = ({ members, decryptKey, userPrivateKey }: AddUser const newWsMembers = members.map(({ orgMembershipId, userPublicKey, projectMembershipRole }) => { const { ciphertext: inviteeCipherText, nonce: inviteeNonce } = encryptAsymmetric( - key, + plaintextProjectKey, userPublicKey, userPrivateKey ); diff --git a/backend/src/services/project/project-queue.ts b/backend/src/services/project/project-queue.ts index 5620a89c4..7bf9f0e51 100644 --- a/backend/src/services/project/project-queue.ts +++ b/backend/src/services/project/project-queue.ts @@ -43,7 +43,7 @@ import { TSecretVersionDALFactory } from "../secret/secret-version-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TUserDALFactory } from "../user/user-dal"; import { TProjectDALFactory } from "./project-dal"; -import { createProjectKey, createWsMembers } from "./project-fns"; +import { assignWorkspaceKeysToMembers, createProjectKey } from "./project-fns"; export type TProjectQueueFactory = ReturnType; @@ -260,7 +260,7 @@ export const projectQueueFactory = ({ throw new Error(`User with ID ${key.receiverId} was not found during upgrade, or user is not in org.`); } - const [newMember] = createWsMembers({ + const [newMember] = assignWorkspaceKeysToMembers({ decryptKey: ghostUserLatestKey, userPrivateKey: ghostUser.keys.plainPrivateKey, members: [ From 0b7efa57be371e48ebf50f2083eb3f606ce2a558 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 08:48:19 +0100 Subject: [PATCH 098/157] Proper update project endpoint --- .../src/services/project/project-service.ts | 28 +++++++++++++++---- backend/src/services/project/project-types.ts | 8 ++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index 6b02d05d7..5826e77f0 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -26,9 +26,15 @@ import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind- import { ROOT_FOLDER_NAME, TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TUserDALFactory } from "../user/user-dal"; import { TProjectDALFactory } from "./project-dal"; -import { createProjectKey, createWsMembers } from "./project-fns"; +import { assignWorkspaceKeysToMembers, createProjectKey } from "./project-fns"; import { TProjectQueueFactory } from "./project-queue"; -import { TCreateProjectDTO, TDeleteProjectDTO, TGetProjectDTO, TUpgradeProjectDTO } from "./project-types"; +import { + TCreateProjectDTO, + TDeleteProjectDTO, + TGetProjectDTO, + TUpdateProjectDTO, + TUpgradeProjectDTO +} from "./project-types"; export const DEFAULT_PROJECT_ENVS = [ { name: "Development", slug: "dev" }, @@ -74,7 +80,7 @@ export const projectServiceFactory = ({ /* * Create workspace. Make user the admin * */ - const createProject = async ({ orgId, actor, actorId, actorOrgId, workspaceName }: TCreateProjectDTO) => { + const createProject = async ({ orgId, actor, actorId, actorOrgId, workspaceName, slug }: TCreateProjectDTO) => { const { permission, membership: orgMembership } = await permissionService.getOrgPermission( actor, actorId, @@ -102,7 +108,7 @@ export const projectServiceFactory = ({ { name: workspaceName, orgId, - slug: slugify(`${workspaceName}-${alphaNumericNanoId(4)}`), + slug: slug || slugify(`${workspaceName}-${alphaNumericNanoId(4)}`), version: ProjectVersion.V2 }, tx @@ -194,7 +200,7 @@ export const projectServiceFactory = ({ throw new Error("User not found"); } - const [projectAdmin] = createWsMembers({ + const [projectAdmin] = assignWorkspaceKeysToMembers({ decryptKey: latestKey, userPrivateKey: ghostUser.keys.plainPrivateKey, members: [ @@ -305,6 +311,17 @@ export const projectServiceFactory = ({ return projectDAL.findProjectById(projectId); }; + const updateProject = async ({ projectId, actor, actorId, actorOrgId, update }: TUpdateProjectDTO) => { + const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId, actorOrgId); + ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Edit, ProjectPermissionSub.Settings); + + const updatedProject = await projectDAL.updateById(projectId, { + name: update.name, + autoCapitalization: update.autoCapitalization + }); + return updatedProject; + }; + const toggleAutoCapitalization = async ({ projectId, actor, @@ -371,6 +388,7 @@ export const projectServiceFactory = ({ createProject, deleteProject, getProjects, + updateProject, findProjectGhostUser, getProjectUpgradeStatus, getAProject, diff --git a/backend/src/services/project/project-types.ts b/backend/src/services/project/project-types.ts index e6468e528..3843450c2 100644 --- a/backend/src/services/project/project-types.ts +++ b/backend/src/services/project/project-types.ts @@ -9,6 +9,7 @@ export type TCreateProjectDTO = { actorOrgId?: string; orgId: string; workspaceName: string; + slug?: string; }; export type TDeleteProjectDTO = { @@ -25,6 +26,13 @@ export type TGetProjectDTO = { projectId: string; }; +export type TUpdateProjectDTO = { + update: { + name?: string; + autoCapitalization?: boolean; + }; +} & TProjectPermission; + export type TUpgradeProjectDTO = { userPrivateKey: string; } & TProjectPermission; From 82fe0bb5c401a3f08f1f9676cd776095f8ce9678 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 08:48:30 +0100 Subject: [PATCH 099/157] Update project-bot-service.ts --- backend/src/services/project-bot/project-bot-service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/services/project-bot/project-bot-service.ts b/backend/src/services/project-bot/project-bot-service.ts index 57170ad81..456a1e7a0 100644 --- a/backend/src/services/project-bot/project-bot-service.ts +++ b/backend/src/services/project-bot/project-bot-service.ts @@ -120,7 +120,7 @@ export const projectBotServiceFactory = ({ } if (project.version === ProjectVersion.V2) { - throw new BadRequestError({ message: "Failed to set bot active, project has a default bot enabled" }); + throw new BadRequestError({ message: "Failed to set bot active for upgraded project. Bot is already active" }); } if (isActive) { From cee8ead78a9417321ed3751b80db2d90abf0948a Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 08:51:46 +0100 Subject: [PATCH 100/157] Update org-dal.ts --- backend/src/services/org/org-dal.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/services/org/org-dal.ts b/backend/src/services/org/org-dal.ts index 4ff5e97a7..05c77e746 100644 --- a/backend/src/services/org/org-dal.ts +++ b/backend/src/services/org/org-dal.ts @@ -110,6 +110,7 @@ export const orgDALFactory = (db: TDbClient) => { db.ref("id").withSchema(TableName.Users).as("userId"), db.ref("publicKey").withSchema(TableName.UserEncryptionKey) ) + .where({ isGhost: false }) .whereIn("email", emails); return members.map(({ email, firstName, lastName, userId, publicKey, ...data }) => ({ ...data, From c20f6e51ae3c6cf188fe7402398117ec6d636e5f Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 08:54:11 +0100 Subject: [PATCH 101/157] Update project-router.ts --- backend/src/server/routes/v2/project-router.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/server/routes/v2/project-router.ts b/backend/src/server/routes/v2/project-router.ts index f9919a974..7cbcff37c 100644 --- a/backend/src/server/routes/v2/project-router.ts +++ b/backend/src/server/routes/v2/project-router.ts @@ -17,7 +17,7 @@ const projectWithEnv = ProjectsSchema.merge( export const registerProjectRouter = async (server: FastifyZodProvider) => { /* Get project key */ server.route({ - url: "/:projectId/encrypted-key", + url: "/:workspaceId/encrypted-key", method: "GET", schema: { description: "Return encrypted project key", @@ -27,7 +27,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { } ], params: z.object({ - projectId: z.string().trim() + workspaceId: z.string().trim() }), response: { 200: ProjectKeysSchema.merge( @@ -45,12 +45,12 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { actor: req.permission.type, actorId: req.permission.id, actorOrgId: req.permission.orgId, - projectId: req.params.projectId + projectId: req.params.workspaceId }); await server.services.auditLog.createAuditLog({ ...req.auditLogInfo, - projectId: req.params.projectId, + projectId: req.params.workspaceId, event: { type: EventType.GET_WORKSPACE_KEY, metadata: { From 65e3f0ec95689ea27a46b1132f223511fdee35f4 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 08:58:22 +0100 Subject: [PATCH 102/157] Update 3-project.ts --- backend/src/db/seeds/3-project.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/backend/src/db/seeds/3-project.ts b/backend/src/db/seeds/3-project.ts index 8b1c2d958..0efb6a249 100644 --- a/backend/src/db/seeds/3-project.ts +++ b/backend/src/db/seeds/3-project.ts @@ -2,9 +2,7 @@ import crypto from "node:crypto"; import { Knex } from "knex"; -import { getConfig, initEnvConfig } from "@app/lib/config/env"; -import { createSecretBlindIndex, encryptAsymmetric, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; -import { initLogger } from "@app/lib/logger"; +import { encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; import { OrgMembershipRole, SecretEncryptionAlgo, SecretKeyEncoding, TableName } from "../schemas"; import { buildUserProjectKey, getUserPrivateKey, seedData1 } from "../seed-data"; From 0fa0e4eb0fb3273d9d67d70ded5068a2386cfd46 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:37:58 +0400 Subject: [PATCH 103/157] Ghost user migration --- .../migrations/20240202093209_ghost_user.ts | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 backend/src/db/migrations/20240202093209_ghost_user.ts diff --git a/backend/src/db/migrations/20240202093209_ghost_user.ts b/backend/src/db/migrations/20240202093209_ghost_user.ts new file mode 100644 index 000000000..7fa23d245 --- /dev/null +++ b/backend/src/db/migrations/20240202093209_ghost_user.ts @@ -0,0 +1,31 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasColumn(TableName.Users, "ghost"))) { + await knex.schema.alterTable(TableName.Users, (t) => { + t.boolean("ghost").defaultTo(false).notNullable(); + }); + } + + if (!(await knex.schema.hasColumn(TableName.Project, "e2ee"))) { + await knex.schema.alterTable(TableName.Project, (t) => { + t.boolean("e2ee").defaultTo(true).notNullable(); + }); + } +} + +export async function down(knex: Knex): Promise { + if (await knex.schema.hasColumn(TableName.Users, "ghost")) { + await knex.schema.alterTable(TableName.Users, (t) => { + t.dropColumn("ghost"); + }); + } + + if (await knex.schema.hasColumn(TableName.Project, "e2ee")) { + await knex.schema.alterTable(TableName.Project, (t) => { + t.dropColumn("e2ee"); + }); + } +} From 5b550a97a1979d837527597417b530cd38188300 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:39:21 +0400 Subject: [PATCH 104/157] TS erros --- backend/src/db/seeds/1-user.ts | 4 ++++ backend/src/db/seeds/2-org.ts | 3 +++ 2 files changed, 7 insertions(+) diff --git a/backend/src/db/seeds/1-user.ts b/backend/src/db/seeds/1-user.ts index 8e2b00b90..af40dbbbc 100644 --- a/backend/src/db/seeds/1-user.ts +++ b/backend/src/db/seeds/1-user.ts @@ -1,3 +1,6 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +// @ts-nocheck + import { Knex } from "knex"; import { AuthMethod } from "../../services/auth/auth-type"; @@ -19,6 +22,7 @@ export async function seed(knex: Knex): Promise { id: seedData1.id, email: seedData1.email, superAdmin: true, + ghost: false, firstName: "test", lastName: "", authMethods: [AuthMethod.EMAIL], diff --git a/backend/src/db/seeds/2-org.ts b/backend/src/db/seeds/2-org.ts index ba2f65a36..a2990054b 100644 --- a/backend/src/db/seeds/2-org.ts +++ b/backend/src/db/seeds/2-org.ts @@ -1,3 +1,6 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +// @ts-nocheck + import { Knex } from "knex"; import { OrgMembershipRole, OrgMembershipStatus, TableName } from "../schemas"; From 2a4bda481d87f84ded17ac0108bace4405c64a08 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:39:55 +0400 Subject: [PATCH 105/157] Crypto stuff --- backend/src/lib/crypto/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/lib/crypto/index.ts b/backend/src/lib/crypto/index.ts index 9d7f4e886..a92508b1d 100644 --- a/backend/src/lib/crypto/index.ts +++ b/backend/src/lib/crypto/index.ts @@ -18,3 +18,4 @@ export { decryptSecretVersions } from "./secret-encryption"; export { generateSrpServerKey, srpCheckClientProof } from "./srp"; +export { decodeBase64, encodeBase64 } from "tweetnacl-util"; From 423ad494903aa16f649ec48922fa279b2f6984a1 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:40:28 +0400 Subject: [PATCH 106/157] Helper functions for adding workspace members on serverside --- backend/src/lib/project/index.ts | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 backend/src/lib/project/index.ts diff --git a/backend/src/lib/project/index.ts b/backend/src/lib/project/index.ts new file mode 100644 index 000000000..20a63cbd5 --- /dev/null +++ b/backend/src/lib/project/index.ts @@ -0,0 +1,39 @@ +import { ProjectMembershipRole, TProjectKeys } from "@app/db/schemas"; + +import { decryptAsymmetric, encryptAsymmetric } from "../crypto"; + +type AddUserToWsDTO = { + decryptKey: TProjectKeys & { sender: { publicKey: string } }; + userPrivateKey: string; + members: { + orgMembershipId: string; + projectMembershipRole: ProjectMembershipRole; + userPublicKey: string; + }[]; +}; + +export const createWsMembers = async ({ members, decryptKey, userPrivateKey }: AddUserToWsDTO) => { + const key = decryptAsymmetric({ + ciphertext: decryptKey.encryptedKey, + nonce: decryptKey.nonce, + publicKey: decryptKey.sender.publicKey, + privateKey: userPrivateKey + }); + + const newWsMembers = members.map(({ orgMembershipId, userPublicKey, projectMembershipRole }) => { + const { ciphertext: inviteeCipherText, nonce: inviteeNonce } = encryptAsymmetric( + key, + userPublicKey, + userPrivateKey + ); + + return { + orgMembershipId, + projectRole: projectMembershipRole, + workspaceEncryptedKey: inviteeCipherText, + workspaceEncryptedNonce: inviteeNonce + }; + }); + + return newWsMembers; +}; From a366dbb16d96c5b4e61eb3233babdb2f43b181f1 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:40:47 +0400 Subject: [PATCH 107/157] Ghost user! --- backend/src/server/routes/v3/index.ts | 2 + .../src/server/routes/v3/project-router.ts | 126 ++++++++++++++++++ .../project-key/project-key-service.ts | 3 +- .../src/pages/org/[id]/overview/index.tsx | 19 +++ 4 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 backend/src/server/routes/v3/project-router.ts diff --git a/backend/src/server/routes/v3/index.ts b/backend/src/server/routes/v3/index.ts index 6a061a635..21d57d359 100644 --- a/backend/src/server/routes/v3/index.ts +++ b/backend/src/server/routes/v3/index.ts @@ -1,4 +1,5 @@ import { registerLoginRouter } from "./login-router"; +import { registerProjectRouter } from "./project-router"; import { registerSecretBlindIndexRouter } from "./secret-blind-index-router"; import { registerSecretRouter } from "./secret-router"; import { registerSignupRouter } from "./signup-router"; @@ -10,4 +11,5 @@ export const registerV3Routes = async (server: FastifyZodProvider) => { await server.register(registerUserRouter, { prefix: "/users" }); await server.register(registerSecretRouter, { prefix: "/secrets" }); await server.register(registerSecretBlindIndexRouter, { prefix: "/workspaces" }); + await server.register(registerProjectRouter, { prefix: "/projects" }); }; diff --git a/backend/src/server/routes/v3/project-router.ts b/backend/src/server/routes/v3/project-router.ts new file mode 100644 index 000000000..5f83167a7 --- /dev/null +++ b/backend/src/server/routes/v3/project-router.ts @@ -0,0 +1,126 @@ +import crypto from "crypto"; +import { z } from "zod"; + +import { ProjectMembershipRole, ProjectsSchema } from "@app/db/schemas"; +import { encryptAsymmetric } from "@app/lib/crypto"; +import { createWsMembers } from "@app/lib/project"; +import { authRateLimit } from "@app/server/config/rateLimiter"; + +const projectWithEnv = ProjectsSchema.merge( + z.object({ + _id: z.string(), + environments: z.object({ name: z.string(), slug: z.string(), id: z.string() }).array() + }) +); + +export const registerProjectRouter = async (server: FastifyZodProvider) => { + /* Create new project */ + server.route({ + method: "POST", + url: "/", + config: { + rateLimit: authRateLimit + }, + schema: { + body: z.object({ + projectName: z.string().trim(), + inviteAllOrgMembers: z.boolean(), + organizationId: z.string().trim() + }), + response: { + 200: z.object({ + workspace: projectWithEnv + }) + } + }, + handler: async (req) => { + // 1. create the ghost user and add it to the org as admin + const ghost = await server.services.org.addGhostUser(req.body.organizationId); + + // 2. create the workspace + const workspace = await server.services.project.createProject({ + actorId: ghost.user.id, + actor: req.permission.type, + orgId: req.body.organizationId, + workspaceName: req.body.projectName + }); + + // 3. create a random key that we'll use as the project key + const randomBytes = crypto.randomBytes(16).toString("hex"); + + const ghostPrivateKey = ghost.keys.plainPrivateKey; + + const { ciphertext: encryptedProjectKey, nonce: encryptedProjectKeyIv } = encryptAsymmetric( + randomBytes, + ghost.keys.publicKey, + ghostPrivateKey + ); + + // 3. create workspace keys for the ghost user + await server.services.projectKey.uploadProjectKeys({ + projectId: workspace.id, + actor: req.permission.type, + actorId: ghost.user.id, + nonce: encryptedProjectKeyIv, + receiverId: ghost.user.id, + encryptedKey: encryptedProjectKey + }); + + // 4. create a project bot + const bot = await server.services.projectBot.findBotByProjectId({ + actorId: ghost.user.id, + actor: req.permission.type, + projectId: workspace.id + }); + + // 5. activate the bot + await server.services.projectBot.setBotActiveState({ + botKey: { + encryptedKey: encryptedProjectKey, + nonce: encryptedProjectKeyIv + }, + actorId: ghost.user.id, + isActive: true, + actor: req.permission.type, + botId: bot.id + }); + + // 6. get the current user & org membership + const user = await server.services.user.getMe(req.permission.id); + const userOrgMembership = await server.services.permission.getUserOrgPermission(user.id, req.body.organizationId); + + // 7. Get the latest key from the ghost! + const latestKey = await server.services.projectKey.getLatestProjectKey({ + actorId: ghost.user.id, + actor: req.permission.type, + projectId: workspace.id + }); + + if (!latestKey) throw new Error("Failed to get latest key"); + + // 8. Create workspace members for the current user + + const projectAdmin = await createWsMembers({ + decryptKey: latestKey, + members: [ + { + userPublicKey: user.publicKey, + orgMembershipId: userOrgMembership.membership.id, + projectMembershipRole: ProjectMembershipRole.Admin // <-- Make the first user an admin + } + ], + userPrivateKey: ghostPrivateKey + }); + + // 9. Add the current user to the workspace + await server.services.projectMembership.addUsersToProject({ + projectId: workspace.id, + actorId: ghost.user.id, + actor: req.permission.type, + members: projectAdmin + }); + + return { workspace }; + } + }); +}; diff --git a/backend/src/services/project-key/project-key-service.ts b/backend/src/services/project-key/project-key-service.ts index fa77760a4..81b589bd0 100644 --- a/backend/src/services/project-key/project-key-service.ts +++ b/backend/src/services/project-key/project-key-service.ts @@ -1,4 +1,5 @@ import { ForbiddenError } from "@casl/ability"; +import { Knex } from "knex"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; @@ -43,7 +44,7 @@ export const projectKeyServiceFactory = ({ name: "Upload project keys" }); - await projectKeyDAL.create({ projectId, receiverId, encryptedKey, nonce, senderId: actorId }); + await projectKeyDAL.create({ projectId, receiverId, encryptedKey, nonce, senderId: actorId }, tx); }; const getLatestProjectKey = async ({ actorId, projectId, actor, actorOrgId }: TGetLatestProjectKeyDTO) => { diff --git a/frontend/src/pages/org/[id]/overview/index.tsx b/frontend/src/pages/org/[id]/overview/index.tsx index 8a703b3dc..f3b6f67b0 100644 --- a/frontend/src/pages/org/[id]/overview/index.tsx +++ b/frontend/src/pages/org/[id]/overview/index.tsx @@ -509,6 +509,7 @@ const OrganizationPage = withPermission( projectName: name }); + /* if (addMembers) { const orgUsers = await fetchOrgUsers(currentOrg); @@ -523,6 +524,24 @@ const OrganizationPage = withPermission( // eslint-disable-next-line no-promise-executor-return -- We do this because the function returns too fast, which sometimes causes an error when the user is redirected. await new Promise((resolve) => setTimeout(resolve, 2_000)); + const decryptKey = await fetchUserWsKey(newWorkspaceId); + + await addWsUser.mutateAsync({ + workspaceId: newWorkspaceId, + decryptKey, + userPrivateKey: PRIVATE_KEY, + members: orgUsers + .filter( + ({ status, user: orgUser }) => status === "accepted" && user.email !== orgUser.email + ) + .map(({ user: orgUser, id: orgMembershipId }) => ({ + userPublicKey: orgUser.publicKey, + orgMembershipId + })) + }); + } + */ + createNotification({ text: "Workspace created", type: "success" }); handlePopUpClose("addNewWs"); createNotification({ text: "Workspace created", type: "success" }); router.push(`/project/${newProjectId}/secrets/overview`); From 4b37b2afbac737b37b2d97bd1dca633d452c72df Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 6 Feb 2024 03:17:33 +0400 Subject: [PATCH 108/157] Wired most of the frontend to support ghost users --- frontend/src/layouts/AppLayout/AppLayout.tsx | 72 ++++++++++--------- .../src/pages/org/[id]/overview/index.tsx | 18 ----- .../MemberListTab/MemberListTab.tsx | 2 + 3 files changed, 39 insertions(+), 53 deletions(-) diff --git a/frontend/src/layouts/AppLayout/AppLayout.tsx b/frontend/src/layouts/AppLayout/AppLayout.tsx index 84dc0f480..357e7c770 100644 --- a/frontend/src/layouts/AppLayout/AppLayout.tsx +++ b/frontend/src/layouts/AppLayout/AppLayout.tsx @@ -238,7 +238,7 @@ export const AppLayout = ({ children }: LayoutProps) => { projectId: newProjectId }); } - + // eslint-disable-next-line no-promise-executor-return -- We do this because the function returns too fast, which sometimes causes an error when the user is redirected. await new Promise((resolve) => setTimeout(resolve, 2_000)); @@ -250,7 +250,7 @@ export const AppLayout = ({ children }: LayoutProps) => { createNotification({ text: "Failed to create workspace", type: "error" }); } }; - + return ( <>
@@ -288,41 +288,43 @@ export const AppLayout = ({ children }: LayoutProps) => {
{user?.email}
- {orgs?.map((org) => { + {orgs?.map((org) => { return ( - - - - ) + > +
+ {org.name} +
+ + + ); })} {/* -
+ +
); }; diff --git a/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx b/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx index 58c34d627..d31eb3b6e 100644 --- a/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx +++ b/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useRef, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import Link from "next/link"; import { useRouter } from "next/router"; @@ -14,8 +14,6 @@ import { useNotificationContext } from "@app/components/context/Notifications/No import NavHeader from "@app/components/navigation/NavHeader"; import { PermissionDeniedBanner } from "@app/components/permissions"; import { - Alert, - AlertDescription, Button, EmptyState, IconButton, @@ -40,8 +38,7 @@ import { useGetFoldersByEnv, useGetProjectSecretsAllEnv, useGetUserWsKey, - useUpdateSecretV3, - useUpgradeProject + useUpdateSecretV3 } from "@app/hooks/api"; import { ProjectVersion } from "@app/hooks/api/workspace/types"; @@ -109,7 +106,6 @@ export const SecretOverviewPage = () => { environments: userAvailableEnvs.map(({ slug }) => slug) }); - const upgradeProject = useUpgradeProject(); const { mutateAsync: createSecretV3 } = useCreateSecretV3(); const { mutateAsync: updateSecretV3 } = useUpdateSecretV3(); const { mutateAsync: deleteSecretV3 } = useDeleteSecretV3(); @@ -203,24 +199,6 @@ export const SecretOverviewPage = () => { } }; - const onUpgradeProject = useCallback(async () => { - const PRIVATE_KEY = localStorage.getItem("PRIVATE_KEY"); - - if (!PRIVATE_KEY) { - createNotification({ - type: "error", - text: "Private key not found" - }); - - return; - } - - await upgradeProject.mutateAsync({ - projectId: workspaceId, - privateKey: PRIVATE_KEY - }); - }, []); - const handleResetSearch = () => setSearchFilter(""); const handleFolderClick = (path: string) => { From 9efece1f01df0155a49832be06dcb31fab0cf24b Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 01:04:32 +0100 Subject: [PATCH 120/157] Update models.ts --- backend/src/db/schemas/models.ts | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 2b838b1bf..a405ba004 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -123,21 +123,9 @@ export enum ProjectUpgradeStatus { Failed = "FAILED" } -// Case study: -/* - -Q: If the upgrade status is null and the project version is V1: -A: It would mean that the project has not been attempted to be upgraded - -Q: If the upgrade status is not null (FAILED or IN_PROGRESS), we a status - -Q: if the project status is null, and the project version is v2, we should display nothing cuz its upgraded! - -*/ - export enum ProjectUpgradeStatus { InProgress = "IN_PROGRESS", - Completed = "Project upgrade completed.", // Will be null if completed. So a completed status is not needed + // Completed = "Project upgrade completed.", // Will be null if completed. So a completed status is not needed Failed = "FAILED" } From 773756d731850f7af79f1d618ee2c7a6d43b6a12 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 14:16:12 +0100 Subject: [PATCH 121/157] Add transaction support --- backend/src/services/project/project-dal.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/services/project/project-dal.ts b/backend/src/services/project/project-dal.ts index f18542216..8b2971595 100644 --- a/backend/src/services/project/project-dal.ts +++ b/backend/src/services/project/project-dal.ts @@ -80,13 +80,13 @@ export const projectDALFactory = (db: TDbClient) => { } }; - const setProjectUpgradeStatus = async (projectId: string, status: ProjectUpgradeStatus) => { + const setProjectUpgradeStatus = async (projectId: string, status: ProjectUpgradeStatus | null, tx?: Knex) => { try { const data: TProjectsUpdate = { upgradeStatus: status } as const; - await db(TableName.Project).where({ id: projectId }).update(data); + await (tx || db)(TableName.Project).where({ id: projectId }).update(data); } catch (error) { throw new DatabaseError({ error, name: "Set project upgrade status" }); } From b587e6a4a1372eea569807eee690c22f3f8de1dd Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 18:15:27 +0100 Subject: [PATCH 122/157] Update models.ts --- backend/src/db/schemas/models.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index a405ba004..afe1c3091 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -125,7 +125,7 @@ export enum ProjectUpgradeStatus { export enum ProjectUpgradeStatus { InProgress = "IN_PROGRESS", - // Completed = "Project upgrade completed.", // Will be null if completed. So a completed status is not needed + // Completed -> Will be null if completed. So a completed status is not needed Failed = "FAILED" } From 4c5100de6b5a7bc5c33f01e015103c57dabff17c Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 18:19:43 +0100 Subject: [PATCH 123/157] Update project-key-service.ts --- backend/src/services/project-key/project-key-service.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/src/services/project-key/project-key-service.ts b/backend/src/services/project-key/project-key-service.ts index 81b589bd0..fa77760a4 100644 --- a/backend/src/services/project-key/project-key-service.ts +++ b/backend/src/services/project-key/project-key-service.ts @@ -1,5 +1,4 @@ import { ForbiddenError } from "@casl/ability"; -import { Knex } from "knex"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; @@ -44,7 +43,7 @@ export const projectKeyServiceFactory = ({ name: "Upload project keys" }); - await projectKeyDAL.create({ projectId, receiverId, encryptedKey, nonce, senderId: actorId }, tx); + await projectKeyDAL.create({ projectId, receiverId, encryptedKey, nonce, senderId: actorId }); }; const getLatestProjectKey = async ({ actorId, projectId, actor, actorOrgId }: TGetLatestProjectKeyDTO) => { From fb9cdb591c889be6327494821b26e354edda130e Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 19:28:28 +0100 Subject: [PATCH 124/157] Error renaming --- backend/src/services/project/project-dal.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/backend/src/services/project/project-dal.ts b/backend/src/services/project/project-dal.ts index 8b2971595..7d0826e12 100644 --- a/backend/src/services/project/project-dal.ts +++ b/backend/src/services/project/project-dal.ts @@ -80,18 +80,6 @@ export const projectDALFactory = (db: TDbClient) => { } }; - const setProjectUpgradeStatus = async (projectId: string, status: ProjectUpgradeStatus | null, tx?: Knex) => { - try { - const data: TProjectsUpdate = { - upgradeStatus: status - } as const; - - await (tx || db)(TableName.Project).where({ id: projectId }).update(data); - } catch (error) { - throw new DatabaseError({ error, name: "Set project upgrade status" }); - } - }; - const findAllProjectsByIdentity = async (identityId: string) => { try { const workspaces = await db(TableName.IdentityProjectMembership) From 178492e9bd1d0d40717595ac88bbb6c4144ec673 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 19:29:20 +0100 Subject: [PATCH 125/157] Add logger to avoid crash --- backend/src/db/seeds/3-project.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/db/seeds/3-project.ts b/backend/src/db/seeds/3-project.ts index 9887373fd..05feda4ef 100644 --- a/backend/src/db/seeds/3-project.ts +++ b/backend/src/db/seeds/3-project.ts @@ -14,7 +14,7 @@ export const DEFAULT_PROJECT_ENVS = [ ]; export async function seed(knex: Knex): Promise { - initEnvConfig(); + initEnvConfig(await initLogger()); const appCfg = getConfig(); // Deletes ALL existing entries await knex(TableName.Project).del(); From 198dc05753bec13a575ee603ed3d95bdbf6e6966 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 20:12:33 +0100 Subject: [PATCH 126/157] Allow getting bot, but not creating --- backend/src/server/routes/v1/bot-router.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/backend/src/server/routes/v1/bot-router.ts b/backend/src/server/routes/v1/bot-router.ts index 20acf95ec..91d27b2e5 100644 --- a/backend/src/server/routes/v1/bot-router.ts +++ b/backend/src/server/routes/v1/bot-router.ts @@ -27,16 +27,6 @@ export const registerProjectBotRouter = async (server: FastifyZodProvider) => { }, onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { - const project = await server.services.project.getAProject({ - actorId: req.permission.id, - actor: req.permission.type, - projectId: req.params.projectId - }); - - if (project.version === "v2") { - throw new BadRequestError({ message: "Failed to find bot, project has E2EE disabled" }); - } - const bot = await server.services.projectBot.findBotByProjectId({ actor: req.permission.type, actorId: req.permission.id, From 28b24115b71af5cfa9044faf7bd90ec5114ee5f2 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 20:18:12 +0100 Subject: [PATCH 127/157] Move check to inside service --- backend/src/server/routes/v1/bot-router.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/backend/src/server/routes/v1/bot-router.ts b/backend/src/server/routes/v1/bot-router.ts index 91d27b2e5..507423b0d 100644 --- a/backend/src/server/routes/v1/bot-router.ts +++ b/backend/src/server/routes/v1/bot-router.ts @@ -1,7 +1,6 @@ import { z } from "zod"; import { ProjectBotsSchema } from "@app/db/schemas"; -import { BadRequestError } from "@app/lib/errors"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; @@ -67,12 +66,6 @@ export const registerProjectBotRouter = async (server: FastifyZodProvider) => { }, onRequest: verifyAuth([AuthMode.JWT]), handler: async (req) => { - const project = await server.services.projectBot.findProjectByBotId(req.params.botId); - - if (project?.version === "v2") { - throw new BadRequestError({ message: "Failed to set bot active, project has E2EE disabled" }); - } - const bot = await server.services.projectBot.setBotActiveState({ actor: req.permission.type, actorId: req.permission.id, From faa842c3d2862446af5586aa7996d46bbbbf2db1 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 13 Feb 2024 23:22:55 +0100 Subject: [PATCH 128/157] Revert seed --- backend/src/db/seeds/1-user.ts | 16 ++------------ backend/src/db/seeds/2-org.ts | 3 --- backend/src/db/seeds/3-project.ts | 36 +++++-------------------------- 3 files changed, 7 insertions(+), 48 deletions(-) diff --git a/backend/src/db/seeds/1-user.ts b/backend/src/db/seeds/1-user.ts index 5f9fcd509..8e2b00b90 100644 --- a/backend/src/db/seeds/1-user.ts +++ b/backend/src/db/seeds/1-user.ts @@ -1,14 +1,8 @@ -/* eslint-disable @typescript-eslint/ban-ts-comment */ -// @ts-nocheck - import { Knex } from "knex"; -// eslint-disable-next-line @typescript-eslint/no-unused-vars -import { generateUserSrpKeys } from "@app/lib/crypto/srp"; - import { AuthMethod } from "../../services/auth/auth-type"; import { TableName } from "../schemas"; -import { seedData1 } from "../seed-data"; +import { generateUserSrpKeys, seedData1 } from "../seed-data"; export async function seed(knex: Knex): Promise { // Deletes ALL existing entries @@ -25,7 +19,6 @@ export async function seed(knex: Knex): Promise { id: seedData1.id, email: seedData1.email, superAdmin: true, - ghost: false, firstName: "test", lastName: "", authMethods: [AuthMethod.EMAIL], @@ -37,7 +30,7 @@ export async function seed(knex: Knex): Promise { ]) .returning("*"); - const encKeys = await generateUserSrpKeys(seedData1.email, seedData1.password); + const encKeys = await generateUserSrpKeys(seedData1.password); // password: testInfisical@1 await knex(TableName.UserEncryptionKey).insert([ { @@ -67,9 +60,4 @@ export async function seed(knex: Knex): Promise { refreshVersion: 1, lastUsed: new Date() }); - - seedData1.encryptionKeys = { - publicKey: encKeys.publicKey, - privateKey: encKeys.plainPrivateKey - }; } diff --git a/backend/src/db/seeds/2-org.ts b/backend/src/db/seeds/2-org.ts index a2990054b..ba2f65a36 100644 --- a/backend/src/db/seeds/2-org.ts +++ b/backend/src/db/seeds/2-org.ts @@ -1,6 +1,3 @@ -/* eslint-disable @typescript-eslint/ban-ts-comment */ -// @ts-nocheck - import { Knex } from "knex"; import { OrgMembershipRole, OrgMembershipStatus, TableName } from "../schemas"; diff --git a/backend/src/db/seeds/3-project.ts b/backend/src/db/seeds/3-project.ts index 05feda4ef..0efb6a249 100644 --- a/backend/src/db/seeds/3-project.ts +++ b/backend/src/db/seeds/3-project.ts @@ -14,8 +14,6 @@ export const DEFAULT_PROJECT_ENVS = [ ]; export async function seed(knex: Knex): Promise { - initEnvConfig(await initLogger()); - const appCfg = getConfig(); // Deletes ALL existing entries await knex(TableName.Project).del(); await knex(TableName.Environment).del(); @@ -27,38 +25,14 @@ export async function seed(knex: Knex): Promise { orgId: seedData1.organization.id, slug: "first-project", // @ts-expect-error exluded type id needs to be inserted here to keep it testable - id: seedData1.project.id, - version: "v1" + id: seedData1.project.id }) .returning("*"); - const blindIndex = createSecretBlindIndex(appCfg.ROOT_ENCRYPTION_KEY, appCfg.ENCRYPTION_KEY); - - await knex(TableName.SecretBlindIndex).insert({ - projectId: project.id, - algorithm: blindIndex.algorithm, - keyEncoding: blindIndex.keyEncoding, - saltIV: blindIndex.iv, - encryptedSaltCipherText: blindIndex.ciphertext, - saltTag: blindIndex.tag - }); - - const randomBytes = crypto.randomBytes(16).toString("hex"); // Project key - // const encKeys = await generateUserSrpKeys(seedData1.email, seedData1.password); // User keys - - const { ciphertext: encryptedProjectKey, nonce: encryptedProjectKeyIv } = encryptAsymmetric( - randomBytes, - seedData1.encryptionKeys.publicKey, - seedData1.encryptionKeys.privateKey - ); - - await knex(TableName.ProjectKeys).insert({ - projectId: project.id, - senderId: seedData1.id, - receiverId: seedData1.id, - encryptedKey: encryptedProjectKey, - nonce: encryptedProjectKeyIv - }); + // await knex(TableName.ProjectKeys).insert({ + // projectId: project.id, + // senderId: seedData1.id + // }); await knex(TableName.ProjectMembership).insert({ projectId: project.id, From b5e64bc8b8f76ef086c051bd7ff23b21b4dd984f Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 14 Feb 2024 00:15:17 +0100 Subject: [PATCH 129/157] Update srp.ts --- backend/src/lib/crypto/srp.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/src/lib/crypto/srp.ts b/backend/src/lib/crypto/srp.ts index 6f4588e3f..d0d290be1 100644 --- a/backend/src/lib/crypto/srp.ts +++ b/backend/src/lib/crypto/srp.ts @@ -8,6 +8,10 @@ import { TUserEncryptionKeys } from "@app/db/schemas"; import { decryptSymmetric, encryptAsymmetric, encryptSymmetric } from "./encryption"; +// Importing the argon2 constants from the argon2 module fails due to an issue with importing commonjs modules. +// Read more: https://stackoverflow.com/questions/70605320/named-export-types-not-found-the-requested-module-mongoose-is-a-commonjs-mo +const ARGON_2_ID = 2; + export const generateSrpServerKey = async (salt: string, verifier: string) => { // eslint-disable-next-line new-cap const server = new jsrp.server(); From 8a39276e04d6df95c02bd1bf77b4c521c4b89ffa Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 14 Feb 2024 00:20:05 +0100 Subject: [PATCH 130/157] Fix for commonJS --- backend/src/lib/crypto/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/lib/crypto/index.ts b/backend/src/lib/crypto/index.ts index a92508b1d..9d7f4e886 100644 --- a/backend/src/lib/crypto/index.ts +++ b/backend/src/lib/crypto/index.ts @@ -18,4 +18,3 @@ export { decryptSecretVersions } from "./secret-encryption"; export { generateSrpServerKey, srpCheckClientProof } from "./srp"; -export { decodeBase64, encodeBase64 } from "tweetnacl-util"; From 021413fbd94913544a9f8cdf62e4801ef4426151 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 14 Feb 2024 00:52:41 +0100 Subject: [PATCH 131/157] Update project-router.ts --- backend/src/server/routes/v1/project-router.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/server/routes/v1/project-router.ts b/backend/src/server/routes/v1/project-router.ts index 96452a549..0a5f8d85e 100644 --- a/backend/src/server/routes/v1/project-router.ts +++ b/backend/src/server/routes/v1/project-router.ts @@ -282,7 +282,6 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { } }); - // Is this actually used..? server.route({ url: "/:workspaceId/integrations", method: "GET", From 4ab47ca1755af2604ccffdf01bbbe58c26769224 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 14 Feb 2024 18:04:09 +0100 Subject: [PATCH 132/157] Fix for "random" crash on creation --- frontend/src/layouts/AppLayout/AppLayout.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/src/layouts/AppLayout/AppLayout.tsx b/frontend/src/layouts/AppLayout/AppLayout.tsx index 357e7c770..9dd2d82e7 100644 --- a/frontend/src/layouts/AppLayout/AppLayout.tsx +++ b/frontend/src/layouts/AppLayout/AppLayout.tsx @@ -238,6 +238,9 @@ export const AppLayout = ({ children }: LayoutProps) => { projectId: newProjectId }); } + + // eslint-disable-next-line no-promise-executor-return -- We do this because the function returns too fast, which sometimes causes an error when the user is redirected. + await new Promise((resolve) => setTimeout(resolve, 2_000)); // eslint-disable-next-line no-promise-executor-return -- We do this because the function returns too fast, which sometimes causes an error when the user is redirected. await new Promise((resolve) => setTimeout(resolve, 2_000)); From debf80cfdc8b1b10c9ad87b1a55251235c0aada3 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 16 Feb 2024 01:30:59 +0100 Subject: [PATCH 133/157] More --- .../components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx b/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx index ec3bdd9e4..15b076187 100644 --- a/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx +++ b/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx @@ -22,7 +22,6 @@ export const UpgradeProjectAlert = ({ project }: UpgradeProjectAlertProps): JSX. const upgradeProject = useUpgradeProject(); const [currentStatus, setCurrentStatus] = useState(null); const [isUpgrading, setIsUpgrading] = useState(false); - const { data: projectStatus, isLoading: statusIsLoading, From 50128bbac696a678c184c73259f91cb0946845ae Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 16 Feb 2024 01:49:52 +0100 Subject: [PATCH 134/157] Test decrypt fix --- backend/src/lib/secret/index.ts | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/backend/src/lib/secret/index.ts b/backend/src/lib/secret/index.ts index 08cb0163d..848ebde70 100644 --- a/backend/src/lib/secret/index.ts +++ b/backend/src/lib/secret/index.ts @@ -1,3 +1,4 @@ +import crypto from "crypto"; import { z } from "zod"; import { SecretKeyEncoding, TProjectKeys } from "@app/db/schemas"; @@ -50,16 +51,30 @@ const symmetricDecrypt = ({ key, isApprovalSecret }: TDecryptSymmetricInput & { keyEncoding: SecretKeyEncoding; isApprovalSecret: boolean }) => { - if (keyEncoding === SecretKeyEncoding.UTF8 || isApprovalSecret) { - const data = decryptSymmetric128BitHexKeyUTF8({ key, iv, tag, ciphertext }); - return data; - } - if (keyEncoding === SecretKeyEncoding.BASE64) { - const data = decryptSymmetric({ key, iv, tag, ciphertext }); - return data; - } + try { + if (keyEncoding === SecretKeyEncoding.UTF8 || isApprovalSecret) { + const data = decryptSymmetric128BitHexKeyUTF8({ key, iv, tag, ciphertext }); + return data; + } + if (keyEncoding === SecretKeyEncoding.BASE64) { + const data = decryptSymmetric({ key, iv, tag, ciphertext }); + return data; + } + throw new Error("BAD_ENCODING"); + } catch (err) { + if (err instanceof Error && err.message === "BAD_ENCODING") { + throw new Error("Invalid key encoding, cannot decrypt secret!"); + } - throw new Error("Missing both encryption keys"); + // This is taken directly from our frontend secret decryption logic. + const decipher = crypto.createDecipheriv("aes-256-gcm", key, Buffer.from(iv, "base64")); + decipher.setAuthTag(Buffer.from(tag, "base64")); + + let data = decipher.update(ciphertext, "base64", "utf8"); + data += decipher.final("utf8"); + + return data; + } }; export const decryptSecrets = ( From 5b1f07a66125410c7c31aa9a89c22f6ed7230900 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 16 Feb 2024 16:42:16 +0100 Subject: [PATCH 135/157] Move migration to latest --- .../migrations/20240202093209_ghost_user.ts | 39 ------------------- 1 file changed, 39 deletions(-) delete mode 100644 backend/src/db/migrations/20240202093209_ghost_user.ts diff --git a/backend/src/db/migrations/20240202093209_ghost_user.ts b/backend/src/db/migrations/20240202093209_ghost_user.ts deleted file mode 100644 index ddfdbe812..000000000 --- a/backend/src/db/migrations/20240202093209_ghost_user.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { Knex } from "knex"; - -import { ProjectVersion, TableName } from "../schemas"; - -export async function up(knex: Knex): Promise { - const hasGhostUserColumn = await knex.schema.hasColumn(TableName.Users, "ghost"); - const hasProjectVersionColumn = await knex.schema.hasColumn(TableName.Project, "version"); - - if (!hasGhostUserColumn) { - await knex.schema.alterTable(TableName.Users, (t) => { - t.boolean("ghost").defaultTo(false).notNullable(); - }); - } - - if (!hasProjectVersionColumn) { - await knex.schema.alterTable(TableName.Project, (t) => { - t.string("version").defaultTo(ProjectVersion.V1).notNullable(); - t.text("upgradeStatus").nullable(); - }); - } -} - -export async function down(knex: Knex): Promise { - const hasGhostUserColumn = await knex.schema.hasColumn(TableName.Users, "ghost"); - const hasProjectVersionColumn = await knex.schema.hasColumn(TableName.Project, "version"); - - if (hasGhostUserColumn) { - await knex.schema.alterTable(TableName.Users, (t) => { - t.dropColumn("ghost"); - }); - } - - if (hasProjectVersionColumn) { - await knex.schema.alterTable(TableName.Project, (t) => { - t.dropColumn("version"); - t.dropColumn("upgradeStatus"); - }); - } -} From 27b47492052c28c371cc712521b4123a5bb284f0 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 16 Feb 2024 18:19:08 +0100 Subject: [PATCH 136/157] More tests --- backend/src/lib/secret/index.ts | 161 +++++++++++++++++++------------- 1 file changed, 95 insertions(+), 66 deletions(-) diff --git a/backend/src/lib/secret/index.ts b/backend/src/lib/secret/index.ts index 848ebde70..361d152de 100644 --- a/backend/src/lib/secret/index.ts +++ b/backend/src/lib/secret/index.ts @@ -1,10 +1,10 @@ import crypto from "crypto"; import { z } from "zod"; -import { SecretKeyEncoding, TProjectKeys } from "@app/db/schemas"; +import { TProjectKeys } from "@app/db/schemas"; -import { decryptAsymmetric, decryptSymmetric } from "../crypto"; -import { decryptSymmetric128BitHexKeyUTF8, TDecryptSymmetricInput } from "../crypto/encryption"; +import { decryptAsymmetric } from "../crypto"; +// import { decryptSymmetric128BitHexKeyUTF8, TDecryptSymmetricInput } from "../crypto/encryption"; export enum SecretDocType { Secret = "secret", @@ -43,38 +43,60 @@ const PartialDecryptedSecretSchema = z.object({ export type TPartialSecret = z.infer; export type TPartialDecryptedSecret = z.infer; -const symmetricDecrypt = ({ - keyEncoding, +// const symmetricDecrypt = ({ +// keyEncoding, +// ciphertext, +// tag, +// iv, +// key, +// isApprovalSecret +// }: TDecryptSymmetricInput & { keyEncoding: SecretKeyEncoding; isApprovalSecret: boolean }) => { +// try { +// if (keyEncoding === SecretKeyEncoding.UTF8 || isApprovalSecret) { +// const data = decryptSymmetric128BitHexKeyUTF8({ key, iv, tag, ciphertext }); +// return data; +// } +// if (keyEncoding === SecretKeyEncoding.BASE64) { +// const data = decryptSymmetric({ key, iv, tag, ciphertext }); +// return data; +// } +// throw new Error("BAD_ENCODING"); +// } catch (err) { +// if (err instanceof Error && err.message === "BAD_ENCODING") { +// throw new Error("Invalid key encoding, cannot decrypt secret!"); +// } + +// // This is taken directly from our frontend secret decryption logic. +// const decipher = crypto.createDecipheriv("aes-256-gcm", key, Buffer.from(iv, "base64")); +// decipher.setAuthTag(Buffer.from(tag, "base64")); + +// let data = decipher.update(ciphertext, "base64", "utf8"); +// data += decipher.final("utf8"); + +// console.log(data); + +// return data; +// } +// }; + +const decryptSecret = ({ ciphertext, - tag, iv, - key, - isApprovalSecret -}: TDecryptSymmetricInput & { keyEncoding: SecretKeyEncoding; isApprovalSecret: boolean }) => { - try { - if (keyEncoding === SecretKeyEncoding.UTF8 || isApprovalSecret) { - const data = decryptSymmetric128BitHexKeyUTF8({ key, iv, tag, ciphertext }); - return data; - } - if (keyEncoding === SecretKeyEncoding.BASE64) { - const data = decryptSymmetric({ key, iv, tag, ciphertext }); - return data; - } - throw new Error("BAD_ENCODING"); - } catch (err) { - if (err instanceof Error && err.message === "BAD_ENCODING") { - throw new Error("Invalid key encoding, cannot decrypt secret!"); - } + tag, + key +}: { + ciphertext: string; + iv: string; + tag: string; + key: string | Buffer; +}) => { + const decipher = crypto.createDecipheriv("aes-256-gcm", key, Buffer.from(iv, "base64")); + decipher.setAuthTag(Buffer.from(tag, "base64")); - // This is taken directly from our frontend secret decryption logic. - const decipher = crypto.createDecipheriv("aes-256-gcm", key, Buffer.from(iv, "base64")); - decipher.setAuthTag(Buffer.from(tag, "base64")); + let cleartext = decipher.update(ciphertext, "base64", "utf8"); + cleartext += decipher.final("utf8"); - let data = decipher.update(ciphertext, "base64", "utf8"); - data += decipher.final("utf8"); - - return data; - } + return cleartext; }; export const decryptSecrets = ( @@ -96,45 +118,52 @@ export const decryptSecrets = ( const secrets: TPartialDecryptedSecret[] = []; encryptedSecrets.forEach((encSecret) => { - const secretKey = symmetricDecrypt({ - ciphertext: encSecret.secretKeyCiphertext, - iv: encSecret.secretKeyIV, - tag: encSecret.secretKeyTag, - key, - keyEncoding: encSecret.keyEncoding as SecretKeyEncoding, - isApprovalSecret: encSecret.docType === SecretDocType.ApprovalSecret - }); + try { + console.log(encSecret.keyEncoding); - const secretValue = symmetricDecrypt({ - ciphertext: encSecret.secretValueCiphertext, - iv: encSecret.secretValueIV, - tag: encSecret.secretValueTag, - key, - keyEncoding: encSecret.keyEncoding as SecretKeyEncoding, - isApprovalSecret: encSecret.docType === SecretDocType.ApprovalSecret - }); + const secretKey = decryptSecret({ + ciphertext: encSecret.secretKeyCiphertext, + iv: encSecret.secretKeyIV, + tag: encSecret.secretKeyTag, + key + // keyEncoding: encSecret.keyEncoding as SecretKeyEncoding, + // isApprovalSecret: encSecret.docType === SecretDocType.ApprovalSecret + }); - const secretComment = - encSecret.secretCommentCiphertext && encSecret.secretCommentIV && encSecret.secretCommentTag - ? symmetricDecrypt({ - ciphertext: encSecret.secretCommentCiphertext, - iv: encSecret.secretCommentIV, - tag: encSecret.secretCommentTag, - key, - keyEncoding: encSecret.keyEncoding as SecretKeyEncoding, - isApprovalSecret: encSecret.docType === SecretDocType.ApprovalSecret - }) - : ""; + const secretValue = decryptSecret({ + ciphertext: encSecret.secretValueCiphertext, + iv: encSecret.secretValueIV, + tag: encSecret.secretValueTag, + key + // keyEncoding: encSecret.keyEncoding as SecretKeyEncoding, + // isApprovalSecret: encSecret.docType === SecretDocType.ApprovalSecret + }); - const decryptedSecret: TPartialDecryptedSecret = { - id: encSecret.id, - secretKey, - secretValue, - secretComment, - docType: encSecret.docType - }; + const secretComment = + encSecret.secretCommentCiphertext && encSecret.secretCommentIV && encSecret.secretCommentTag + ? decryptSecret({ + ciphertext: encSecret.secretCommentCiphertext, + iv: encSecret.secretCommentIV, + tag: encSecret.secretCommentTag, + key + // keyEncoding: encSecret.keyEncoding as SecretKeyEncoding, + // isApprovalSecret: encSecret.docType === SecretDocType.ApprovalSecret + }) + : ""; - secrets.push(decryptedSecret); + const decryptedSecret: TPartialDecryptedSecret = { + id: encSecret.id, + secretKey, + secretValue, + secretComment, + docType: encSecret.docType + }; + + secrets.push(decryptedSecret); + } catch (err) { + // This is ok, because we check that the decrypted secrets array length is the same as the encrypted secrets array length. + console.log(`[${encSecret.id}] - failed to decrypt`, err); + } }); return secrets; From 8e49825e1612abb182a6dbaad5a0b53346ee3f6a Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sat, 17 Feb 2024 00:13:43 +0100 Subject: [PATCH 137/157] Cleanup --- backend/src/lib/secret/index.ts | 87 ++++++++------------------------- 1 file changed, 20 insertions(+), 67 deletions(-) diff --git a/backend/src/lib/secret/index.ts b/backend/src/lib/secret/index.ts index 361d152de..4292c7945 100644 --- a/backend/src/lib/secret/index.ts +++ b/backend/src/lib/secret/index.ts @@ -2,9 +2,9 @@ import crypto from "crypto"; import { z } from "zod"; import { TProjectKeys } from "@app/db/schemas"; +import { logger } from "@app/lib/logger"; import { decryptAsymmetric } from "../crypto"; -// import { decryptSymmetric128BitHexKeyUTF8, TDecryptSymmetricInput } from "../crypto/encryption"; export enum SecretDocType { Secret = "secret", @@ -12,24 +12,23 @@ export enum SecretDocType { ApprovalSecret = "approvalSecret" } -const PartialSecretSchema = z.object({ - id: z.string(), - secretKeyCiphertext: z.string(), - secretKeyIV: z.string(), - secretKeyTag: z.string(), +export interface TPartialSecret { + id: string; + secretKeyCiphertext: string; + secretKeyIV: string; + secretKeyTag: string; - secretValueCiphertext: z.string(), - secretValueIV: z.string(), - secretValueTag: z.string(), + secretValueCiphertext: string; + secretValueIV: string; + secretValueTag: string; - secretCommentCiphertext: z.string().nullish(), - secretCommentIV: z.string().nullish(), - secretCommentTag: z.string().nullish(), + secretCommentCiphertext?: string | null; + secretCommentIV?: string | null; + secretCommentTag?: string | null; - docType: z.nativeEnum(SecretDocType), - - keyEncoding: z.string() -}); + docType: SecretDocType; + keyEncoding: string; +} const PartialDecryptedSecretSchema = z.object({ id: z.string(), @@ -39,46 +38,8 @@ const PartialDecryptedSecretSchema = z.object({ docType: z.nativeEnum(SecretDocType) }); - -export type TPartialSecret = z.infer; export type TPartialDecryptedSecret = z.infer; -// const symmetricDecrypt = ({ -// keyEncoding, -// ciphertext, -// tag, -// iv, -// key, -// isApprovalSecret -// }: TDecryptSymmetricInput & { keyEncoding: SecretKeyEncoding; isApprovalSecret: boolean }) => { -// try { -// if (keyEncoding === SecretKeyEncoding.UTF8 || isApprovalSecret) { -// const data = decryptSymmetric128BitHexKeyUTF8({ key, iv, tag, ciphertext }); -// return data; -// } -// if (keyEncoding === SecretKeyEncoding.BASE64) { -// const data = decryptSymmetric({ key, iv, tag, ciphertext }); -// return data; -// } -// throw new Error("BAD_ENCODING"); -// } catch (err) { -// if (err instanceof Error && err.message === "BAD_ENCODING") { -// throw new Error("Invalid key encoding, cannot decrypt secret!"); -// } - -// // This is taken directly from our frontend secret decryption logic. -// const decipher = crypto.createDecipheriv("aes-256-gcm", key, Buffer.from(iv, "base64")); -// decipher.setAuthTag(Buffer.from(tag, "base64")); - -// let data = decipher.update(ciphertext, "base64", "utf8"); -// data += decipher.final("utf8"); - -// console.log(data); - -// return data; -// } -// }; - const decryptSecret = ({ ciphertext, iv, @@ -115,19 +76,15 @@ export const decryptSecrets = ( privateKey }); - const secrets: TPartialDecryptedSecret[] = []; + const decryptedSecrets: TPartialDecryptedSecret[] = []; encryptedSecrets.forEach((encSecret) => { try { - console.log(encSecret.keyEncoding); - const secretKey = decryptSecret({ ciphertext: encSecret.secretKeyCiphertext, iv: encSecret.secretKeyIV, tag: encSecret.secretKeyTag, key - // keyEncoding: encSecret.keyEncoding as SecretKeyEncoding, - // isApprovalSecret: encSecret.docType === SecretDocType.ApprovalSecret }); const secretValue = decryptSecret({ @@ -135,8 +92,6 @@ export const decryptSecrets = ( iv: encSecret.secretValueIV, tag: encSecret.secretValueTag, key - // keyEncoding: encSecret.keyEncoding as SecretKeyEncoding, - // isApprovalSecret: encSecret.docType === SecretDocType.ApprovalSecret }); const secretComment = @@ -146,8 +101,6 @@ export const decryptSecrets = ( iv: encSecret.secretCommentIV, tag: encSecret.secretCommentTag, key - // keyEncoding: encSecret.keyEncoding as SecretKeyEncoding, - // isApprovalSecret: encSecret.docType === SecretDocType.ApprovalSecret }) : ""; @@ -159,12 +112,12 @@ export const decryptSecrets = ( docType: encSecret.docType }; - secrets.push(decryptedSecret); + decryptedSecrets.push(PartialDecryptedSecretSchema.parse(decryptedSecret)); } catch (err) { - // This is ok, because we check that the decrypted secrets array length is the same as the encrypted secrets array length. - console.log(`[${encSecret.id}] - failed to decrypt`, err); + // This is ok, because we check that the decrypted secrets array length is the same as the encrypted secrets input array length. + logger.error(`[${encSecret.id}] - failed to decrypt`, err); } }); - return secrets; + return decryptedSecrets; }; From bf9b47ad664c9c506bfaf28027b36fc12ff1b473 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sat, 17 Feb 2024 01:02:30 +0100 Subject: [PATCH 138/157] Requested changes --- backend/src/db/schemas/models.ts | 6 ---- backend/src/services/org/org-dal.ts | 33 ------------------- .../project-membership-service.ts | 1 - 3 files changed, 40 deletions(-) diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index afe1c3091..abfb67949 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -123,12 +123,6 @@ export enum ProjectUpgradeStatus { Failed = "FAILED" } -export enum ProjectUpgradeStatus { - InProgress = "IN_PROGRESS", - // Completed -> Will be null if completed. So a completed status is not needed - Failed = "FAILED" -} - export enum IdentityAuthMethod { Univeral = "universal-auth" } diff --git a/backend/src/services/org/org-dal.ts b/backend/src/services/org/org-dal.ts index bad1ba9f6..05c77e746 100644 --- a/backend/src/services/org/org-dal.ts +++ b/backend/src/services/org/org-dal.ts @@ -121,39 +121,6 @@ export const orgDALFactory = (db: TDbClient) => { } }; - const findOrgMembersByEmail = async (orgId: string, emails: string[]) => { - try { - const members = await db(TableName.OrgMembership) - .where({ orgId }) - .join(TableName.Users, `${TableName.OrgMembership}.userId`, `${TableName.Users}.id`) - .leftJoin( - TableName.UserEncryptionKey, - `${TableName.UserEncryptionKey}.userId`, - `${TableName.Users}.id` - ) - .select( - db.ref("id").withSchema(TableName.OrgMembership), - db.ref("inviteEmail").withSchema(TableName.OrgMembership), - db.ref("orgId").withSchema(TableName.OrgMembership), - db.ref("role").withSchema(TableName.OrgMembership), - db.ref("roleId").withSchema(TableName.OrgMembership), - db.ref("status").withSchema(TableName.OrgMembership), - db.ref("email").withSchema(TableName.Users), - db.ref("firstName").withSchema(TableName.Users), - db.ref("lastName").withSchema(TableName.Users), - db.ref("id").withSchema(TableName.Users).as("userId"), - db.ref("publicKey").withSchema(TableName.UserEncryptionKey) - ) - .whereIn("email", emails); - return members.map(({ email, firstName, lastName, userId, publicKey, ...data }) => ({ - ...data, - user: { email, firstName, lastName, id: userId, publicKey } - })); - } catch (error) { - throw new DatabaseError({ error, name: "Find all org members" }); - } - }; - const findOrgGhostUser = async (orgId: string) => { try { const member = await db(TableName.OrgMembership) diff --git a/backend/src/services/project-membership/project-membership-service.ts b/backend/src/services/project-membership/project-membership-service.ts index 0450450d8..e96247c97 100644 --- a/backend/src/services/project-membership/project-membership-service.ts +++ b/backend/src/services/project-membership/project-membership-service.ts @@ -17,7 +17,6 @@ import { getConfig } from "@app/lib/config/env"; import { infisicalSymmetricDecrypt } from "@app/lib/crypto/encryption"; import { BadRequestError } from "@app/lib/errors"; import { groupBy } from "@app/lib/fn"; -import { createWsMembers } from "@app/lib/project"; import { ActorType } from "../auth/auth-type"; import { TOrgDALFactory } from "../org/org-dal"; From fe7524fca19639c0b4e5431a878b1b211548f503 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sat, 17 Feb 2024 01:02:52 +0100 Subject: [PATCH 139/157] Requested changes --- backend/src/lib/crypto/srp.ts | 4 --- backend/src/lib/project/index.ts | 60 -------------------------------- 2 files changed, 64 deletions(-) delete mode 100644 backend/src/lib/project/index.ts diff --git a/backend/src/lib/crypto/srp.ts b/backend/src/lib/crypto/srp.ts index d0d290be1..6f4588e3f 100644 --- a/backend/src/lib/crypto/srp.ts +++ b/backend/src/lib/crypto/srp.ts @@ -8,10 +8,6 @@ import { TUserEncryptionKeys } from "@app/db/schemas"; import { decryptSymmetric, encryptAsymmetric, encryptSymmetric } from "./encryption"; -// Importing the argon2 constants from the argon2 module fails due to an issue with importing commonjs modules. -// Read more: https://stackoverflow.com/questions/70605320/named-export-types-not-found-the-requested-module-mongoose-is-a-commonjs-mo -const ARGON_2_ID = 2; - export const generateSrpServerKey = async (salt: string, verifier: string) => { // eslint-disable-next-line new-cap const server = new jsrp.server(); diff --git a/backend/src/lib/project/index.ts b/backend/src/lib/project/index.ts deleted file mode 100644 index e0ed0d70b..000000000 --- a/backend/src/lib/project/index.ts +++ /dev/null @@ -1,60 +0,0 @@ -import crypto from "crypto"; - -import { ProjectMembershipRole, TProjectKeys } from "@app/db/schemas"; - -import { decryptAsymmetric, encryptAsymmetric } from "../crypto"; - -type AddUserToWsDTO = { - decryptKey: TProjectKeys & { sender: { publicKey: string } }; - userPrivateKey: string; - members: { - orgMembershipId: string; - projectMembershipRole: ProjectMembershipRole; - userPublicKey: string; - }[]; -}; - -export const createWsMembers = ({ members, decryptKey, userPrivateKey }: AddUserToWsDTO) => { - const key = decryptAsymmetric({ - ciphertext: decryptKey.encryptedKey, - nonce: decryptKey.nonce, - publicKey: decryptKey.sender.publicKey, - privateKey: userPrivateKey - }); - - const newWsMembers = members.map(({ orgMembershipId, userPublicKey, projectMembershipRole }) => { - const { ciphertext: inviteeCipherText, nonce: inviteeNonce } = encryptAsymmetric( - key, - userPublicKey, - userPrivateKey - ); - - return { - orgMembershipId, - projectRole: projectMembershipRole, - workspaceEncryptedKey: inviteeCipherText, - workspaceEncryptedNonce: inviteeNonce - }; - }); - - return newWsMembers; -}; - -type TCreateProjectKeyDTO = { - publicKey: string; - privateKey: string; -}; - -export const createProjectKey = ({ publicKey, privateKey }: TCreateProjectKeyDTO) => { - // 3. Create a random key that we'll use as the project key. - const randomBytes = crypto.randomBytes(16).toString("hex"); - - // 4. Encrypt the project key with the users key pair. - const { ciphertext: encryptedProjectKey, nonce: encryptedProjectKeyIv } = encryptAsymmetric( - randomBytes, - publicKey, - privateKey - ); - - return { key: encryptedProjectKey, iv: encryptedProjectKeyIv }; -}; From 68ac1d285ada44e2dfa3819df4851b2670b8a609 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sat, 17 Feb 2024 02:49:16 +0100 Subject: [PATCH 140/157] Improved polling --- .../components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx b/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx index 15b076187..ec3bdd9e4 100644 --- a/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx +++ b/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx @@ -22,6 +22,7 @@ export const UpgradeProjectAlert = ({ project }: UpgradeProjectAlertProps): JSX. const upgradeProject = useUpgradeProject(); const [currentStatus, setCurrentStatus] = useState(null); const [isUpgrading, setIsUpgrading] = useState(false); + const { data: projectStatus, isLoading: statusIsLoading, From 4657985468bf6373a2e847befbaf799c38940ff9 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sat, 17 Feb 2024 02:49:57 +0100 Subject: [PATCH 141/157] Update project-service.ts --- backend/src/services/project/project-service.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index e88a5879e..5826e77f0 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -1,6 +1,3 @@ -/* eslint-disable @typescript-eslint/no-redeclare */ -/* eslint-disable no-console */ -/* eslint-disable no-await-in-loop */ import { ForbiddenError } from "@casl/ability"; import slugify from "@sindresorhus/slugify"; From 10fbb99a156a9c6bb90d156c7c217560541646ab Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Mon, 19 Feb 2024 22:01:32 +0100 Subject: [PATCH 142/157] Refactor migration to work with conflict/merge update logic --- backend/src/lib/secret/index.ts | 269 +++++++++++++----- backend/src/services/project/project-queue.ts | 15 + 2 files changed, 214 insertions(+), 70 deletions(-) diff --git a/backend/src/lib/secret/index.ts b/backend/src/lib/secret/index.ts index 4292c7945..036b49970 100644 --- a/backend/src/lib/secret/index.ts +++ b/backend/src/lib/secret/index.ts @@ -1,46 +1,45 @@ import crypto from "crypto"; import { z } from "zod"; -import { TProjectKeys } from "@app/db/schemas"; -import { logger } from "@app/lib/logger"; +import { + SecretApprovalRequestsSecretsSchema, + SecretsSchema, + SecretVersionsSchema, + TProjectKeys, + TSecretApprovalRequestsSecrets, + TSecrets, + TSecretVersions +} from "@app/db/schemas"; import { decryptAsymmetric } from "../crypto"; -export enum SecretDocType { - Secret = "secret", - SecretVersion = "secretVersion", - ApprovalSecret = "approvalSecret" -} - -export interface TPartialSecret { - id: string; - secretKeyCiphertext: string; - secretKeyIV: string; - secretKeyTag: string; - - secretValueCiphertext: string; - secretValueIV: string; - secretValueTag: string; - - secretCommentCiphertext?: string | null; - secretCommentIV?: string | null; - secretCommentTag?: string | null; - - docType: SecretDocType; - keyEncoding: string; -} - -const PartialDecryptedSecretSchema = z.object({ +const DecryptedValuesSchema = z.object({ id: z.string(), secretKey: z.string(), secretValue: z.string(), - secretComment: z.string().optional(), - - docType: z.nativeEnum(SecretDocType) + secretComment: z.string().optional() }); -export type TPartialDecryptedSecret = z.infer; -const decryptSecret = ({ +const DecryptedSecretSchema = z.object({ + decrypted: DecryptedValuesSchema, + original: SecretsSchema +}); + +const DecryptedSecretVersionsSchema = z.object({ + decrypted: DecryptedValuesSchema, + original: SecretVersionsSchema +}); + +export const DecryptedSecretApprovalsSchema = z.object({ + decrypted: DecryptedValuesSchema, + original: SecretApprovalRequestsSecretsSchema +}); + +export type DecryptedSecret = z.infer; +export type DecryptedSecretVersions = z.infer; +export type DecryptedSecretApprovals = z.infer; + +const decryptCipher = ({ ciphertext, iv, tag, @@ -60,8 +59,62 @@ const decryptSecret = ({ return cleartext; }; +const getDecryptedValues = ({ + secretKeyCiphertext, + secretKeyIV, + secretKeyTag, + secretValueCiphertext, + secretValueIV, + secretValueTag, + + secretCommentCiphertext, + secretCommentIV, + secretCommentTag, + key +}: { + secretKeyCiphertext: string; + secretKeyIV: string; + secretKeyTag: string; + secretValueCiphertext: string; + secretValueIV: string; + secretValueTag: string; + secretCommentCiphertext?: string | null; + secretCommentIV?: string | null; + secretCommentTag?: string | null; + key: string | Buffer; +}) => { + const secretKey = decryptCipher({ + ciphertext: secretKeyCiphertext, + iv: secretKeyIV, + tag: secretKeyTag, + key + }); + + const secretValue = decryptCipher({ + ciphertext: secretValueCiphertext, + iv: secretValueIV, + tag: secretValueTag, + key + }); + + const secretComment = + secretCommentCiphertext && secretCommentIV && secretCommentTag + ? decryptCipher({ + ciphertext: secretCommentCiphertext, + iv: secretCommentIV, + tag: secretCommentTag, + key + }) + : ""; + + return { + secretKey, + secretValue, + secretComment + }; +}; export const decryptSecrets = ( - encryptedSecrets: TPartialSecret[], + encryptedSecrets: TSecrets[], privateKey: string, latestKey: TProjectKeys & { sender: { @@ -76,47 +129,123 @@ export const decryptSecrets = ( privateKey }); - const decryptedSecrets: TPartialDecryptedSecret[] = []; + const decryptedSecrets: DecryptedSecret[] = []; encryptedSecrets.forEach((encSecret) => { - try { - const secretKey = decryptSecret({ - ciphertext: encSecret.secretKeyCiphertext, - iv: encSecret.secretKeyIV, - tag: encSecret.secretKeyTag, - key - }); + const decrypted = getDecryptedValues({ + secretKeyCiphertext: encSecret.secretKeyCiphertext, + secretKeyIV: encSecret.secretKeyIV, + secretKeyTag: encSecret.secretKeyTag, + secretValueCiphertext: encSecret.secretValueCiphertext, + secretValueIV: encSecret.secretValueIV, + secretValueTag: encSecret.secretValueTag, + secretCommentCiphertext: encSecret.secretCommentCiphertext, + secretCommentIV: encSecret.secretCommentIV, + secretCommentTag: encSecret.secretCommentTag, + key + }); - const secretValue = decryptSecret({ - ciphertext: encSecret.secretValueCiphertext, - iv: encSecret.secretValueIV, - tag: encSecret.secretValueTag, - key - }); + const decryptedSecret: DecryptedSecret = { + decrypted: { + ...decrypted, + id: encSecret.id + }, + original: encSecret + }; - const secretComment = - encSecret.secretCommentCiphertext && encSecret.secretCommentIV && encSecret.secretCommentTag - ? decryptSecret({ - ciphertext: encSecret.secretCommentCiphertext, - iv: encSecret.secretCommentIV, - tag: encSecret.secretCommentTag, - key - }) - : ""; - - const decryptedSecret: TPartialDecryptedSecret = { - id: encSecret.id, - secretKey, - secretValue, - secretComment, - docType: encSecret.docType - }; - - decryptedSecrets.push(PartialDecryptedSecretSchema.parse(decryptedSecret)); - } catch (err) { - // This is ok, because we check that the decrypted secrets array length is the same as the encrypted secrets input array length. - logger.error(`[${encSecret.id}] - failed to decrypt`, err); - } + decryptedSecrets.push(DecryptedSecretSchema.parse(decryptedSecret)); + }); + + return decryptedSecrets; +}; + +export const decryptSecretVersions = ( + encryptedSecretVersions: TSecretVersions[], + privateKey: string, + latestKey: TProjectKeys & { + sender: { + publicKey: string; + }; + } +) => { + const key = decryptAsymmetric({ + ciphertext: latestKey.encryptedKey, + nonce: latestKey.nonce, + publicKey: latestKey.sender.publicKey, + privateKey + }); + + const decryptedSecrets: DecryptedSecretVersions[] = []; + + encryptedSecretVersions.forEach((encSecret) => { + const decrypted = getDecryptedValues({ + secretKeyCiphertext: encSecret.secretKeyCiphertext, + secretKeyIV: encSecret.secretKeyIV, + secretKeyTag: encSecret.secretKeyTag, + secretValueCiphertext: encSecret.secretValueCiphertext, + secretValueIV: encSecret.secretValueIV, + secretValueTag: encSecret.secretValueTag, + secretCommentCiphertext: encSecret.secretCommentCiphertext, + secretCommentIV: encSecret.secretCommentIV, + secretCommentTag: encSecret.secretCommentTag, + key + }); + + const decryptedSecret: DecryptedSecretVersions = { + decrypted: { + ...decrypted, + id: encSecret.id + }, + original: encSecret + }; + + decryptedSecrets.push(DecryptedSecretVersionsSchema.parse(decryptedSecret)); + }); + + return decryptedSecrets; +}; + +export const decryptSecretApprovals = ( + encryptedSecretApprovals: TSecretApprovalRequestsSecrets[], + privateKey: string, + latestKey: TProjectKeys & { + sender: { + publicKey: string; + }; + } +) => { + const key = decryptAsymmetric({ + ciphertext: latestKey.encryptedKey, + nonce: latestKey.nonce, + publicKey: latestKey.sender.publicKey, + privateKey + }); + + const decryptedSecrets: DecryptedSecretApprovals[] = []; + + encryptedSecretApprovals.forEach((encSecret) => { + const decrypted = getDecryptedValues({ + secretKeyCiphertext: encSecret.secretKeyCiphertext, + secretKeyIV: encSecret.secretKeyIV, + secretKeyTag: encSecret.secretKeyTag, + secretValueCiphertext: encSecret.secretValueCiphertext, + secretValueIV: encSecret.secretValueIV, + secretValueTag: encSecret.secretValueTag, + secretCommentCiphertext: encSecret.secretCommentCiphertext, + secretCommentIV: encSecret.secretCommentIV, + secretCommentTag: encSecret.secretCommentTag, + key + }); + + const decryptedSecret: DecryptedSecretApprovals = { + decrypted: { + ...decrypted, + id: encSecret.id + }, + original: encSecret + }; + + decryptedSecrets.push(DecryptedSecretApprovalsSchema.parse(decryptedSecret)); }); return decryptedSecrets; diff --git a/backend/src/services/project/project-queue.ts b/backend/src/services/project/project-queue.ts index 7bf9f0e51..15ee9abec 100644 --- a/backend/src/services/project/project-queue.ts +++ b/backend/src/services/project/project-queue.ts @@ -504,6 +504,21 @@ export const projectQueueFactory = ({ throw new Error("Parts of the upgrade failed. Some secrets were not updated"); } + const secretUpdates = await secretDAL.bulkUpdateNoVersionIncrement(updatedSecrets, tx); + const secretVersionUpdates = await secretVersionDAL.bulkUpdateNoVersionIncrement(updatedSecretVersions, tx); + const secretApprovalUpdates = await secretApprovalSecretDAL.bulkUpdateNoVersionIncrement( + updatedSecretApprovals, + tx + ); + + if ( + secretUpdates.length !== updatedSecrets.length || + secretVersionUpdates.length !== updatedSecretVersions.length || + secretApprovalUpdates.length !== updatedSecretApprovals.length + ) { + throw new Error("Parts of the upgrade failed. Some secrets were not updated"); + } + await projectDAL.setProjectUpgradeStatus(data.projectId, null, tx); // await new Promise((resolve) => setTimeout(resolve, 15_000)); From aa8171182498a97bc772269b5443443af3383d01 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 20 Feb 2024 02:34:30 +0100 Subject: [PATCH 143/157] Fixed integrations & bulk update issue --- backend/src/lib/secret/index.ts | 262 ++++++++++-------- backend/src/services/project/project-queue.ts | 19 +- 2 files changed, 156 insertions(+), 125 deletions(-) diff --git a/backend/src/lib/secret/index.ts b/backend/src/lib/secret/index.ts index 036b49970..dd23c318a 100644 --- a/backend/src/lib/secret/index.ts +++ b/backend/src/lib/secret/index.ts @@ -2,9 +2,11 @@ import crypto from "crypto"; import { z } from "zod"; import { + IntegrationAuthsSchema, SecretApprovalRequestsSecretsSchema, SecretsSchema, SecretVersionsSchema, + TIntegrationAuths, TProjectKeys, TSecretApprovalRequestsSecrets, TSecrets, @@ -25,6 +27,16 @@ const DecryptedSecretSchema = z.object({ original: SecretsSchema }); +const DecryptedIntegrationAuthsSchema = z.object({ + decrypted: z.object({ + id: z.string(), + access: z.string(), + accessId: z.string(), + refresh: z.string() + }), + original: IntegrationAuthsSchema +}); + const DecryptedSecretVersionsSchema = z.object({ decrypted: DecryptedValuesSchema, original: SecretVersionsSchema @@ -38,6 +50,13 @@ export const DecryptedSecretApprovalsSchema = z.object({ export type DecryptedSecret = z.infer; export type DecryptedSecretVersions = z.infer; export type DecryptedSecretApprovals = z.infer; +export type DecryptedIntegrationAuths = z.infer; + +type TLatestKey = TProjectKeys & { + sender: { + publicKey: string; + }; +}; const decryptCipher = ({ ciphertext, @@ -59,69 +78,20 @@ const decryptCipher = ({ return cleartext; }; -const getDecryptedValues = ({ - secretKeyCiphertext, - secretKeyIV, - secretKeyTag, - secretValueCiphertext, - secretValueIV, - secretValueTag, +const getDecryptedValues = (data: Array<{ ciphertext: string; iv: string; tag: string }>, key: string | Buffer) => { + const results = []; - secretCommentCiphertext, - secretCommentIV, - secretCommentTag, - key -}: { - secretKeyCiphertext: string; - secretKeyIV: string; - secretKeyTag: string; - secretValueCiphertext: string; - secretValueIV: string; - secretValueTag: string; - secretCommentCiphertext?: string | null; - secretCommentIV?: string | null; - secretCommentTag?: string | null; - key: string | Buffer; -}) => { - const secretKey = decryptCipher({ - ciphertext: secretKeyCiphertext, - iv: secretKeyIV, - tag: secretKeyTag, - key - }); - - const secretValue = decryptCipher({ - ciphertext: secretValueCiphertext, - iv: secretValueIV, - tag: secretValueTag, - key - }); - - const secretComment = - secretCommentCiphertext && secretCommentIV && secretCommentTag - ? decryptCipher({ - ciphertext: secretCommentCiphertext, - iv: secretCommentIV, - tag: secretCommentTag, - key - }) - : ""; - - return { - secretKey, - secretValue, - secretComment - }; -}; -export const decryptSecrets = ( - encryptedSecrets: TSecrets[], - privateKey: string, - latestKey: TProjectKeys & { - sender: { - publicKey: string; - }; + for (const { ciphertext, iv, tag } of data) { + if (!ciphertext || !iv || !tag) { + results.push(""); + } else { + results.push(decryptCipher({ ciphertext, iv, tag, key })); + } } -) => { + + return results; +}; +export const decryptSecrets = (encryptedSecrets: TSecrets[], privateKey: string, latestKey: TLatestKey) => { const key = decryptAsymmetric({ ciphertext: latestKey.encryptedKey, nonce: latestKey.nonce, @@ -132,22 +102,32 @@ export const decryptSecrets = ( const decryptedSecrets: DecryptedSecret[] = []; encryptedSecrets.forEach((encSecret) => { - const decrypted = getDecryptedValues({ - secretKeyCiphertext: encSecret.secretKeyCiphertext, - secretKeyIV: encSecret.secretKeyIV, - secretKeyTag: encSecret.secretKeyTag, - secretValueCiphertext: encSecret.secretValueCiphertext, - secretValueIV: encSecret.secretValueIV, - secretValueTag: encSecret.secretValueTag, - secretCommentCiphertext: encSecret.secretCommentCiphertext, - secretCommentIV: encSecret.secretCommentIV, - secretCommentTag: encSecret.secretCommentTag, + const [secretKey, secretValue, secretComment] = getDecryptedValues( + [ + { + ciphertext: encSecret.secretKeyCiphertext, + iv: encSecret.secretKeyIV, + tag: encSecret.secretKeyTag + }, + { + ciphertext: encSecret.secretValueCiphertext, + iv: encSecret.secretValueIV, + tag: encSecret.secretValueTag + }, + { + ciphertext: encSecret.secretCommentCiphertext || "", + iv: encSecret.secretCommentIV || "", + tag: encSecret.secretCommentTag || "" + } + ], key - }); + ); const decryptedSecret: DecryptedSecret = { decrypted: { - ...decrypted, + secretKey, + secretValue, + secretComment, id: encSecret.id }, original: encSecret @@ -162,11 +142,7 @@ export const decryptSecrets = ( export const decryptSecretVersions = ( encryptedSecretVersions: TSecretVersions[], privateKey: string, - latestKey: TProjectKeys & { - sender: { - publicKey: string; - }; - } + latestKey: TLatestKey ) => { const key = decryptAsymmetric({ ciphertext: latestKey.encryptedKey, @@ -178,22 +154,32 @@ export const decryptSecretVersions = ( const decryptedSecrets: DecryptedSecretVersions[] = []; encryptedSecretVersions.forEach((encSecret) => { - const decrypted = getDecryptedValues({ - secretKeyCiphertext: encSecret.secretKeyCiphertext, - secretKeyIV: encSecret.secretKeyIV, - secretKeyTag: encSecret.secretKeyTag, - secretValueCiphertext: encSecret.secretValueCiphertext, - secretValueIV: encSecret.secretValueIV, - secretValueTag: encSecret.secretValueTag, - secretCommentCiphertext: encSecret.secretCommentCiphertext, - secretCommentIV: encSecret.secretCommentIV, - secretCommentTag: encSecret.secretCommentTag, + const [secretKey, secretValue, secretComment] = getDecryptedValues( + [ + { + ciphertext: encSecret.secretKeyCiphertext, + iv: encSecret.secretKeyIV, + tag: encSecret.secretKeyTag + }, + { + ciphertext: encSecret.secretValueCiphertext, + iv: encSecret.secretValueIV, + tag: encSecret.secretValueTag + }, + { + ciphertext: encSecret.secretCommentCiphertext || "", + iv: encSecret.secretCommentIV || "", + tag: encSecret.secretCommentTag || "" + } + ], key - }); + ); const decryptedSecret: DecryptedSecretVersions = { decrypted: { - ...decrypted, + secretKey, + secretValue, + secretComment, id: encSecret.id }, original: encSecret @@ -208,11 +194,7 @@ export const decryptSecretVersions = ( export const decryptSecretApprovals = ( encryptedSecretApprovals: TSecretApprovalRequestsSecrets[], privateKey: string, - latestKey: TProjectKeys & { - sender: { - publicKey: string; - }; - } + latestKey: TLatestKey ) => { const key = decryptAsymmetric({ ciphertext: latestKey.encryptedKey, @@ -223,26 +205,36 @@ export const decryptSecretApprovals = ( const decryptedSecrets: DecryptedSecretApprovals[] = []; - encryptedSecretApprovals.forEach((encSecret) => { - const decrypted = getDecryptedValues({ - secretKeyCiphertext: encSecret.secretKeyCiphertext, - secretKeyIV: encSecret.secretKeyIV, - secretKeyTag: encSecret.secretKeyTag, - secretValueCiphertext: encSecret.secretValueCiphertext, - secretValueIV: encSecret.secretValueIV, - secretValueTag: encSecret.secretValueTag, - secretCommentCiphertext: encSecret.secretCommentCiphertext, - secretCommentIV: encSecret.secretCommentIV, - secretCommentTag: encSecret.secretCommentTag, + encryptedSecretApprovals.forEach((encApproval) => { + const [secretKey, secretValue, secretComment] = getDecryptedValues( + [ + { + ciphertext: encApproval.secretKeyCiphertext, + iv: encApproval.secretKeyIV, + tag: encApproval.secretKeyTag + }, + { + ciphertext: encApproval.secretValueCiphertext, + iv: encApproval.secretValueIV, + tag: encApproval.secretValueTag + }, + { + ciphertext: encApproval.secretCommentCiphertext || "", + iv: encApproval.secretCommentIV || "", + tag: encApproval.secretCommentTag || "" + } + ], key - }); + ); const decryptedSecret: DecryptedSecretApprovals = { decrypted: { - ...decrypted, - id: encSecret.id + secretKey, + secretValue, + secretComment, + id: encApproval.id }, - original: encSecret + original: encApproval }; decryptedSecrets.push(DecryptedSecretApprovalsSchema.parse(decryptedSecret)); @@ -250,3 +242,53 @@ export const decryptSecretApprovals = ( return decryptedSecrets; }; + +export const decryptIntegrationAuths = ( + encryptedIntegrationAuths: TIntegrationAuths[], + privateKey: string, + latestKey: TLatestKey +) => { + const key = decryptAsymmetric({ + ciphertext: latestKey.encryptedKey, + nonce: latestKey.nonce, + publicKey: latestKey.sender.publicKey, + privateKey + }); + + const decryptedIntegrationAuths: DecryptedIntegrationAuths[] = []; + + encryptedIntegrationAuths.forEach((encAuth) => { + const [access, accessId, refresh] = getDecryptedValues( + [ + { + ciphertext: encAuth.accessCiphertext || "", + iv: encAuth.accessIV || "", + tag: encAuth.accessTag || "" + }, + { + ciphertext: encAuth.accessIdCiphertext || "", + iv: encAuth.accessIdIV || "", + tag: encAuth.accessIdTag || "" + }, + { + ciphertext: encAuth.refreshCiphertext || "", + iv: encAuth.refreshIV || "", + tag: encAuth.refreshTag || "" + } + ], + key + ); + + decryptedIntegrationAuths.push({ + decrypted: { + id: encAuth.id, + access, + accessId, + refresh + }, + original: encAuth + }); + }); + + return decryptedIntegrationAuths; +}; diff --git a/backend/src/services/project/project-queue.ts b/backend/src/services/project/project-queue.ts index 15ee9abec..57bd46de7 100644 --- a/backend/src/services/project/project-queue.ts +++ b/backend/src/services/project/project-queue.ts @@ -178,6 +178,10 @@ export const projectQueueFactory = ({ approvalSecrets.push(...secretApprovals); } + const projectIntegrationAuths = await integrationAuthDAL.find({ + projectId: project.id + }); + const decryptedSecrets = decryptSecrets(secrets, userPrivateKey, oldProjectKey); const decryptedSecretVersions = decryptSecretVersions(secretVersions, userPrivateKey, oldProjectKey); const decryptedApprovalSecrets = decryptSecretApprovals(approvalSecrets, userPrivateKey, oldProjectKey); @@ -504,21 +508,6 @@ export const projectQueueFactory = ({ throw new Error("Parts of the upgrade failed. Some secrets were not updated"); } - const secretUpdates = await secretDAL.bulkUpdateNoVersionIncrement(updatedSecrets, tx); - const secretVersionUpdates = await secretVersionDAL.bulkUpdateNoVersionIncrement(updatedSecretVersions, tx); - const secretApprovalUpdates = await secretApprovalSecretDAL.bulkUpdateNoVersionIncrement( - updatedSecretApprovals, - tx - ); - - if ( - secretUpdates.length !== updatedSecrets.length || - secretVersionUpdates.length !== updatedSecretVersions.length || - secretApprovalUpdates.length !== updatedSecretApprovals.length - ) { - throw new Error("Parts of the upgrade failed. Some secrets were not updated"); - } - await projectDAL.setProjectUpgradeStatus(data.projectId, null, tx); // await new Promise((resolve) => setTimeout(resolve, 15_000)); From cab47d0b9815175a8a655c5919a6dcc3e7d42023 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 20 Feb 2024 03:23:28 +0100 Subject: [PATCH 144/157] Update project-queue.ts --- backend/src/services/project/project-queue.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/services/project/project-queue.ts b/backend/src/services/project/project-queue.ts index 57bd46de7..8f4e7f876 100644 --- a/backend/src/services/project/project-queue.ts +++ b/backend/src/services/project/project-queue.ts @@ -137,7 +137,6 @@ export const projectQueueFactory = ({ const secretVersions: TSecretVersions[] = []; const approvalSecrets: TSecretApprovalRequestsSecrets[] = []; const folderSecretVersionIdsToDelete: string[] = []; - for (const folder of projectFolders) { const folderSecrets = await secretDAL.find({ folderId: folder.id }); From 8be976a694534b1e83d04219ad492b5aa78402a9 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 02:49:57 +0100 Subject: [PATCH 145/157] Get last 700 secret versions instead of 100 --- backend/src/services/project/project-queue.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/backend/src/services/project/project-queue.ts b/backend/src/services/project/project-queue.ts index 8f4e7f876..7bf9f0e51 100644 --- a/backend/src/services/project/project-queue.ts +++ b/backend/src/services/project/project-queue.ts @@ -137,6 +137,7 @@ export const projectQueueFactory = ({ const secretVersions: TSecretVersions[] = []; const approvalSecrets: TSecretApprovalRequestsSecrets[] = []; const folderSecretVersionIdsToDelete: string[] = []; + for (const folder of projectFolders) { const folderSecrets = await secretDAL.find({ folderId: folder.id }); @@ -177,10 +178,6 @@ export const projectQueueFactory = ({ approvalSecrets.push(...secretApprovals); } - const projectIntegrationAuths = await integrationAuthDAL.find({ - projectId: project.id - }); - const decryptedSecrets = decryptSecrets(secrets, userPrivateKey, oldProjectKey); const decryptedSecretVersions = decryptSecretVersions(secretVersions, userPrivateKey, oldProjectKey); const decryptedApprovalSecrets = decryptSecretApprovals(approvalSecrets, userPrivateKey, oldProjectKey); From 368974cf0189bdcf95e397a3d740e8688d1da736 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 03:04:23 +0100 Subject: [PATCH 146/157] Moved --- backend/src/lib/secret/index.ts | 294 -------------------------------- 1 file changed, 294 deletions(-) delete mode 100644 backend/src/lib/secret/index.ts diff --git a/backend/src/lib/secret/index.ts b/backend/src/lib/secret/index.ts deleted file mode 100644 index dd23c318a..000000000 --- a/backend/src/lib/secret/index.ts +++ /dev/null @@ -1,294 +0,0 @@ -import crypto from "crypto"; -import { z } from "zod"; - -import { - IntegrationAuthsSchema, - SecretApprovalRequestsSecretsSchema, - SecretsSchema, - SecretVersionsSchema, - TIntegrationAuths, - TProjectKeys, - TSecretApprovalRequestsSecrets, - TSecrets, - TSecretVersions -} from "@app/db/schemas"; - -import { decryptAsymmetric } from "../crypto"; - -const DecryptedValuesSchema = z.object({ - id: z.string(), - secretKey: z.string(), - secretValue: z.string(), - secretComment: z.string().optional() -}); - -const DecryptedSecretSchema = z.object({ - decrypted: DecryptedValuesSchema, - original: SecretsSchema -}); - -const DecryptedIntegrationAuthsSchema = z.object({ - decrypted: z.object({ - id: z.string(), - access: z.string(), - accessId: z.string(), - refresh: z.string() - }), - original: IntegrationAuthsSchema -}); - -const DecryptedSecretVersionsSchema = z.object({ - decrypted: DecryptedValuesSchema, - original: SecretVersionsSchema -}); - -export const DecryptedSecretApprovalsSchema = z.object({ - decrypted: DecryptedValuesSchema, - original: SecretApprovalRequestsSecretsSchema -}); - -export type DecryptedSecret = z.infer; -export type DecryptedSecretVersions = z.infer; -export type DecryptedSecretApprovals = z.infer; -export type DecryptedIntegrationAuths = z.infer; - -type TLatestKey = TProjectKeys & { - sender: { - publicKey: string; - }; -}; - -const decryptCipher = ({ - ciphertext, - iv, - tag, - key -}: { - ciphertext: string; - iv: string; - tag: string; - key: string | Buffer; -}) => { - const decipher = crypto.createDecipheriv("aes-256-gcm", key, Buffer.from(iv, "base64")); - decipher.setAuthTag(Buffer.from(tag, "base64")); - - let cleartext = decipher.update(ciphertext, "base64", "utf8"); - cleartext += decipher.final("utf8"); - - return cleartext; -}; - -const getDecryptedValues = (data: Array<{ ciphertext: string; iv: string; tag: string }>, key: string | Buffer) => { - const results = []; - - for (const { ciphertext, iv, tag } of data) { - if (!ciphertext || !iv || !tag) { - results.push(""); - } else { - results.push(decryptCipher({ ciphertext, iv, tag, key })); - } - } - - return results; -}; -export const decryptSecrets = (encryptedSecrets: TSecrets[], privateKey: string, latestKey: TLatestKey) => { - const key = decryptAsymmetric({ - ciphertext: latestKey.encryptedKey, - nonce: latestKey.nonce, - publicKey: latestKey.sender.publicKey, - privateKey - }); - - const decryptedSecrets: DecryptedSecret[] = []; - - encryptedSecrets.forEach((encSecret) => { - const [secretKey, secretValue, secretComment] = getDecryptedValues( - [ - { - ciphertext: encSecret.secretKeyCiphertext, - iv: encSecret.secretKeyIV, - tag: encSecret.secretKeyTag - }, - { - ciphertext: encSecret.secretValueCiphertext, - iv: encSecret.secretValueIV, - tag: encSecret.secretValueTag - }, - { - ciphertext: encSecret.secretCommentCiphertext || "", - iv: encSecret.secretCommentIV || "", - tag: encSecret.secretCommentTag || "" - } - ], - key - ); - - const decryptedSecret: DecryptedSecret = { - decrypted: { - secretKey, - secretValue, - secretComment, - id: encSecret.id - }, - original: encSecret - }; - - decryptedSecrets.push(DecryptedSecretSchema.parse(decryptedSecret)); - }); - - return decryptedSecrets; -}; - -export const decryptSecretVersions = ( - encryptedSecretVersions: TSecretVersions[], - privateKey: string, - latestKey: TLatestKey -) => { - const key = decryptAsymmetric({ - ciphertext: latestKey.encryptedKey, - nonce: latestKey.nonce, - publicKey: latestKey.sender.publicKey, - privateKey - }); - - const decryptedSecrets: DecryptedSecretVersions[] = []; - - encryptedSecretVersions.forEach((encSecret) => { - const [secretKey, secretValue, secretComment] = getDecryptedValues( - [ - { - ciphertext: encSecret.secretKeyCiphertext, - iv: encSecret.secretKeyIV, - tag: encSecret.secretKeyTag - }, - { - ciphertext: encSecret.secretValueCiphertext, - iv: encSecret.secretValueIV, - tag: encSecret.secretValueTag - }, - { - ciphertext: encSecret.secretCommentCiphertext || "", - iv: encSecret.secretCommentIV || "", - tag: encSecret.secretCommentTag || "" - } - ], - key - ); - - const decryptedSecret: DecryptedSecretVersions = { - decrypted: { - secretKey, - secretValue, - secretComment, - id: encSecret.id - }, - original: encSecret - }; - - decryptedSecrets.push(DecryptedSecretVersionsSchema.parse(decryptedSecret)); - }); - - return decryptedSecrets; -}; - -export const decryptSecretApprovals = ( - encryptedSecretApprovals: TSecretApprovalRequestsSecrets[], - privateKey: string, - latestKey: TLatestKey -) => { - const key = decryptAsymmetric({ - ciphertext: latestKey.encryptedKey, - nonce: latestKey.nonce, - publicKey: latestKey.sender.publicKey, - privateKey - }); - - const decryptedSecrets: DecryptedSecretApprovals[] = []; - - encryptedSecretApprovals.forEach((encApproval) => { - const [secretKey, secretValue, secretComment] = getDecryptedValues( - [ - { - ciphertext: encApproval.secretKeyCiphertext, - iv: encApproval.secretKeyIV, - tag: encApproval.secretKeyTag - }, - { - ciphertext: encApproval.secretValueCiphertext, - iv: encApproval.secretValueIV, - tag: encApproval.secretValueTag - }, - { - ciphertext: encApproval.secretCommentCiphertext || "", - iv: encApproval.secretCommentIV || "", - tag: encApproval.secretCommentTag || "" - } - ], - key - ); - - const decryptedSecret: DecryptedSecretApprovals = { - decrypted: { - secretKey, - secretValue, - secretComment, - id: encApproval.id - }, - original: encApproval - }; - - decryptedSecrets.push(DecryptedSecretApprovalsSchema.parse(decryptedSecret)); - }); - - return decryptedSecrets; -}; - -export const decryptIntegrationAuths = ( - encryptedIntegrationAuths: TIntegrationAuths[], - privateKey: string, - latestKey: TLatestKey -) => { - const key = decryptAsymmetric({ - ciphertext: latestKey.encryptedKey, - nonce: latestKey.nonce, - publicKey: latestKey.sender.publicKey, - privateKey - }); - - const decryptedIntegrationAuths: DecryptedIntegrationAuths[] = []; - - encryptedIntegrationAuths.forEach((encAuth) => { - const [access, accessId, refresh] = getDecryptedValues( - [ - { - ciphertext: encAuth.accessCiphertext || "", - iv: encAuth.accessIV || "", - tag: encAuth.accessTag || "" - }, - { - ciphertext: encAuth.accessIdCiphertext || "", - iv: encAuth.accessIdIV || "", - tag: encAuth.accessIdTag || "" - }, - { - ciphertext: encAuth.refreshCiphertext || "", - iv: encAuth.refreshIV || "", - tag: encAuth.refreshTag || "" - } - ], - key - ); - - decryptedIntegrationAuths.push({ - decrypted: { - id: encAuth.id, - access, - accessId, - refresh - }, - original: encAuth - }); - }); - - return decryptedIntegrationAuths; -}; From 2cbd2ee75f8dcff708e40ed00c5b7a378d0245ab Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 08:54:11 +0100 Subject: [PATCH 147/157] Update project-router.ts --- backend/src/server/routes/v2/project-router.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/server/routes/v2/project-router.ts b/backend/src/server/routes/v2/project-router.ts index faeb89f4c..7cbcff37c 100644 --- a/backend/src/server/routes/v2/project-router.ts +++ b/backend/src/server/routes/v2/project-router.ts @@ -17,7 +17,7 @@ const projectWithEnv = ProjectsSchema.merge( export const registerProjectRouter = async (server: FastifyZodProvider) => { /* Get project key */ server.route({ - url: "/:projectId/encrypted-key", + url: "/:workspaceId/encrypted-key", method: "GET", schema: { description: "Return encrypted project key", @@ -27,7 +27,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { } ], params: z.object({ - projectId: z.string().trim() + workspaceId: z.string().trim() }), response: { 200: ProjectKeysSchema.merge( @@ -50,7 +50,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { await server.services.auditLog.createAuditLog({ ...req.auditLogInfo, - projectId: req.params.projectId, + projectId: req.params.workspaceId, event: { type: EventType.GET_WORKSPACE_KEY, metadata: { From 9f0d09f8ed7fd4aef63304e53597291a5faad201 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 18:20:15 +0100 Subject: [PATCH 148/157] Update seed-data.ts --- backend/src/db/seed-data.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/backend/src/db/seed-data.ts b/backend/src/db/seed-data.ts index e484b09b1..6406e44ab 100644 --- a/backend/src/db/seed-data.ts +++ b/backend/src/db/seed-data.ts @@ -19,9 +19,6 @@ import { TSecrets, TUserEncryptionKeys } from "./schemas"; export let userPrivateKey: string | undefined; export let userPublicKey: string | undefined; -export let userPrivateKey: string | undefined; -export let userPublicKey: string | undefined; - export const seedData1 = { id: "3dafd81d-4388-432b-a4c5-f735616868c1", email: process.env.TEST_USER_EMAIL || "test@localhost.local", From 816c793ae3d0e9df1a0ed1852627074d4fe87268 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 21 Feb 2024 20:56:37 +0100 Subject: [PATCH 149/157] Update index.tsx --- frontend/src/pages/org/[id]/overview/index.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/pages/org/[id]/overview/index.tsx b/frontend/src/pages/org/[id]/overview/index.tsx index 10d0924f3..8a703b3dc 100644 --- a/frontend/src/pages/org/[id]/overview/index.tsx +++ b/frontend/src/pages/org/[id]/overview/index.tsx @@ -523,7 +523,6 @@ const OrganizationPage = withPermission( // eslint-disable-next-line no-promise-executor-return -- We do this because the function returns too fast, which sometimes causes an error when the user is redirected. await new Promise((resolve) => setTimeout(resolve, 2_000)); - createNotification({ text: "Workspace created", type: "success" }); handlePopUpClose("addNewWs"); createNotification({ text: "Workspace created", type: "success" }); router.push(`/project/${newProjectId}/secrets/overview`); From 7816d8593e0cf16ec596ee891cb34d3a426d03a4 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Thu, 22 Feb 2024 04:06:50 +0100 Subject: [PATCH 150/157] Final changes --- backend/src/services/org/org-dal.ts | 10 ++++ backend/src/services/project/project-queue.ts | 3 ++ .../src/services/project/project-service.ts | 23 ++++++---- .../v2/UpgradeOverlay/UpgradeOverlay.tsx | 46 +++++++++++++++++++ .../components/v2/UpgradeOverlay/index.tsx | 1 + .../UpgradeProjectAlert.tsx | 2 +- frontend/src/layouts/AppLayout/AppLayout.tsx | 4 +- .../SecretOverviewPage/SecretOverviewPage.tsx | 6 ++- 8 files changed, 83 insertions(+), 12 deletions(-) create mode 100644 frontend/src/components/v2/UpgradeOverlay/UpgradeOverlay.tsx create mode 100644 frontend/src/components/v2/UpgradeOverlay/index.tsx diff --git a/backend/src/services/org/org-dal.ts b/backend/src/services/org/org-dal.ts index 05c77e746..3fbf54b4c 100644 --- a/backend/src/services/org/org-dal.ts +++ b/backend/src/services/org/org-dal.ts @@ -230,6 +230,15 @@ export const orgDALFactory = (db: TDbClient) => { } }; + const deleteMembershipByUserId = async (userId: string, orgId: string, tx?: Knex) => { + try { + const [membership] = await (tx || db)(TableName.OrgMembership).where({ userId, orgId }).delete().returning("*"); + return membership; + } catch (error) { + throw new DatabaseError({ error, name: "Delete org membership" }); + } + }; + const findMembership = async ( filter: TFindFilter, { offset, limit, sort, tx }: TFindOpt = {} @@ -275,6 +284,7 @@ export const orgDALFactory = (db: TDbClient) => { createMembership, updateMembershipById, deleteMembershipById, + deleteMembershipByUserId, updateMembership }); }; diff --git a/backend/src/services/project/project-queue.ts b/backend/src/services/project/project-queue.ts index 7bf9f0e51..3a76d9c65 100644 --- a/backend/src/services/project/project-queue.ts +++ b/backend/src/services/project/project-queue.ts @@ -112,6 +112,9 @@ export const projectQueueFactory = ({ await projectDAL.setProjectUpgradeStatus(data.projectId, ProjectUpgradeStatus.InProgress); // Set the status to in progress. This is important to prevent multiple upgrades at the same time. + // eslint-disable-next-line no-promise-executor-return + // await new Promise((resolve) => setTimeout(resolve, 50_000)); + const userPrivateKey = infisicalSymmetricDecrypt({ keyEncoding: data.encryptedPrivateKey.keyEncoding, ciphertext: data.encryptedPrivateKey.encryptedKey, diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index 5826e77f0..d5fa45bfa 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -17,6 +17,7 @@ import { TProjectPermission } from "@app/lib/types"; import { ActorType } from "../auth/auth-type"; import { TIdentityOrgDALFactory } from "../identity/identity-org-dal"; import { TIdentityProjectDALFactory } from "../identity-project/identity-project-dal"; +import { TOrgDALFactory } from "../org/org-dal"; import { TOrgServiceFactory } from "../org/org-service"; import { TProjectBotDALFactory } from "../project-bot/project-bot-dal"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; @@ -53,6 +54,7 @@ type TProjectServiceFactoryDep = { projectKeyDAL: Pick; projectBotDAL: Pick; projectMembershipDAL: Pick; + orgDAL: Pick; secretBlindIndexDAL: Pick; permissionService: TPermissionServiceFactory; orgService: Pick; @@ -68,6 +70,7 @@ export const projectServiceFactory = ({ permissionService, userDAL, folderDAL, + orgDAL, orgService, identityProjectDAL, projectBotDAL, @@ -287,17 +290,22 @@ export const projectServiceFactory = ({ return results; }; - const findProjectGhostUser = async (projectId: string) => { - const user = await projectMembershipDAL.findProjectGhostUser(projectId); - - return user; - }; - const deleteProject = async ({ actor, actorId, actorOrgId, projectId }: TDeleteProjectDTO) => { const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId, actorOrgId); ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Delete, ProjectPermissionSub.Project); - const deletedProject = await projectDAL.deleteById(projectId); + const deletedProject = await projectDAL.transaction(async (tx) => { + const project = await projectDAL.deleteById(projectId, tx); + const projectGhostUser = await projectMembershipDAL.findProjectGhostUser(projectId).catch(() => null); + + // Delete the org membership for the ghost user if it's found. + if (projectGhostUser) { + await orgDAL.deleteMembershipByUserId(projectGhostUser.id, deletedProject.orgId, tx); + } + + return project; + }); + return deletedProject; }; @@ -389,7 +397,6 @@ export const projectServiceFactory = ({ deleteProject, getProjects, updateProject, - findProjectGhostUser, getProjectUpgradeStatus, getAProject, toggleAutoCapitalization, diff --git a/frontend/src/components/v2/UpgradeOverlay/UpgradeOverlay.tsx b/frontend/src/components/v2/UpgradeOverlay/UpgradeOverlay.tsx new file mode 100644 index 000000000..aea5bb582 --- /dev/null +++ b/frontend/src/components/v2/UpgradeOverlay/UpgradeOverlay.tsx @@ -0,0 +1,46 @@ +import { useRouter } from "next/router"; + +import { Spinner } from "@app/components/v2"; +import { useWorkspace } from "@app/context"; +import { useToggle } from "@app/hooks"; +import { useGetUpgradeProjectStatus } from "@app/hooks/api/workspace/queries"; +import { ProjectVersion } from "@app/hooks/api/workspace/types"; + +export const UpgradeOverlay = () => { + const router = useRouter(); + const { currentWorkspace } = useWorkspace(); + const [isUpgrading, setIsUpgrading] = useToggle(false); + const { isLoading: isUpgradeStatusLoading } = useGetUpgradeProjectStatus({ + projectId: currentWorkspace?.id ?? "", + enabled: currentWorkspace?.version === ProjectVersion.V1, + refetchInterval: 5_000, + onSuccess: (data) => { + if (!data) return; + + if (data.status !== "IN_PROGRESS") { + setIsUpgrading.off(); + } else if (data?.status === "IN_PROGRESS") { + setIsUpgrading.on(); + } + } + }); + + // make sure only to display this on /project routes + if (!currentWorkspace || !router.pathname.includes("/project")) { + return null; + } + + // for non admin this would throw an error + // so no need to render + return !isUpgradeStatusLoading && isUpgrading ? ( // isUpgrading +
+ +
+
Please wait
+ Upgrading your project... +
+
+ ) : ( +
+ ); +}; diff --git a/frontend/src/components/v2/UpgradeOverlay/index.tsx b/frontend/src/components/v2/UpgradeOverlay/index.tsx new file mode 100644 index 000000000..1a74fb6f1 --- /dev/null +++ b/frontend/src/components/v2/UpgradeOverlay/index.tsx @@ -0,0 +1 @@ +export { UpgradeOverlay } from "./UpgradeOverlay"; diff --git a/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx b/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx index ec3bdd9e4..db5eb1179 100644 --- a/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx +++ b/frontend/src/components/v2/UpgradeProjectAlert/UpgradeProjectAlert.tsx @@ -29,7 +29,7 @@ export const UpgradeProjectAlert = ({ project }: UpgradeProjectAlertProps): JSX. refetch: manualProjectStatusRefetch } = useGetUpgradeProjectStatus({ projectId: project.id, - enabled: membership.role === "admin", + enabled: membership.role === "admin" && project.version === ProjectVersion.V1, refetchInterval: 5_000, onSuccess: (data) => { if (membership.role !== "admin") { diff --git a/frontend/src/layouts/AppLayout/AppLayout.tsx b/frontend/src/layouts/AppLayout/AppLayout.tsx index 9dd2d82e7..295369bc4 100644 --- a/frontend/src/layouts/AppLayout/AppLayout.tsx +++ b/frontend/src/layouts/AppLayout/AppLayout.tsx @@ -49,6 +49,7 @@ import { SelectItem, UpgradePlanModal } from "@app/components/v2"; +import { UpgradeOverlay } from "@app/components/v2/UpgradeOverlay"; import { OrgPermissionActions, OrgPermissionSubjects, @@ -238,7 +239,7 @@ export const AppLayout = ({ children }: LayoutProps) => { projectId: newProjectId }); } - + // eslint-disable-next-line no-promise-executor-return -- We do this because the function returns too fast, which sometimes causes an error when the user is redirected. await new Promise((resolve) => setTimeout(resolve, 2_000)); @@ -261,6 +262,7 @@ export const AppLayout = ({ children }: LayoutProps) => {