From c00f6601bde723027193eab62d9c59eff6fee4c6 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Wed, 11 Dec 2024 05:48:45 +0400 Subject: [PATCH 1/7] fix(secrets-api): deletion of secret reminders on secret delete --- backend/src/services/secret/secret-service.ts | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 6f058e023..fbf90a7f8 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -491,8 +491,8 @@ export const secretServiceFactory = ({ secretDAL }); - const deletedSecret = await secretDAL.transaction(async (tx) => - fnSecretBulkDelete({ + const deletedSecret = await secretDAL.transaction(async (tx) => { + const secrets = await fnSecretBulkDelete({ projectId, folderId, actorId, @@ -505,8 +505,19 @@ export const secretServiceFactory = ({ } ], tx - }) - ); + }); + + for await (const secret of secrets) { + if (secret.secretReminderRepeatDays !== null && secret.secretReminderRepeatDays !== undefined) { + await secretQueueService.removeSecretReminder({ + repeatDays: secret.secretReminderRepeatDays, + secretId: secret.id + }); + } + } + + return secrets; + }); if (inputSecret.type === SecretType.Shared) { await snapshotService.performSnapshot(folderId); @@ -971,8 +982,8 @@ export const secretServiceFactory = ({ secretDAL }); - const secretsDeleted = await secretDAL.transaction(async (tx) => - fnSecretBulkDelete({ + const secretsDeleted = await secretDAL.transaction(async (tx) => { + const secrets = await fnSecretBulkDelete({ secretDAL, secretQueueService, inputSecrets: inputSecrets.map(({ type, secretName }) => ({ @@ -983,8 +994,19 @@ export const secretServiceFactory = ({ folderId, actorId, tx - }) - ); + }); + + for await (const secret of secrets) { + if (secret.secretReminderRepeatDays !== null && secret.secretReminderRepeatDays !== undefined) { + await secretQueueService.removeSecretReminder({ + repeatDays: secret.secretReminderRepeatDays, + secretId: secret.id + }); + } + } + + return secrets; + }); await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ From 181ba75f2a800b880ca8c46b24ceceecf4c3c05c Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Wed, 11 Dec 2024 08:15:29 +0400 Subject: [PATCH 2/7] fix(dashboard): creation of new org when user is apart of no orgs --- .../server/routes/v2/organization-router.ts | 33 ++++++++--- .../src/services/auth-token/auth-token-dal.ts | 5 +- .../services/auth-token/auth-token-service.ts | 12 ++-- .../src/services/auth/auth-login-service.ts | 57 +++++++++++-------- frontend/src/hooks/api/auth/queries.tsx | 9 ++- .../src/hooks/api/organization/queries.tsx | 10 +++- frontend/src/views/Login/Login.utils.tsx | 2 +- .../views/Org/components/CreateOrgModal.tsx | 15 +++-- 8 files changed, 89 insertions(+), 54 deletions(-) diff --git a/backend/src/server/routes/v2/organization-router.ts b/backend/src/server/routes/v2/organization-router.ts index cb630b143..8ca105ad4 100644 --- a/backend/src/server/routes/v2/organization-router.ts +++ b/backend/src/server/routes/v2/organization-router.ts @@ -10,6 +10,7 @@ import { UsersSchema } from "@app/db/schemas"; import { ORGANIZATIONS } from "@app/lib/api-docs"; +import { getConfig } from "@app/lib/config/env"; import { readLimit, writeLimit } from "@app/server/config/rateLimiter"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { ActorType, AuthMode } from "@app/services/auth/auth-type"; @@ -363,21 +364,35 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => { }), response: { 200: z.object({ - organization: OrganizationsSchema + organization: OrganizationsSchema, + accessToken: z.string() }) } }, onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY]), - handler: async (req) => { + handler: async (req, res) => { if (req.auth.actor !== ActorType.USER) return; - const organization = await server.services.org.deleteOrganizationById( - req.permission.id, - req.params.organizationId, - req.permission.authMethod, - req.permission.orgId - ); - return { organization }; + const cfg = getConfig(); + + const { organization, tokens } = await server.services.org.deleteOrganizationById({ + userId: req.permission.id, + orgId: req.params.organizationId, + actorAuthMethod: req.permission.authMethod, + actorOrgId: req.permission.orgId, + authorizationHeader: req.headers.authorization, + userAgentHeader: req.headers["user-agent"], + ipAddress: req.realIp + }); + + void res.setCookie("jid", tokens.refreshToken, { + httpOnly: true, + path: "/", + sameSite: "strict", + secure: cfg.HTTPS_ENABLED + }); + + return { organization, accessToken: tokens.accessToken }; } }); }; diff --git a/backend/src/services/auth-token/auth-token-dal.ts b/backend/src/services/auth-token/auth-token-dal.ts index c058c13e8..ed45d505e 100644 --- a/backend/src/services/auth-token/auth-token-dal.ts +++ b/backend/src/services/auth-token/auth-token-dal.ts @@ -54,10 +54,11 @@ export const tokenDALFactory = (db: TDbClient) => { const insertTokenSession = async ( userId: string, ip: string, - userAgent: string + userAgent: string, + tx?: Knex ): Promise => { try { - const [session] = await db(TableName.AuthTokenSession) + const [session] = await (tx || db)(TableName.AuthTokenSession) .insert({ userId, ip, diff --git a/backend/src/services/auth-token/auth-token-service.ts b/backend/src/services/auth-token/auth-token-service.ts index 321abb5b3..b9fc98106 100644 --- a/backend/src/services/auth-token/auth-token-service.ts +++ b/backend/src/services/auth-token/auth-token-service.ts @@ -1,6 +1,7 @@ import crypto from "node:crypto"; import bcrypt from "bcrypt"; +import { Knex } from "knex"; import { TAuthTokens, TAuthTokenSessions } from "@app/db/schemas"; import { getConfig } from "@app/lib/config/env"; @@ -123,14 +124,13 @@ export const tokenServiceFactory = ({ tokenDAL, userDAL, orgMembershipDAL }: TAu return deletedToken?.[0]; }; - const getUserTokenSession = async ({ - userId, - ip, - userAgent - }: TIssueAuthTokenDTO): Promise => { + const getUserTokenSession = async ( + { userId, ip, userAgent }: TIssueAuthTokenDTO, + tx?: Knex + ): Promise => { let session = await tokenDAL.findOneTokenSession({ userId, ip, userAgent }); if (!session) { - session = await tokenDAL.insertTokenSession(userId, ip, userAgent); + session = await tokenDAL.insertTokenSession(userId, ip, userAgent, tx); } return session; }; diff --git a/backend/src/services/auth/auth-login-service.ts b/backend/src/services/auth/auth-login-service.ts index dea41e60b..8dfe69643 100644 --- a/backend/src/services/auth/auth-login-service.ts +++ b/backend/src/services/auth/auth-login-service.ts @@ -1,5 +1,6 @@ import bcrypt from "bcrypt"; import jwt from "jsonwebtoken"; +import { Knex } from "knex"; import { TUsers, UserDeviceSchema } from "@app/db/schemas"; import { isAuthMethodSaml } from "@app/ee/services/permission/permission-fns"; @@ -50,13 +51,13 @@ export const authLoginServiceFactory = ({ * Not exported. This is to update user device list * If new device is found. Will be saved and a mail will be send */ - const updateUserDeviceSession = async (user: TUsers, ip: string, userAgent: string) => { + const updateUserDeviceSession = async (user: TUsers, ip: string, userAgent: string, tx?: Knex) => { const devices = await UserDeviceSchema.parseAsync(user.devices || []); const isDeviceSeen = devices.some((device) => device.ip === ip && device.userAgent === userAgent); if (!isDeviceSeen) { const newDeviceList = devices.concat([{ ip, userAgent }]); - await userDAL.updateById(user.id, { devices: JSON.stringify(newDeviceList) }); + await userDAL.updateById(user.id, { devices: JSON.stringify(newDeviceList) }, tx); if (user.email) { await smtpService.sendMail({ template: SmtpTemplates.NewDeviceJoin, @@ -97,30 +98,36 @@ export const authLoginServiceFactory = ({ * Check user device and send mail if new device * generate the auth and refresh token. fn shared by mfa verification and login verification with mfa disabled */ - const generateUserTokens = async ({ - user, - ip, - userAgent, - organizationId, - authMethod, - isMfaVerified, - mfaMethod - }: { - user: TUsers; - ip: string; - userAgent: string; - organizationId?: string; - authMethod: AuthMethod; - isMfaVerified?: boolean; - mfaMethod?: MfaMethod; - }) => { - const cfg = getConfig(); - await updateUserDeviceSession(user, ip, userAgent); - const tokenSession = await tokenService.getUserTokenSession({ - userAgent, + const generateUserTokens = async ( + { + user, ip, - userId: user.id - }); + userAgent, + organizationId, + authMethod, + isMfaVerified, + mfaMethod + }: { + user: TUsers; + ip: string; + userAgent: string; + organizationId?: string; + authMethod: AuthMethod; + isMfaVerified?: boolean; + mfaMethod?: MfaMethod; + }, + tx?: Knex + ) => { + const cfg = getConfig(); + await updateUserDeviceSession(user, ip, userAgent, tx); + const tokenSession = await tokenService.getUserTokenSession( + { + userAgent, + ip, + userId: user.id + }, + tx + ); if (!tokenSession) throw new Error("Failed to create token"); const accessToken = jwt.sign( diff --git a/frontend/src/hooks/api/auth/queries.tsx b/frontend/src/hooks/api/auth/queries.tsx index de3c60d46..0c3dbe0d9 100644 --- a/frontend/src/hooks/api/auth/queries.tsx +++ b/frontend/src/hooks/api/auth/queries.tsx @@ -77,11 +77,16 @@ export const selectOrganization = async (data: { export const useSelectOrganization = () => { const queryClient = useQueryClient(); return useMutation({ - mutationFn: async (details: { organizationId: string; userAgent?: UserAgentType }) => { + mutationFn: async (details: { + organizationId: string; + userAgent?: UserAgentType; + forceSetCredentials?: boolean; + }) => { const data = await selectOrganization(details); // If a custom user agent is set, then this session is meant for another consuming application, not the web application. - if (!details.userAgent && !data.isMfaEnabled) { + if ((!details.userAgent && !data.isMfaEnabled) || details.forceSetCredentials) { + localStorage.setItem("orgData.id", details.organizationId); SecurityClient.setToken(data.token); SecurityClient.setProviderAuthToken(""); } diff --git a/frontend/src/hooks/api/organization/queries.tsx b/frontend/src/hooks/api/organization/queries.tsx index 4923177ba..82894d988 100644 --- a/frontend/src/hooks/api/organization/queries.tsx +++ b/frontend/src/hooks/api/organization/queries.tsx @@ -1,5 +1,6 @@ import { useMutation, useQuery, useQueryClient, UseQueryOptions } from "@tanstack/react-query"; +import SecurityClient from "@app/components/utilities/SecurityClient"; import { apiRequest } from "@app/config/request"; import { OrderByDirection } from "@app/hooks/api/generic/types"; @@ -67,7 +68,7 @@ export const useCreateOrg = (options: { invalidate: boolean } = { invalidate: tr mutationFn: async ({ name }: { name: string }) => { const { data: { organization } - } = await apiRequest.post("/api/v2/organizations", { + } = await apiRequest.post<{ organization: { id: string } }>("/api/v2/organizations", { name }); @@ -437,10 +438,13 @@ export const useDeleteOrgById = () => { return useMutation({ mutationFn: async ({ organizationId }: { organizationId: string }) => { const { - data: { organization } - } = await apiRequest.delete<{ organization: Organization }>( + data: { organization, accessToken } + } = await apiRequest.delete<{ organization: Organization; accessToken: string }>( `/api/v2/organizations/${organizationId}` ); + SecurityClient.setToken(accessToken); + localStorage.removeItem("orgData.id"); + return organization; }, onSuccess(_, dto) { diff --git a/frontend/src/views/Login/Login.utils.tsx b/frontend/src/views/Login/Login.utils.tsx index dc714e4b0..0562d383a 100644 --- a/frontend/src/views/Login/Login.utils.tsx +++ b/frontend/src/views/Login/Login.utils.tsx @@ -7,7 +7,7 @@ import { ProjectType } from "@app/hooks/api/workspace/types"; import { queryClient } from "@app/reactQuery"; export const navigateUserToOrg = async (router: NextRouter, organizationId?: string) => { - const userOrgs = await fetchOrganizations(); + const userOrgs = await fetchOrganizations().catch(() => []); const nonAuthEnforcedOrgs = userOrgs.filter((org) => !org.authEnforced); diff --git a/frontend/src/views/Org/components/CreateOrgModal.tsx b/frontend/src/views/Org/components/CreateOrgModal.tsx index d1baf2c9c..faa02eccd 100644 --- a/frontend/src/views/Org/components/CreateOrgModal.tsx +++ b/frontend/src/views/Org/components/CreateOrgModal.tsx @@ -6,7 +6,7 @@ import z from "zod"; import { createNotification } from "@app/components/notifications"; import { Button, FormControl, Input, Modal, ModalContent } from "@app/components/v2"; -import { useCreateOrg, useSelectOrganization } from "@app/hooks/api"; +import { useCreateOrg, useGetOrganizations, useSelectOrganization } from "@app/hooks/api"; import { ProjectType } from "@app/hooks/api/workspace/types"; const schema = z @@ -23,9 +23,10 @@ interface CreateOrgModalProps { } export const CreateOrgModal: FC = ({ isOpen, onClose }) => { - const router = useRouter(); + const { refetch: refetchOrganizations } = useGetOrganizations(); + const { control, handleSubmit, @@ -50,19 +51,21 @@ export const CreateOrgModal: FC = ({ isOpen, onClose }) => }); await selectOrg({ - organizationId: organization.id + organizationId: organization.id, + forceSetCredentials: true }); + await refetchOrganizations(); + createNotification({ text: "Successfully created organization", type: "success" }); - if (router.isReady) router.push(`/org/${organization.id}/${ProjectType.SecretManager}/overview`); + if (router.isReady) + router.push(`/org/${organization.id}/${ProjectType.SecretManager}/overview`); else window.location.href = `/org/${organization.id}/${ProjectType.SecretManager}/overview`; - localStorage.setItem("orgData.id", organization.id); - reset(); onClose(); } catch (err) { From c6a0e363186070ca884d6a75f46e16370c377dc1 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Wed, 11 Dec 2024 08:16:11 +0400 Subject: [PATCH 3/7] fix(api): secret reminders not getting deleted --- backend/src/lib/knex/index.ts | 10 +- backend/src/server/routes/index.ts | 95 ++++++++------- backend/src/services/org/org-service.ts | 108 +++++++++++++++--- .../src/services/project/project-service.ts | 24 +++- backend/src/services/secret/secret-fns.ts | 50 ++++++++ 5 files changed, 229 insertions(+), 58 deletions(-) diff --git a/backend/src/lib/knex/index.ts b/backend/src/lib/knex/index.ts index f55d8e6e6..0022ee8ea 100644 --- a/backend/src/lib/knex/index.ts +++ b/backend/src/lib/knex/index.ts @@ -20,11 +20,12 @@ export const withTransaction = (db: Knex, dal: K) => ({ export type TFindFilter = Partial & { $in?: Partial<{ [k in keyof R]: R[k][] }>; + $notNull?: Array; $search?: Partial<{ [k in keyof R]: R[k] }>; $complex?: TKnexDynamicOperator; }; export const buildFindFilter = - ({ $in, $search, $complex, ...filter }: TFindFilter) => + ({ $in, $notNull, $search, $complex, ...filter }: TFindFilter) => (bd: Knex.QueryBuilder) => { void bd.where(filter); if ($in) { @@ -34,6 +35,13 @@ export const buildFindFilter = } }); } + + if ($notNull?.length) { + $notNull.forEach((key) => { + void bd.whereNotNull(key as never); + }); + } + if ($search) { Object.entries($search).forEach(([key, val]) => { if (val) { diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 8890b0f4d..dd6520875 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -538,7 +538,11 @@ export const registerRoutes = async ( const orgService = orgServiceFactory({ userAliasDAL, + queueService, identityMetadataDAL, + secretDAL, + secretV2BridgeDAL, + folderDAL, licenseService, samlConfigDAL, orgRoleDAL, @@ -559,6 +563,7 @@ export const registerRoutes = async ( groupDAL, orgBotDAL, oidcConfigDAL, + loginService, projectBotService }); const signupService = authSignupServiceFactory({ @@ -776,10 +781,58 @@ export const registerRoutes = async ( projectTemplateDAL }); + const integrationAuthService = integrationAuthServiceFactory({ + integrationAuthDAL, + integrationDAL, + permissionService, + projectBotService, + kmsService + }); + + const secretQueueService = secretQueueFactory({ + keyStore, + queueService, + secretDAL, + folderDAL, + integrationAuthService, + projectBotService, + integrationDAL, + secretImportDAL, + projectEnvDAL, + webhookDAL, + orgDAL, + auditLogService, + userDAL, + projectMembershipDAL, + smtpService, + projectDAL, + projectBotDAL, + secretVersionDAL, + secretBlindIndexDAL, + secretTagDAL, + secretVersionTagDAL, + kmsService, + secretVersionV2BridgeDAL, + secretV2BridgeDAL, + secretVersionTagV2BridgeDAL, + secretRotationDAL, + integrationAuthDAL, + snapshotDAL, + snapshotSecretV2BridgeDAL, + secretApprovalRequestDAL, + projectKeyDAL, + projectUserMembershipRoleDAL, + orgService + }); + const projectService = projectServiceFactory({ permissionService, projectDAL, + secretDAL, + secretV2BridgeDAL, + queueService, projectQueue: projectQueueService, + projectBotService, identityProjectDAL, identityOrgMembershipDAL, projectKeyDAL, @@ -859,48 +912,6 @@ export const registerRoutes = async ( projectDAL }); - const integrationAuthService = integrationAuthServiceFactory({ - integrationAuthDAL, - integrationDAL, - permissionService, - projectBotService, - kmsService - }); - const secretQueueService = secretQueueFactory({ - keyStore, - queueService, - secretDAL, - folderDAL, - integrationAuthService, - projectBotService, - integrationDAL, - secretImportDAL, - projectEnvDAL, - webhookDAL, - orgDAL, - auditLogService, - userDAL, - projectMembershipDAL, - smtpService, - projectDAL, - projectBotDAL, - secretVersionDAL, - secretBlindIndexDAL, - secretTagDAL, - secretVersionTagDAL, - kmsService, - secretVersionV2BridgeDAL, - secretV2BridgeDAL, - secretVersionTagV2BridgeDAL, - secretRotationDAL, - integrationAuthDAL, - snapshotDAL, - snapshotSecretV2BridgeDAL, - secretApprovalRequestDAL, - projectKeyDAL, - projectUserMembershipRoleDAL, - orgService - }); const secretImportService = secretImportServiceFactory({ licenseService, projectBotService, diff --git a/backend/src/services/org/org-service.ts b/backend/src/services/org/org-service.ts index 33931bf26..73b8c04e5 100644 --- a/backend/src/services/org/org-service.ts +++ b/backend/src/services/org/org-service.ts @@ -31,11 +31,13 @@ import { BadRequestError, ForbiddenRequestError, NotFoundError, UnauthorizedErro import { groupBy } from "@app/lib/fn"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { isDisposableEmail } from "@app/lib/validator"; +import { TQueueServiceFactory } from "@app/queue"; import { getDefaultOrgMembershipRoleForUpdateOrg } from "@app/services/org/org-role-fns"; import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal"; import { TUserAliasDALFactory } from "@app/services/user-alias/user-alias-dal"; -import { ActorAuthMethod, ActorType, AuthMethod, AuthTokenType } from "../auth/auth-type"; +import { TAuthLoginFactory } from "../auth/auth-login-service"; +import { ActorAuthMethod, ActorType, AuthMethod, AuthModeJwtTokenPayload, AuthTokenType } from "../auth/auth-type"; import { TAuthTokenServiceFactory } from "../auth-token/auth-token-service"; import { TokenType } from "../auth-token/auth-token-types"; import { TIdentityMetadataDALFactory } from "../identity/identity-metadata-dal"; @@ -47,6 +49,10 @@ import { TProjectKeyDALFactory } from "../project-key/project-key-dal"; import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal"; import { TProjectRoleDALFactory } from "../project-role/project-role-dal"; +import { TSecretDALFactory } from "../secret/secret-dal"; +import { fnDeleteProjectSecretReminders } from "../secret/secret-fns"; +import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service"; import { TUserDALFactory } from "../user/user-dal"; import { TIncidentContactsDALFactory } from "./incident-contacts-dal"; @@ -69,6 +75,9 @@ import { type TOrgServiceFactoryDep = { userAliasDAL: Pick; + secretDAL: Pick; + secretV2BridgeDAL: Pick; + folderDAL: Pick; orgDAL: TOrgDALFactory; orgBotDAL: TOrgBotDALFactory; orgRoleDAL: TOrgRoleDALFactory; @@ -97,6 +106,8 @@ type TOrgServiceFactoryDep = { projectBotDAL: Pick; projectUserMembershipRoleDAL: Pick; projectBotService: Pick; + queueService: Pick; + loginService: Pick; }; export type TOrgServiceFactory = ReturnType; @@ -104,6 +115,9 @@ export type TOrgServiceFactory = ReturnType; export const orgServiceFactory = ({ userAliasDAL, orgDAL, + secretDAL, + secretV2BridgeDAL, + folderDAL, userDAL, groupDAL, orgRoleDAL, @@ -124,7 +138,9 @@ export const orgServiceFactory = ({ projectBotDAL, projectUserMembershipRoleDAL, identityMetadataDAL, - projectBotService + projectBotService, + queueService, + loginService }: TOrgServiceFactoryDep) => { /* * Get organization details by the organization id @@ -419,24 +435,88 @@ export const orgServiceFactory = ({ /* * Delete organization by id * */ - const deleteOrganizationById = async ( - userId: string, - orgId: string, - actorAuthMethod: ActorAuthMethod, - actorOrgId: string | undefined - ) => { + const deleteOrganizationById = async ({ + userId, + authorizationHeader, + userAgentHeader, + ipAddress, + orgId, + actorAuthMethod, + actorOrgId + }: { + userId: string; + authorizationHeader?: string; + userAgentHeader?: string; + ipAddress: string; + orgId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string | undefined; + }) => { const { membership } = await permissionService.getUserOrgPermission(userId, orgId, actorAuthMethod, actorOrgId); - if ((membership.role as OrgMembershipRole) !== OrgMembershipRole.Admin) + if ((membership.role as OrgMembershipRole) !== OrgMembershipRole.Admin) { throw new ForbiddenRequestError({ name: "DeleteOrganizationById", message: "Insufficient privileges" }); - - const organization = await orgDAL.deleteById(orgId); - if (organization.customerId) { - await licenseService.removeOrgCustomer(organization.customerId); } - return organization; + + if (!authorizationHeader) { + throw new UnauthorizedError({ name: "Authorization header not set on request." }); + } + + if (!userAgentHeader) { + throw new BadRequestError({ name: "User agent not set on request." }); + } + + const cfg = getConfig(); + const authToken = authorizationHeader.replace("Bearer ", ""); + + const decodedToken = jwt.verify(authToken, cfg.AUTH_SECRET) as AuthModeJwtTokenPayload; + if (!decodedToken.authMethod) throw new UnauthorizedError({ name: "Auth method not found on existing token" }); + + const response = await orgDAL.transaction(async (tx) => { + const projects = await projectDAL.find({ orgId }, { tx }); + + for await (const project of projects) { + await fnDeleteProjectSecretReminders(project.id, { + secretDAL, + secretV2BridgeDAL, + queueService, + projectBotService, + folderDAL + }); + } + + const deletedOrg = await orgDAL.deleteById(orgId, tx); + + if (deletedOrg.customerId) { + await licenseService.removeOrgCustomer(deletedOrg.customerId); + } + + // Generate new tokens without the organization ID present + const user = await userDAL.findById(userId, tx); + const { access: accessToken, refresh: refreshToken } = await loginService.generateUserTokens( + { + user, + authMethod: decodedToken.authMethod, + ip: ipAddress, + userAgent: userAgentHeader, + isMfaVerified: decodedToken.isMfaVerified, + mfaMethod: decodedToken.mfaMethod + }, + tx + ); + + return { + organization: deletedOrg, + tokens: { + accessToken, + refreshToken + } + }; + }); + + return response; }; /* * Org membership management diff --git a/backend/src/services/project/project-service.ts b/backend/src/services/project/project-service.ts index c8a86f499..47af31f0f 100644 --- a/backend/src/services/project/project-service.ts +++ b/backend/src/services/project/project-service.ts @@ -14,6 +14,7 @@ import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/ import { groupBy } from "@app/lib/fn"; import { alphaNumericNanoId } from "@app/lib/nanoid"; import { TProjectPermission } from "@app/lib/types"; +import { TQueueServiceFactory } from "@app/queue"; import { ActorType } from "../auth/auth-type"; import { TCertificateDALFactory } from "../certificate/certificate-dal"; @@ -28,13 +29,17 @@ import { TOrgServiceFactory } from "../org/org-service"; import { TPkiAlertDALFactory } from "../pki-alert/pki-alert-dal"; import { TPkiCollectionDALFactory } from "../pki-collection/pki-collection-dal"; import { TProjectBotDALFactory } from "../project-bot/project-bot-dal"; +import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TProjectKeyDALFactory } from "../project-key/project-key-dal"; import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal"; import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal"; import { TProjectRoleDALFactory } from "../project-role/project-role-dal"; import { getPredefinedRoles } from "../project-role/project-role-fns"; +import { TSecretDALFactory } from "../secret/secret-dal"; +import { fnDeleteProjectSecretReminders } from "../secret/secret-fns"; import { ROOT_FOLDER_NAME, TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { TProjectSlackConfigDALFactory } from "../slack/project-slack-config-dal"; import { TSlackIntegrationDALFactory } from "../slack/slack-integration-dal"; import { TUserDALFactory } from "../user/user-dal"; @@ -74,7 +79,10 @@ type TProjectServiceFactoryDep = { projectDAL: TProjectDALFactory; projectQueue: TProjectQueueFactory; userDAL: TUserDALFactory; - folderDAL: TSecretFolderDALFactory; + projectBotService: Pick; + folderDAL: Pick; + secretDAL: Pick; + secretV2BridgeDAL: Pick; projectEnvDAL: Pick; identityOrgMembershipDAL: TIdentityOrgDALFactory; identityProjectDAL: TIdentityProjectDALFactory; @@ -92,6 +100,8 @@ type TProjectServiceFactoryDep = { permissionService: TPermissionServiceFactory; orgService: Pick; licenseService: Pick; + queueService: Pick; + orgDAL: Pick; keyStore: Pick; projectBotDAL: Pick; @@ -112,9 +122,13 @@ export type TProjectServiceFactory = ReturnType; export const projectServiceFactory = ({ projectDAL, + secretDAL, + secretV2BridgeDAL, projectQueue, projectKeyDAL, permissionService, + queueService, + projectBotService, orgDAL, userDAL, folderDAL, @@ -424,6 +438,14 @@ export const projectServiceFactory = ({ await userDAL.deleteById(projectGhostUser.id, tx); } + await fnDeleteProjectSecretReminders(project.id, { + secretDAL, + secretV2BridgeDAL, + queueService, + projectBotService, + folderDAL + }); + return delProject; }); diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index 65691fcbb..6336c479d 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -19,9 +19,11 @@ import { decryptSymmetric128BitHexKeyUTF8, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto"; +import { daysToMillisecond, secondsToMillis } from "@app/lib/dates"; import { BadRequestError, NotFoundError } from "@app/lib/errors"; import { groupBy, unique } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; +import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { fnSecretBulkInsert as fnSecretV2BridgeBulkInsert, fnSecretBulkUpdate as fnSecretV2BridgeBulkUpdate, @@ -31,8 +33,10 @@ import { import { ActorAuthMethod, ActorType } from "../auth/auth-type"; import { KmsDataKey } from "../kms/kms-types"; import { getBotKeyFnFactory } from "../project-bot/project-bot-fns"; +import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; +import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal"; import { TSecretDALFactory } from "./secret-dal"; import { TCreateManySecretsRawFn, @@ -1138,3 +1142,49 @@ export const decryptSecretWithBot = ( secretComment }; }; + +type TFnDeleteProjectSecretReminders = { + secretDAL: Pick; + secretV2BridgeDAL: Pick; + queueService: Pick; + projectBotService: Pick; + folderDAL: Pick; +}; + +export const fnDeleteProjectSecretReminders = async ( + projectId: string, + { secretDAL, secretV2BridgeDAL, queueService, projectBotService, folderDAL }: TFnDeleteProjectSecretReminders +) => { + const projectFolders = await folderDAL.findByProjectId(projectId); + const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId, false); + + const projectSecrets = shouldUseSecretV2Bridge + ? await secretV2BridgeDAL.find({ + $in: { folderId: projectFolders.map((folder) => folder.id) }, + $notNull: ["reminderRepeatDays"] + }) + : await secretDAL.find({ + $in: { folderId: projectFolders.map((folder) => folder.id) }, + $notNull: ["secretReminderRepeatDays"] + }); + + const appCfg = getConfig(); + for await (const secret of projectSecrets) { + const repeatDays = shouldUseSecretV2Bridge + ? (secret as { reminderRepeatDays: number }).reminderRepeatDays + : (secret as { secretReminderRepeatDays: number }).secretReminderRepeatDays; + + // We're using the queue service directly to get around conflicting imports. + if (repeatDays) { + await queueService.stopRepeatableJob( + QueueName.SecretReminder, + QueueJobs.SecretReminder, + { + // on prod it this will be in days, in development this will be second + every: appCfg.NODE_ENV === "development" ? secondsToMillis(repeatDays) : daysToMillisecond(repeatDays) + }, + `reminder-${secret.id}` + ); + } + } +}; From ccb757ec3e878ff6e81b4e7e20a9acdc91cea13a Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Mon, 16 Dec 2024 20:58:56 +0100 Subject: [PATCH 4/7] fix: missed transaction --- backend/src/services/auth-token/auth-token-dal.ts | 7 +++++-- backend/src/services/auth-token/auth-token-service.ts | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/src/services/auth-token/auth-token-dal.ts b/backend/src/services/auth-token/auth-token-dal.ts index ed45d505e..221b691cf 100644 --- a/backend/src/services/auth-token/auth-token-dal.ts +++ b/backend/src/services/auth-token/auth-token-dal.ts @@ -12,9 +12,12 @@ export type TTokenDALFactory = ReturnType; export const tokenDALFactory = (db: TDbClient) => { const authOrm = ormify(db, TableName.AuthTokens); - const findOneTokenSession = async (filter: Partial): Promise => { + const findOneTokenSession = async ( + filter: Partial, + tx?: Knex + ): Promise => { try { - const doc = await db.replicaNode()(TableName.AuthTokenSession).where(filter).first(); + const doc = await (tx || db.replicaNode())(TableName.AuthTokenSession).where(filter).first(); return doc; } catch (error) { throw new DatabaseError({ error, name: "FindOneTokenSession" }); diff --git a/backend/src/services/auth-token/auth-token-service.ts b/backend/src/services/auth-token/auth-token-service.ts index b9fc98106..c0bb7dc17 100644 --- a/backend/src/services/auth-token/auth-token-service.ts +++ b/backend/src/services/auth-token/auth-token-service.ts @@ -128,7 +128,7 @@ export const tokenServiceFactory = ({ tokenDAL, userDAL, orgMembershipDAL }: TAu { userId, ip, userAgent }: TIssueAuthTokenDTO, tx?: Knex ): Promise => { - let session = await tokenDAL.findOneTokenSession({ userId, ip, userAgent }); + let session = await tokenDAL.findOneTokenSession({ userId, ip, userAgent }, tx); if (!session) { session = await tokenDAL.insertTokenSession(userId, ip, userAgent, tx); } From cc3e0d19227e60acb5ecc1185007d59c5d402fda Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Tue, 17 Dec 2024 23:32:04 +0100 Subject: [PATCH 5/7] fix: remove completed and failed reminder jobs --- backend/src/services/secret/secret-queue.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 84a8584ee..57d32ab11 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -248,7 +248,9 @@ export const secretQueueFactory = ({ ? secondsToMillis(newSecret.secretReminderRepeatDays) : daysToMillisecond(newSecret.secretReminderRepeatDays), immediately: true - } + }, + removeOnComplete: true, + removeOnFail: true } ); } catch (err) { From 761007208db2e1c0332c4074ffae7398d484ac41 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Tue, 17 Dec 2024 23:32:45 +0100 Subject: [PATCH 6/7] misc: daily cleanup of rogue secret reminder jobs --- backend/src/queue/queue-service.ts | 14 +++ backend/src/server/routes/index.ts | 1 + .../resource-cleanup-queue.ts | 4 + backend/src/services/secret/secret-dal.ts | 93 ++++++++++++++++++- 4 files changed, 111 insertions(+), 1 deletion(-) diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index 051fe9cbd..330193052 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -317,6 +317,13 @@ export const queueServiceFactory = ( } }; + const getRepeatableJobs = (name: QueueName, startOffset?: number, endOffset?: number) => { + const q = queueContainer[name]; + if (!q) throw new Error(`Queue '${name}' not initialized`); + + return q.getRepeatableJobs(startOffset, endOffset); + }; + const stopRepeatableJobByJobId = async (name: T, jobId: string) => { const q = queueContainer[name]; const job = await q.getJob(jobId); @@ -326,6 +333,11 @@ export const queueServiceFactory = ( return q.removeRepeatableByKey(job.repeatJobKey); }; + const stopRepeatableJobByKey = async (name: T, repeatJobKey: string) => { + const q = queueContainer[name]; + return q.removeRepeatableByKey(repeatJobKey); + }; + const stopJobById = async (name: T, jobId: string) => { const q = queueContainer[name]; const job = await q.getJob(jobId); @@ -349,8 +361,10 @@ export const queueServiceFactory = ( shutdown, stopRepeatableJob, stopRepeatableJobByJobId, + stopRepeatableJobByKey, clearQueue, stopJobById, + getRepeatableJobs, startPg, queuePg }; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index dd6520875..65a878bbf 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -1240,6 +1240,7 @@ export const registerRoutes = async ( auditLogDAL, queueService, secretVersionDAL, + secretDAL, secretFolderVersionDAL: folderVersionDAL, snapshotDAL, identityAccessTokenDAL, diff --git a/backend/src/services/resource-cleanup/resource-cleanup-queue.ts b/backend/src/services/resource-cleanup/resource-cleanup-queue.ts index dab70806f..aa1ed9d25 100644 --- a/backend/src/services/resource-cleanup/resource-cleanup-queue.ts +++ b/backend/src/services/resource-cleanup/resource-cleanup-queue.ts @@ -5,6 +5,7 @@ import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue"; import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identity-access-token-dal"; import { TIdentityUaClientSecretDALFactory } from "../identity-ua/identity-ua-client-secret-dal"; +import { TSecretDALFactory } from "../secret/secret-dal"; import { TSecretVersionDALFactory } from "../secret/secret-version-dal"; import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal"; import { TSecretSharingDALFactory } from "../secret-sharing/secret-sharing-dal"; @@ -16,6 +17,7 @@ type TDailyResourceCleanUpQueueServiceFactoryDep = { identityUniversalAuthClientSecretDAL: Pick; secretVersionDAL: Pick; secretVersionV2DAL: Pick; + secretDAL: Pick; secretFolderVersionDAL: Pick; snapshotDAL: Pick; secretSharingDAL: Pick; @@ -30,6 +32,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({ snapshotDAL, secretVersionDAL, secretFolderVersionDAL, + secretDAL, identityAccessTokenDAL, secretSharingDAL, secretVersionV2DAL, @@ -37,6 +40,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({ }: TDailyResourceCleanUpQueueServiceFactoryDep) => { queueService.start(QueueName.DailyResourceCleanUp, async () => { logger.info(`${QueueName.DailyResourceCleanUp}: queue task started`); + await secretDAL.pruneSecretReminders(queueService); await auditLogDAL.pruneAuditLog(); await identityAccessTokenDAL.removeExpiredTokens(); await identityUniversalAuthClientSecretDAL.removeExpiredClientSecrets(); diff --git a/backend/src/services/secret/secret-dal.ts b/backend/src/services/secret/secret-dal.ts index 0d4ae0cda..cbaf7ddcd 100644 --- a/backend/src/services/secret/secret-dal.ts +++ b/backend/src/services/secret/secret-dal.ts @@ -5,6 +5,8 @@ import { TDbClient } from "@app/db"; import { SecretsSchema, SecretType, TableName, TSecrets, TSecretsUpdate } from "@app/db/schemas"; import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex"; +import { logger } from "@app/lib/logger"; +import { QueueName, TQueueServiceFactory } from "@app/queue"; export type TSecretDALFactory = ReturnType; @@ -339,6 +341,94 @@ export const secretDALFactory = (db: TDbClient) => { } }; + const pruneSecretReminders = async (queueService: TQueueServiceFactory) => { + const REMINDER_PRUNE_BATCH_SIZE = 5_000; + const MAX_RETRY_ON_FAILURE = 3; + let numberOfRetryOnFailure = 0; + let deletedReminderCount = 0; + + logger.info(`${QueueName.DailyResourceCleanUp}: secret reminders started`); + + try { + const repeatableJobs = await queueService.getRepeatableJobs(QueueName.SecretReminder); + const reminderJobs = repeatableJobs + .map((job) => ({ secretId: job.id?.replace("reminder-", "") as string, jobKey: job.key })) + .filter(Boolean); + + if (reminderJobs.length === 0) { + logger.info(`${QueueName.DailyResourceCleanUp}: no reminder jobs found`); + return; + } + + for (let offset = 0; offset < reminderJobs.length; offset += REMINDER_PRUNE_BATCH_SIZE) { + try { + const batchIds = reminderJobs.slice(offset, offset + REMINDER_PRUNE_BATCH_SIZE).map((r) => r.secretId); + + const payload = { + $in: { + id: batchIds + } + }; + + const opts = { + limit: REMINDER_PRUNE_BATCH_SIZE + }; + + // Find existing secrets with pagination + // eslint-disable-next-line no-await-in-loop + const [secrets, secretsV2] = await Promise.all([ + ormify(db, TableName.Secret).find(payload, opts), + ormify(db, TableName.SecretV2).find(payload, opts) + ]); + + const foundSecretIds = new Set([ + ...secrets.map((secret) => secret.id), + ...secretsV2.map((secret) => secret.id) + ]); + + // Find IDs that don't exist in either table + const secretIdsNotFound = batchIds.filter((secretId) => !foundSecretIds.has(secretId)); + + // Delete reminders for non-existent secrets + for (const secretId of secretIdsNotFound) { + const jobKey = reminderJobs.find((r) => r.secretId === secretId)?.jobKey; + + if (jobKey) { + // eslint-disable-next-line no-await-in-loop + await queueService.stopRepeatableJobByKey(QueueName.SecretReminder, jobKey); + deletedReminderCount += 1; + } + } + + numberOfRetryOnFailure = 0; + } catch (error) { + numberOfRetryOnFailure += 1; + logger.error(error, `Failed to process batch at offset ${offset}`); + + if (numberOfRetryOnFailure >= MAX_RETRY_ON_FAILURE) { + break; + } + + // Retry the current batch + offset -= REMINDER_PRUNE_BATCH_SIZE; + + // eslint-disable-next-line no-promise-executor-return, @typescript-eslint/no-loop-func, no-await-in-loop + await new Promise((resolve) => setTimeout(resolve, 500 * numberOfRetryOnFailure)); + } + + // Small delay between batches + // eslint-disable-next-line no-promise-executor-return, @typescript-eslint/no-loop-func, no-await-in-loop + await new Promise((resolve) => setTimeout(resolve, 10)); + } + } catch (error) { + logger.error(error, "Failed to complete secret reminder pruning"); + } finally { + logger.info( + `${QueueName.DailyResourceCleanUp}: secret reminders completed. Deleted ${deletedReminderCount} reminders` + ); + } + }; + return { ...secretOrm, update, @@ -352,6 +442,7 @@ export const secretDALFactory = (db: TDbClient) => { findByBlindIndexes, upsertSecretReferences, findReferencedSecretReferences, - findAllProjectSecretValues + findAllProjectSecretValues, + pruneSecretReminders }; }; From a92e61575d69715920a1cbe74dbecac8c26ba64b Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Wed, 18 Dec 2024 01:10:23 +0100 Subject: [PATCH 7/7] fix: test types --- backend/e2e-test/mocks/queue.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/e2e-test/mocks/queue.ts b/backend/e2e-test/mocks/queue.ts index 0028381bd..99e3999e1 100644 --- a/backend/e2e-test/mocks/queue.ts +++ b/backend/e2e-test/mocks/queue.ts @@ -22,8 +22,10 @@ export const mockQueue = (): TQueueServiceFactory => { listen: (name, event) => { events[name] = event; }, + getRepeatableJobs: async () => [], clearQueue: async () => {}, stopJobById: async () => {}, - stopRepeatableJobByJobId: async () => true + stopRepeatableJobByJobId: async () => true, + stopRepeatableJobByKey: async () => true }; };