Merge pull request #2865 from Infisical/daniel/fix-reminder-cleanup

fix(secret-reminders): proper cleanup on deleted resources
This commit is contained in:
Daniel Hougaard
2024-12-18 02:57:07 +01:00
committed by GitHub
19 changed files with 471 additions and 126 deletions

View File

@@ -22,8 +22,10 @@ export const mockQueue = (): TQueueServiceFactory => {
listen: (name, event) => {
events[name] = event;
},
getRepeatableJobs: async () => [],
clearQueue: async () => {},
stopJobById: async () => {},
stopRepeatableJobByJobId: async () => true
stopRepeatableJobByJobId: async () => true,
stopRepeatableJobByKey: async () => true
};
};

View File

@@ -20,11 +20,12 @@ export const withTransaction = <K extends object>(db: Knex, dal: K) => ({
export type TFindFilter<R extends object = object> = Partial<R> & {
$in?: Partial<{ [k in keyof R]: R[k][] }>;
$notNull?: Array<keyof R>;
$search?: Partial<{ [k in keyof R]: R[k] }>;
$complex?: TKnexDynamicOperator<R>;
};
export const buildFindFilter =
<R extends object = object>({ $in, $search, $complex, ...filter }: TFindFilter<R>) =>
<R extends object = object>({ $in, $notNull, $search, $complex, ...filter }: TFindFilter<R>) =>
(bd: Knex.QueryBuilder<R, R>) => {
void bd.where(filter);
if ($in) {
@@ -34,6 +35,13 @@ export const buildFindFilter =
}
});
}
if ($notNull?.length) {
$notNull.forEach((key) => {
void bd.whereNotNull(key as never);
});
}
if ($search) {
Object.entries($search).forEach(([key, val]) => {
if (val) {

View File

@@ -317,6 +317,13 @@ export const queueServiceFactory = (
}
};
const getRepeatableJobs = (name: QueueName, startOffset?: number, endOffset?: number) => {
const q = queueContainer[name];
if (!q) throw new Error(`Queue '${name}' not initialized`);
return q.getRepeatableJobs(startOffset, endOffset);
};
const stopRepeatableJobByJobId = async <T extends QueueName>(name: T, jobId: string) => {
const q = queueContainer[name];
const job = await q.getJob(jobId);
@@ -326,6 +333,11 @@ export const queueServiceFactory = (
return q.removeRepeatableByKey(job.repeatJobKey);
};
const stopRepeatableJobByKey = async <T extends QueueName>(name: T, repeatJobKey: string) => {
const q = queueContainer[name];
return q.removeRepeatableByKey(repeatJobKey);
};
const stopJobById = async <T extends QueueName>(name: T, jobId: string) => {
const q = queueContainer[name];
const job = await q.getJob(jobId);
@@ -349,8 +361,10 @@ export const queueServiceFactory = (
shutdown,
stopRepeatableJob,
stopRepeatableJobByJobId,
stopRepeatableJobByKey,
clearQueue,
stopJobById,
getRepeatableJobs,
startPg,
queuePg
};

View File

@@ -538,7 +538,11 @@ export const registerRoutes = async (
const orgService = orgServiceFactory({
userAliasDAL,
queueService,
identityMetadataDAL,
secretDAL,
secretV2BridgeDAL,
folderDAL,
licenseService,
samlConfigDAL,
orgRoleDAL,
@@ -559,6 +563,7 @@ export const registerRoutes = async (
groupDAL,
orgBotDAL,
oidcConfigDAL,
loginService,
projectBotService
});
const signupService = authSignupServiceFactory({
@@ -776,10 +781,58 @@ export const registerRoutes = async (
projectTemplateDAL
});
const integrationAuthService = integrationAuthServiceFactory({
integrationAuthDAL,
integrationDAL,
permissionService,
projectBotService,
kmsService
});
const secretQueueService = secretQueueFactory({
keyStore,
queueService,
secretDAL,
folderDAL,
integrationAuthService,
projectBotService,
integrationDAL,
secretImportDAL,
projectEnvDAL,
webhookDAL,
orgDAL,
auditLogService,
userDAL,
projectMembershipDAL,
smtpService,
projectDAL,
projectBotDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
kmsService,
secretVersionV2BridgeDAL,
secretV2BridgeDAL,
secretVersionTagV2BridgeDAL,
secretRotationDAL,
integrationAuthDAL,
snapshotDAL,
snapshotSecretV2BridgeDAL,
secretApprovalRequestDAL,
projectKeyDAL,
projectUserMembershipRoleDAL,
orgService
});
const projectService = projectServiceFactory({
permissionService,
projectDAL,
secretDAL,
secretV2BridgeDAL,
queueService,
projectQueue: projectQueueService,
projectBotService,
identityProjectDAL,
identityOrgMembershipDAL,
projectKeyDAL,
@@ -859,48 +912,6 @@ export const registerRoutes = async (
projectDAL
});
const integrationAuthService = integrationAuthServiceFactory({
integrationAuthDAL,
integrationDAL,
permissionService,
projectBotService,
kmsService
});
const secretQueueService = secretQueueFactory({
keyStore,
queueService,
secretDAL,
folderDAL,
integrationAuthService,
projectBotService,
integrationDAL,
secretImportDAL,
projectEnvDAL,
webhookDAL,
orgDAL,
auditLogService,
userDAL,
projectMembershipDAL,
smtpService,
projectDAL,
projectBotDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
kmsService,
secretVersionV2BridgeDAL,
secretV2BridgeDAL,
secretVersionTagV2BridgeDAL,
secretRotationDAL,
integrationAuthDAL,
snapshotDAL,
snapshotSecretV2BridgeDAL,
secretApprovalRequestDAL,
projectKeyDAL,
projectUserMembershipRoleDAL,
orgService
});
const secretImportService = secretImportServiceFactory({
licenseService,
projectBotService,
@@ -1229,6 +1240,7 @@ export const registerRoutes = async (
auditLogDAL,
queueService,
secretVersionDAL,
secretDAL,
secretFolderVersionDAL: folderVersionDAL,
snapshotDAL,
identityAccessTokenDAL,

View File

@@ -10,6 +10,7 @@ import {
UsersSchema
} from "@app/db/schemas";
import { ORGANIZATIONS } from "@app/lib/api-docs";
import { getConfig } from "@app/lib/config/env";
import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
import { ActorType, AuthMode } from "@app/services/auth/auth-type";
@@ -363,21 +364,35 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => {
}),
response: {
200: z.object({
organization: OrganizationsSchema
organization: OrganizationsSchema,
accessToken: z.string()
})
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY]),
handler: async (req) => {
handler: async (req, res) => {
if (req.auth.actor !== ActorType.USER) return;
const organization = await server.services.org.deleteOrganizationById(
req.permission.id,
req.params.organizationId,
req.permission.authMethod,
req.permission.orgId
);
return { organization };
const cfg = getConfig();
const { organization, tokens } = await server.services.org.deleteOrganizationById({
userId: req.permission.id,
orgId: req.params.organizationId,
actorAuthMethod: req.permission.authMethod,
actorOrgId: req.permission.orgId,
authorizationHeader: req.headers.authorization,
userAgentHeader: req.headers["user-agent"],
ipAddress: req.realIp
});
void res.setCookie("jid", tokens.refreshToken, {
httpOnly: true,
path: "/",
sameSite: "strict",
secure: cfg.HTTPS_ENABLED
});
return { organization, accessToken: tokens.accessToken };
}
});
};

View File

@@ -12,9 +12,12 @@ export type TTokenDALFactory = ReturnType<typeof tokenDALFactory>;
export const tokenDALFactory = (db: TDbClient) => {
const authOrm = ormify(db, TableName.AuthTokens);
const findOneTokenSession = async (filter: Partial<TAuthTokenSessions>): Promise<TAuthTokenSessions | undefined> => {
const findOneTokenSession = async (
filter: Partial<TAuthTokenSessions>,
tx?: Knex
): Promise<TAuthTokenSessions | undefined> => {
try {
const doc = await db.replicaNode()(TableName.AuthTokenSession).where(filter).first();
const doc = await (tx || db.replicaNode())(TableName.AuthTokenSession).where(filter).first();
return doc;
} catch (error) {
throw new DatabaseError({ error, name: "FindOneTokenSession" });
@@ -54,10 +57,11 @@ export const tokenDALFactory = (db: TDbClient) => {
const insertTokenSession = async (
userId: string,
ip: string,
userAgent: string
userAgent: string,
tx?: Knex
): Promise<TAuthTokenSessions | undefined> => {
try {
const [session] = await db(TableName.AuthTokenSession)
const [session] = await (tx || db)(TableName.AuthTokenSession)
.insert({
userId,
ip,

View File

@@ -1,6 +1,7 @@
import crypto from "node:crypto";
import bcrypt from "bcrypt";
import { Knex } from "knex";
import { TAuthTokens, TAuthTokenSessions } from "@app/db/schemas";
import { getConfig } from "@app/lib/config/env";
@@ -123,14 +124,13 @@ export const tokenServiceFactory = ({ tokenDAL, userDAL, orgMembershipDAL }: TAu
return deletedToken?.[0];
};
const getUserTokenSession = async ({
userId,
ip,
userAgent
}: TIssueAuthTokenDTO): Promise<TAuthTokenSessions | undefined> => {
let session = await tokenDAL.findOneTokenSession({ userId, ip, userAgent });
const getUserTokenSession = async (
{ userId, ip, userAgent }: TIssueAuthTokenDTO,
tx?: Knex
): Promise<TAuthTokenSessions | undefined> => {
let session = await tokenDAL.findOneTokenSession({ userId, ip, userAgent }, tx);
if (!session) {
session = await tokenDAL.insertTokenSession(userId, ip, userAgent);
session = await tokenDAL.insertTokenSession(userId, ip, userAgent, tx);
}
return session;
};

View File

@@ -1,5 +1,6 @@
import bcrypt from "bcrypt";
import jwt from "jsonwebtoken";
import { Knex } from "knex";
import { TUsers, UserDeviceSchema } from "@app/db/schemas";
import { isAuthMethodSaml } from "@app/ee/services/permission/permission-fns";
@@ -50,13 +51,13 @@ export const authLoginServiceFactory = ({
* Not exported. This is to update user device list
* If new device is found. Will be saved and a mail will be send
*/
const updateUserDeviceSession = async (user: TUsers, ip: string, userAgent: string) => {
const updateUserDeviceSession = async (user: TUsers, ip: string, userAgent: string, tx?: Knex) => {
const devices = await UserDeviceSchema.parseAsync(user.devices || []);
const isDeviceSeen = devices.some((device) => device.ip === ip && device.userAgent === userAgent);
if (!isDeviceSeen) {
const newDeviceList = devices.concat([{ ip, userAgent }]);
await userDAL.updateById(user.id, { devices: JSON.stringify(newDeviceList) });
await userDAL.updateById(user.id, { devices: JSON.stringify(newDeviceList) }, tx);
if (user.email) {
await smtpService.sendMail({
template: SmtpTemplates.NewDeviceJoin,
@@ -97,30 +98,36 @@ export const authLoginServiceFactory = ({
* Check user device and send mail if new device
* generate the auth and refresh token. fn shared by mfa verification and login verification with mfa disabled
*/
const generateUserTokens = async ({
user,
ip,
userAgent,
organizationId,
authMethod,
isMfaVerified,
mfaMethod
}: {
user: TUsers;
ip: string;
userAgent: string;
organizationId?: string;
authMethod: AuthMethod;
isMfaVerified?: boolean;
mfaMethod?: MfaMethod;
}) => {
const cfg = getConfig();
await updateUserDeviceSession(user, ip, userAgent);
const tokenSession = await tokenService.getUserTokenSession({
userAgent,
const generateUserTokens = async (
{
user,
ip,
userId: user.id
});
userAgent,
organizationId,
authMethod,
isMfaVerified,
mfaMethod
}: {
user: TUsers;
ip: string;
userAgent: string;
organizationId?: string;
authMethod: AuthMethod;
isMfaVerified?: boolean;
mfaMethod?: MfaMethod;
},
tx?: Knex
) => {
const cfg = getConfig();
await updateUserDeviceSession(user, ip, userAgent, tx);
const tokenSession = await tokenService.getUserTokenSession(
{
userAgent,
ip,
userId: user.id
},
tx
);
if (!tokenSession) throw new Error("Failed to create token");
const accessToken = jwt.sign(

View File

@@ -31,11 +31,13 @@ import { BadRequestError, ForbiddenRequestError, NotFoundError, UnauthorizedErro
import { groupBy } from "@app/lib/fn";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { isDisposableEmail } from "@app/lib/validator";
import { TQueueServiceFactory } from "@app/queue";
import { getDefaultOrgMembershipRoleForUpdateOrg } from "@app/services/org/org-role-fns";
import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal";
import { TUserAliasDALFactory } from "@app/services/user-alias/user-alias-dal";
import { ActorAuthMethod, ActorType, AuthMethod, AuthTokenType } from "../auth/auth-type";
import { TAuthLoginFactory } from "../auth/auth-login-service";
import { ActorAuthMethod, ActorType, AuthMethod, AuthModeJwtTokenPayload, AuthTokenType } from "../auth/auth-type";
import { TAuthTokenServiceFactory } from "../auth-token/auth-token-service";
import { TokenType } from "../auth-token/auth-token-types";
import { TIdentityMetadataDALFactory } from "../identity/identity-metadata-dal";
@@ -47,6 +49,10 @@ import { TProjectKeyDALFactory } from "../project-key/project-key-dal";
import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal";
import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal";
import { TProjectRoleDALFactory } from "../project-role/project-role-dal";
import { TSecretDALFactory } from "../secret/secret-dal";
import { fnDeleteProjectSecretReminders } from "../secret/secret-fns";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service";
import { TUserDALFactory } from "../user/user-dal";
import { TIncidentContactsDALFactory } from "./incident-contacts-dal";
@@ -69,6 +75,9 @@ import {
type TOrgServiceFactoryDep = {
userAliasDAL: Pick<TUserAliasDALFactory, "delete">;
secretDAL: Pick<TSecretDALFactory, "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find">;
folderDAL: Pick<TSecretFolderDALFactory, "findByProjectId">;
orgDAL: TOrgDALFactory;
orgBotDAL: TOrgBotDALFactory;
orgRoleDAL: TOrgRoleDALFactory;
@@ -97,6 +106,8 @@ type TOrgServiceFactoryDep = {
projectBotDAL: Pick<TProjectBotDALFactory, "findOne" | "updateById">;
projectUserMembershipRoleDAL: Pick<TProjectUserMembershipRoleDALFactory, "insertMany" | "create">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
queueService: Pick<TQueueServiceFactory, "stopRepeatableJob">;
loginService: Pick<TAuthLoginFactory, "generateUserTokens">;
};
export type TOrgServiceFactory = ReturnType<typeof orgServiceFactory>;
@@ -104,6 +115,9 @@ export type TOrgServiceFactory = ReturnType<typeof orgServiceFactory>;
export const orgServiceFactory = ({
userAliasDAL,
orgDAL,
secretDAL,
secretV2BridgeDAL,
folderDAL,
userDAL,
groupDAL,
orgRoleDAL,
@@ -124,7 +138,9 @@ export const orgServiceFactory = ({
projectBotDAL,
projectUserMembershipRoleDAL,
identityMetadataDAL,
projectBotService
projectBotService,
queueService,
loginService
}: TOrgServiceFactoryDep) => {
/*
* Get organization details by the organization id
@@ -419,24 +435,88 @@ export const orgServiceFactory = ({
/*
* Delete organization by id
* */
const deleteOrganizationById = async (
userId: string,
orgId: string,
actorAuthMethod: ActorAuthMethod,
actorOrgId: string | undefined
) => {
const deleteOrganizationById = async ({
userId,
authorizationHeader,
userAgentHeader,
ipAddress,
orgId,
actorAuthMethod,
actorOrgId
}: {
userId: string;
authorizationHeader?: string;
userAgentHeader?: string;
ipAddress: string;
orgId: string;
actorAuthMethod: ActorAuthMethod;
actorOrgId: string | undefined;
}) => {
const { membership } = await permissionService.getUserOrgPermission(userId, orgId, actorAuthMethod, actorOrgId);
if ((membership.role as OrgMembershipRole) !== OrgMembershipRole.Admin)
if ((membership.role as OrgMembershipRole) !== OrgMembershipRole.Admin) {
throw new ForbiddenRequestError({
name: "DeleteOrganizationById",
message: "Insufficient privileges"
});
const organization = await orgDAL.deleteById(orgId);
if (organization.customerId) {
await licenseService.removeOrgCustomer(organization.customerId);
}
return organization;
if (!authorizationHeader) {
throw new UnauthorizedError({ name: "Authorization header not set on request." });
}
if (!userAgentHeader) {
throw new BadRequestError({ name: "User agent not set on request." });
}
const cfg = getConfig();
const authToken = authorizationHeader.replace("Bearer ", "");
const decodedToken = jwt.verify(authToken, cfg.AUTH_SECRET) as AuthModeJwtTokenPayload;
if (!decodedToken.authMethod) throw new UnauthorizedError({ name: "Auth method not found on existing token" });
const response = await orgDAL.transaction(async (tx) => {
const projects = await projectDAL.find({ orgId }, { tx });
for await (const project of projects) {
await fnDeleteProjectSecretReminders(project.id, {
secretDAL,
secretV2BridgeDAL,
queueService,
projectBotService,
folderDAL
});
}
const deletedOrg = await orgDAL.deleteById(orgId, tx);
if (deletedOrg.customerId) {
await licenseService.removeOrgCustomer(deletedOrg.customerId);
}
// Generate new tokens without the organization ID present
const user = await userDAL.findById(userId, tx);
const { access: accessToken, refresh: refreshToken } = await loginService.generateUserTokens(
{
user,
authMethod: decodedToken.authMethod,
ip: ipAddress,
userAgent: userAgentHeader,
isMfaVerified: decodedToken.isMfaVerified,
mfaMethod: decodedToken.mfaMethod
},
tx
);
return {
organization: deletedOrg,
tokens: {
accessToken,
refreshToken
}
};
});
return response;
};
/*
* Org membership management

View File

@@ -14,6 +14,7 @@ import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/
import { groupBy } from "@app/lib/fn";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { TProjectPermission } from "@app/lib/types";
import { TQueueServiceFactory } from "@app/queue";
import { ActorType } from "../auth/auth-type";
import { TCertificateDALFactory } from "../certificate/certificate-dal";
@@ -28,13 +29,17 @@ import { TOrgServiceFactory } from "../org/org-service";
import { TPkiAlertDALFactory } from "../pki-alert/pki-alert-dal";
import { TPkiCollectionDALFactory } from "../pki-collection/pki-collection-dal";
import { TProjectBotDALFactory } from "../project-bot/project-bot-dal";
import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { TProjectKeyDALFactory } from "../project-key/project-key-dal";
import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal";
import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal";
import { TProjectRoleDALFactory } from "../project-role/project-role-dal";
import { getPredefinedRoles } from "../project-role/project-role-fns";
import { TSecretDALFactory } from "../secret/secret-dal";
import { fnDeleteProjectSecretReminders } from "../secret/secret-fns";
import { ROOT_FOLDER_NAME, TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { TProjectSlackConfigDALFactory } from "../slack/project-slack-config-dal";
import { TSlackIntegrationDALFactory } from "../slack/slack-integration-dal";
import { TUserDALFactory } from "../user/user-dal";
@@ -74,7 +79,10 @@ type TProjectServiceFactoryDep = {
projectDAL: TProjectDALFactory;
projectQueue: TProjectQueueFactory;
userDAL: TUserDALFactory;
folderDAL: TSecretFolderDALFactory;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
folderDAL: Pick<TSecretFolderDALFactory, "insertMany" | "findByProjectId">;
secretDAL: Pick<TSecretDALFactory, "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find">;
projectEnvDAL: Pick<TProjectEnvDALFactory, "insertMany" | "find">;
identityOrgMembershipDAL: TIdentityOrgDALFactory;
identityProjectDAL: TIdentityProjectDALFactory;
@@ -92,6 +100,8 @@ type TProjectServiceFactoryDep = {
permissionService: TPermissionServiceFactory;
orgService: Pick<TOrgServiceFactory, "addGhostUser">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
queueService: Pick<TQueueServiceFactory, "stopRepeatableJob">;
orgDAL: Pick<TOrgDALFactory, "findOne">;
keyStore: Pick<TKeyStoreFactory, "deleteItem">;
projectBotDAL: Pick<TProjectBotDALFactory, "create">;
@@ -112,9 +122,13 @@ export type TProjectServiceFactory = ReturnType<typeof projectServiceFactory>;
export const projectServiceFactory = ({
projectDAL,
secretDAL,
secretV2BridgeDAL,
projectQueue,
projectKeyDAL,
permissionService,
queueService,
projectBotService,
orgDAL,
userDAL,
folderDAL,
@@ -424,6 +438,14 @@ export const projectServiceFactory = ({
await userDAL.deleteById(projectGhostUser.id, tx);
}
await fnDeleteProjectSecretReminders(project.id, {
secretDAL,
secretV2BridgeDAL,
queueService,
projectBotService,
folderDAL
});
return delProject;
});

View File

@@ -5,6 +5,7 @@ import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identity-access-token-dal";
import { TIdentityUaClientSecretDALFactory } from "../identity-ua/identity-ua-client-secret-dal";
import { TSecretDALFactory } from "../secret/secret-dal";
import { TSecretVersionDALFactory } from "../secret/secret-version-dal";
import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal";
import { TSecretSharingDALFactory } from "../secret-sharing/secret-sharing-dal";
@@ -16,6 +17,7 @@ type TDailyResourceCleanUpQueueServiceFactoryDep = {
identityUniversalAuthClientSecretDAL: Pick<TIdentityUaClientSecretDALFactory, "removeExpiredClientSecrets">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "pruneExcessVersions">;
secretVersionV2DAL: Pick<TSecretVersionV2DALFactory, "pruneExcessVersions">;
secretDAL: Pick<TSecretDALFactory, "pruneSecretReminders">;
secretFolderVersionDAL: Pick<TSecretFolderVersionDALFactory, "pruneExcessVersions">;
snapshotDAL: Pick<TSnapshotDALFactory, "pruneExcessSnapshots">;
secretSharingDAL: Pick<TSecretSharingDALFactory, "pruneExpiredSharedSecrets">;
@@ -30,6 +32,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
snapshotDAL,
secretVersionDAL,
secretFolderVersionDAL,
secretDAL,
identityAccessTokenDAL,
secretSharingDAL,
secretVersionV2DAL,
@@ -37,6 +40,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
}: TDailyResourceCleanUpQueueServiceFactoryDep) => {
queueService.start(QueueName.DailyResourceCleanUp, async () => {
logger.info(`${QueueName.DailyResourceCleanUp}: queue task started`);
await secretDAL.pruneSecretReminders(queueService);
await auditLogDAL.pruneAuditLog();
await identityAccessTokenDAL.removeExpiredTokens();
await identityUniversalAuthClientSecretDAL.removeExpiredClientSecrets();

View File

@@ -5,6 +5,8 @@ import { TDbClient } from "@app/db";
import { SecretsSchema, SecretType, TableName, TSecrets, TSecretsUpdate } from "@app/db/schemas";
import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
import { logger } from "@app/lib/logger";
import { QueueName, TQueueServiceFactory } from "@app/queue";
export type TSecretDALFactory = ReturnType<typeof secretDALFactory>;
@@ -339,6 +341,94 @@ export const secretDALFactory = (db: TDbClient) => {
}
};
const pruneSecretReminders = async (queueService: TQueueServiceFactory) => {
const REMINDER_PRUNE_BATCH_SIZE = 5_000;
const MAX_RETRY_ON_FAILURE = 3;
let numberOfRetryOnFailure = 0;
let deletedReminderCount = 0;
logger.info(`${QueueName.DailyResourceCleanUp}: secret reminders started`);
try {
const repeatableJobs = await queueService.getRepeatableJobs(QueueName.SecretReminder);
const reminderJobs = repeatableJobs
.map((job) => ({ secretId: job.id?.replace("reminder-", "") as string, jobKey: job.key }))
.filter(Boolean);
if (reminderJobs.length === 0) {
logger.info(`${QueueName.DailyResourceCleanUp}: no reminder jobs found`);
return;
}
for (let offset = 0; offset < reminderJobs.length; offset += REMINDER_PRUNE_BATCH_SIZE) {
try {
const batchIds = reminderJobs.slice(offset, offset + REMINDER_PRUNE_BATCH_SIZE).map((r) => r.secretId);
const payload = {
$in: {
id: batchIds
}
};
const opts = {
limit: REMINDER_PRUNE_BATCH_SIZE
};
// Find existing secrets with pagination
// eslint-disable-next-line no-await-in-loop
const [secrets, secretsV2] = await Promise.all([
ormify(db, TableName.Secret).find(payload, opts),
ormify(db, TableName.SecretV2).find(payload, opts)
]);
const foundSecretIds = new Set([
...secrets.map((secret) => secret.id),
...secretsV2.map((secret) => secret.id)
]);
// Find IDs that don't exist in either table
const secretIdsNotFound = batchIds.filter((secretId) => !foundSecretIds.has(secretId));
// Delete reminders for non-existent secrets
for (const secretId of secretIdsNotFound) {
const jobKey = reminderJobs.find((r) => r.secretId === secretId)?.jobKey;
if (jobKey) {
// eslint-disable-next-line no-await-in-loop
await queueService.stopRepeatableJobByKey(QueueName.SecretReminder, jobKey);
deletedReminderCount += 1;
}
}
numberOfRetryOnFailure = 0;
} catch (error) {
numberOfRetryOnFailure += 1;
logger.error(error, `Failed to process batch at offset ${offset}`);
if (numberOfRetryOnFailure >= MAX_RETRY_ON_FAILURE) {
break;
}
// Retry the current batch
offset -= REMINDER_PRUNE_BATCH_SIZE;
// eslint-disable-next-line no-promise-executor-return, @typescript-eslint/no-loop-func, no-await-in-loop
await new Promise((resolve) => setTimeout(resolve, 500 * numberOfRetryOnFailure));
}
// Small delay between batches
// eslint-disable-next-line no-promise-executor-return, @typescript-eslint/no-loop-func, no-await-in-loop
await new Promise((resolve) => setTimeout(resolve, 10));
}
} catch (error) {
logger.error(error, "Failed to complete secret reminder pruning");
} finally {
logger.info(
`${QueueName.DailyResourceCleanUp}: secret reminders completed. Deleted ${deletedReminderCount} reminders`
);
}
};
return {
...secretOrm,
update,
@@ -352,6 +442,7 @@ export const secretDALFactory = (db: TDbClient) => {
findByBlindIndexes,
upsertSecretReferences,
findReferencedSecretReferences,
findAllProjectSecretValues
findAllProjectSecretValues,
pruneSecretReminders
};
};

View File

@@ -19,9 +19,11 @@ import {
decryptSymmetric128BitHexKeyUTF8,
encryptSymmetric128BitHexKeyUTF8
} from "@app/lib/crypto";
import { daysToMillisecond, secondsToMillis } from "@app/lib/dates";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { groupBy, unique } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import {
fnSecretBulkInsert as fnSecretV2BridgeBulkInsert,
fnSecretBulkUpdate as fnSecretV2BridgeBulkUpdate,
@@ -31,8 +33,10 @@ import {
import { ActorAuthMethod, ActorType } from "../auth/auth-type";
import { KmsDataKey } from "../kms/kms-types";
import { getBotKeyFnFactory } from "../project-bot/project-bot-fns";
import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { TSecretDALFactory } from "./secret-dal";
import {
TCreateManySecretsRawFn,
@@ -1138,3 +1142,49 @@ export const decryptSecretWithBot = (
secretComment
};
};
type TFnDeleteProjectSecretReminders = {
secretDAL: Pick<TSecretDALFactory, "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find">;
queueService: Pick<TQueueServiceFactory, "stopRepeatableJob">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
folderDAL: Pick<TSecretFolderDALFactory, "findByProjectId">;
};
export const fnDeleteProjectSecretReminders = async (
projectId: string,
{ secretDAL, secretV2BridgeDAL, queueService, projectBotService, folderDAL }: TFnDeleteProjectSecretReminders
) => {
const projectFolders = await folderDAL.findByProjectId(projectId);
const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId, false);
const projectSecrets = shouldUseSecretV2Bridge
? await secretV2BridgeDAL.find({
$in: { folderId: projectFolders.map((folder) => folder.id) },
$notNull: ["reminderRepeatDays"]
})
: await secretDAL.find({
$in: { folderId: projectFolders.map((folder) => folder.id) },
$notNull: ["secretReminderRepeatDays"]
});
const appCfg = getConfig();
for await (const secret of projectSecrets) {
const repeatDays = shouldUseSecretV2Bridge
? (secret as { reminderRepeatDays: number }).reminderRepeatDays
: (secret as { secretReminderRepeatDays: number }).secretReminderRepeatDays;
// We're using the queue service directly to get around conflicting imports.
if (repeatDays) {
await queueService.stopRepeatableJob(
QueueName.SecretReminder,
QueueJobs.SecretReminder,
{
// on prod it this will be in days, in development this will be second
every: appCfg.NODE_ENV === "development" ? secondsToMillis(repeatDays) : daysToMillisecond(repeatDays)
},
`reminder-${secret.id}`
);
}
}
};

View File

@@ -248,7 +248,9 @@ export const secretQueueFactory = ({
? secondsToMillis(newSecret.secretReminderRepeatDays)
: daysToMillisecond(newSecret.secretReminderRepeatDays),
immediately: true
}
},
removeOnComplete: true,
removeOnFail: true
}
);
} catch (err) {

View File

@@ -491,8 +491,8 @@ export const secretServiceFactory = ({
secretDAL
});
const deletedSecret = await secretDAL.transaction(async (tx) =>
fnSecretBulkDelete({
const deletedSecret = await secretDAL.transaction(async (tx) => {
const secrets = await fnSecretBulkDelete({
projectId,
folderId,
actorId,
@@ -505,8 +505,19 @@ export const secretServiceFactory = ({
}
],
tx
})
);
});
for await (const secret of secrets) {
if (secret.secretReminderRepeatDays !== null && secret.secretReminderRepeatDays !== undefined) {
await secretQueueService.removeSecretReminder({
repeatDays: secret.secretReminderRepeatDays,
secretId: secret.id
});
}
}
return secrets;
});
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
@@ -971,8 +982,8 @@ export const secretServiceFactory = ({
secretDAL
});
const secretsDeleted = await secretDAL.transaction(async (tx) =>
fnSecretBulkDelete({
const secretsDeleted = await secretDAL.transaction(async (tx) => {
const secrets = await fnSecretBulkDelete({
secretDAL,
secretQueueService,
inputSecrets: inputSecrets.map(({ type, secretName }) => ({
@@ -983,8 +994,19 @@ export const secretServiceFactory = ({
folderId,
actorId,
tx
})
);
});
for await (const secret of secrets) {
if (secret.secretReminderRepeatDays !== null && secret.secretReminderRepeatDays !== undefined) {
await secretQueueService.removeSecretReminder({
repeatDays: secret.secretReminderRepeatDays,
secretId: secret.id
});
}
}
return secrets;
});
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({

View File

@@ -77,11 +77,16 @@ export const selectOrganization = async (data: {
export const useSelectOrganization = () => {
const queryClient = useQueryClient();
return useMutation({
mutationFn: async (details: { organizationId: string; userAgent?: UserAgentType }) => {
mutationFn: async (details: {
organizationId: string;
userAgent?: UserAgentType;
forceSetCredentials?: boolean;
}) => {
const data = await selectOrganization(details);
// If a custom user agent is set, then this session is meant for another consuming application, not the web application.
if (!details.userAgent && !data.isMfaEnabled) {
if ((!details.userAgent && !data.isMfaEnabled) || details.forceSetCredentials) {
localStorage.setItem("orgData.id", details.organizationId);
SecurityClient.setToken(data.token);
SecurityClient.setProviderAuthToken("");
}

View File

@@ -1,5 +1,6 @@
import { useMutation, useQuery, useQueryClient, UseQueryOptions } from "@tanstack/react-query";
import SecurityClient from "@app/components/utilities/SecurityClient";
import { apiRequest } from "@app/config/request";
import { OrderByDirection } from "@app/hooks/api/generic/types";
@@ -67,7 +68,7 @@ export const useCreateOrg = (options: { invalidate: boolean } = { invalidate: tr
mutationFn: async ({ name }: { name: string }) => {
const {
data: { organization }
} = await apiRequest.post("/api/v2/organizations", {
} = await apiRequest.post<{ organization: { id: string } }>("/api/v2/organizations", {
name
});
@@ -437,10 +438,13 @@ export const useDeleteOrgById = () => {
return useMutation({
mutationFn: async ({ organizationId }: { organizationId: string }) => {
const {
data: { organization }
} = await apiRequest.delete<{ organization: Organization }>(
data: { organization, accessToken }
} = await apiRequest.delete<{ organization: Organization; accessToken: string }>(
`/api/v2/organizations/${organizationId}`
);
SecurityClient.setToken(accessToken);
localStorage.removeItem("orgData.id");
return organization;
},
onSuccess(_, dto) {

View File

@@ -7,7 +7,7 @@ import { ProjectType } from "@app/hooks/api/workspace/types";
import { queryClient } from "@app/reactQuery";
export const navigateUserToOrg = async (router: NextRouter, organizationId?: string) => {
const userOrgs = await fetchOrganizations();
const userOrgs = await fetchOrganizations().catch(() => []);
const nonAuthEnforcedOrgs = userOrgs.filter((org) => !org.authEnforced);

View File

@@ -6,7 +6,7 @@ import z from "zod";
import { createNotification } from "@app/components/notifications";
import { Button, FormControl, Input, Modal, ModalContent } from "@app/components/v2";
import { useCreateOrg, useSelectOrganization } from "@app/hooks/api";
import { useCreateOrg, useGetOrganizations, useSelectOrganization } from "@app/hooks/api";
import { ProjectType } from "@app/hooks/api/workspace/types";
const schema = z
@@ -23,9 +23,10 @@ interface CreateOrgModalProps {
}
export const CreateOrgModal: FC<CreateOrgModalProps> = ({ isOpen, onClose }) => {
const router = useRouter();
const { refetch: refetchOrganizations } = useGetOrganizations();
const {
control,
handleSubmit,
@@ -50,19 +51,21 @@ export const CreateOrgModal: FC<CreateOrgModalProps> = ({ isOpen, onClose }) =>
});
await selectOrg({
organizationId: organization.id
organizationId: organization.id,
forceSetCredentials: true
});
await refetchOrganizations();
createNotification({
text: "Successfully created organization",
type: "success"
});
if (router.isReady) router.push(`/org/${organization.id}/${ProjectType.SecretManager}/overview`);
if (router.isReady)
router.push(`/org/${organization.id}/${ProjectType.SecretManager}/overview`);
else window.location.href = `/org/${organization.id}/${ProjectType.SecretManager}/overview`;
localStorage.setItem("orgData.id", organization.id);
reset();
onClose();
} catch (err) {