merge main

This commit is contained in:
Scott Wilson
2024-12-18 10:09:23 -08:00
165 changed files with 10179 additions and 602 deletions
@@ -12,9 +12,12 @@ export type TTokenDALFactory = ReturnType<typeof tokenDALFactory>;
export const tokenDALFactory = (db: TDbClient) => {
const authOrm = ormify(db, TableName.AuthTokens);
const findOneTokenSession = async (filter: Partial<TAuthTokenSessions>): Promise<TAuthTokenSessions | undefined> => {
const findOneTokenSession = async (
filter: Partial<TAuthTokenSessions>,
tx?: Knex
): Promise<TAuthTokenSessions | undefined> => {
try {
const doc = await db.replicaNode()(TableName.AuthTokenSession).where(filter).first();
const doc = await (tx || db.replicaNode())(TableName.AuthTokenSession).where(filter).first();
return doc;
} catch (error) {
throw new DatabaseError({ error, name: "FindOneTokenSession" });
@@ -54,10 +57,11 @@ export const tokenDALFactory = (db: TDbClient) => {
const insertTokenSession = async (
userId: string,
ip: string,
userAgent: string
userAgent: string,
tx?: Knex
): Promise<TAuthTokenSessions | undefined> => {
try {
const [session] = await db(TableName.AuthTokenSession)
const [session] = await (tx || db)(TableName.AuthTokenSession)
.insert({
userId,
ip,
@@ -1,6 +1,7 @@
import crypto from "node:crypto";
import bcrypt from "bcrypt";
import { Knex } from "knex";
import { TAuthTokens, TAuthTokenSessions } from "@app/db/schemas";
import { getConfig } from "@app/lib/config/env";
@@ -123,14 +124,13 @@ export const tokenServiceFactory = ({ tokenDAL, userDAL, orgMembershipDAL }: TAu
return deletedToken?.[0];
};
const getUserTokenSession = async ({
userId,
ip,
userAgent
}: TIssueAuthTokenDTO): Promise<TAuthTokenSessions | undefined> => {
let session = await tokenDAL.findOneTokenSession({ userId, ip, userAgent });
const getUserTokenSession = async (
{ userId, ip, userAgent }: TIssueAuthTokenDTO,
tx?: Knex
): Promise<TAuthTokenSessions | undefined> => {
let session = await tokenDAL.findOneTokenSession({ userId, ip, userAgent }, tx);
if (!session) {
session = await tokenDAL.insertTokenSession(userId, ip, userAgent);
session = await tokenDAL.insertTokenSession(userId, ip, userAgent, tx);
}
return session;
};
+32 -25
View File
@@ -1,5 +1,6 @@
import bcrypt from "bcrypt";
import jwt from "jsonwebtoken";
import { Knex } from "knex";
import { TUsers, UserDeviceSchema } from "@app/db/schemas";
import { isAuthMethodSaml } from "@app/ee/services/permission/permission-fns";
@@ -50,13 +51,13 @@ export const authLoginServiceFactory = ({
* Not exported. This is to update user device list
* If new device is found. Will be saved and a mail will be send
*/
const updateUserDeviceSession = async (user: TUsers, ip: string, userAgent: string) => {
const updateUserDeviceSession = async (user: TUsers, ip: string, userAgent: string, tx?: Knex) => {
const devices = await UserDeviceSchema.parseAsync(user.devices || []);
const isDeviceSeen = devices.some((device) => device.ip === ip && device.userAgent === userAgent);
if (!isDeviceSeen) {
const newDeviceList = devices.concat([{ ip, userAgent }]);
await userDAL.updateById(user.id, { devices: JSON.stringify(newDeviceList) });
await userDAL.updateById(user.id, { devices: JSON.stringify(newDeviceList) }, tx);
if (user.email) {
await smtpService.sendMail({
template: SmtpTemplates.NewDeviceJoin,
@@ -97,30 +98,36 @@ export const authLoginServiceFactory = ({
* Check user device and send mail if new device
* generate the auth and refresh token. fn shared by mfa verification and login verification with mfa disabled
*/
const generateUserTokens = async ({
user,
ip,
userAgent,
organizationId,
authMethod,
isMfaVerified,
mfaMethod
}: {
user: TUsers;
ip: string;
userAgent: string;
organizationId?: string;
authMethod: AuthMethod;
isMfaVerified?: boolean;
mfaMethod?: MfaMethod;
}) => {
const cfg = getConfig();
await updateUserDeviceSession(user, ip, userAgent);
const tokenSession = await tokenService.getUserTokenSession({
userAgent,
const generateUserTokens = async (
{
user,
ip,
userId: user.id
});
userAgent,
organizationId,
authMethod,
isMfaVerified,
mfaMethod
}: {
user: TUsers;
ip: string;
userAgent: string;
organizationId?: string;
authMethod: AuthMethod;
isMfaVerified?: boolean;
mfaMethod?: MfaMethod;
},
tx?: Knex
) => {
const cfg = getConfig();
await updateUserDeviceSession(user, ip, userAgent, tx);
const tokenSession = await tokenService.getUserTokenSession(
{
userAgent,
ip,
userId: user.id
},
tx
);
if (!tokenSession) throw new Error("Failed to create token");
const accessToken = jwt.sign(
@@ -15,7 +15,7 @@ import {
/* eslint-disable no-bitwise */
export const createSerialNumber = () => {
const randomBytes = crypto.randomBytes(20);
const randomBytes = crypto.randomBytes(20); // 20 bytes = 160 bits
randomBytes[0] &= 0x7f; // ensure the first bit is 0
return randomBytes.toString("hex");
};
@@ -0,0 +1,35 @@
export const isAzureKeyVaultReference = (uri: string) => {
const tryJsonDecode = () => {
try {
return (JSON.parse(uri) as { uri: string }).uri || uri;
} catch {
return uri;
}
};
const cleanUri = tryJsonDecode();
if (!cleanUri.startsWith("https://")) {
return false;
}
if (!cleanUri.includes(".vault.azure.net/secrets/")) {
return false;
}
// 3. Check for non-empty string between https:// and .vault.azure.net/secrets/
const parts = cleanUri.split(".vault.azure.net/secrets/");
const vaultName = parts[0].replace("https://", "");
if (!vaultName) {
return false;
}
// 4. Check for non-empty secret name
const secretParts = parts[1].split("/");
const secretName = secretParts[0];
if (!secretName) {
return false;
}
return true;
};
@@ -51,6 +51,7 @@ import {
Integrations,
IntegrationUrls
} from "./integration-list";
import { isAzureKeyVaultReference } from "./integration-sync-secret-fns";
const getSecretKeyValuePair = (secrets: Record<string, { value: string | null; comment?: string } | null>) =>
Object.keys(secrets).reduce<Record<string, string | null | undefined>>((prev, key) => {
@@ -325,11 +326,12 @@ const syncSecretsAzureAppConfig = async ({
};
const metadata = IntegrationMetadataSchema.parse(integration.metadata);
const azureAppConfigSecrets = (
await getCompleteAzureAppConfigValues(
`${integration.app}/kv?api-version=2023-11-01&key=${metadata.secretPrefix || ""}*`
)
).reduce(
const azureAppConfigValuesUrl = `${integration.app}/kv?api-version=2023-11-01&key=${metadata.secretPrefix}*${
metadata.azureLabel ? `&label=${metadata.azureLabel}` : ""
}`;
const azureAppConfigSecrets = (await getCompleteAzureAppConfigValues(azureAppConfigValuesUrl)).reduce(
(accum, entry) => {
accum[entry.key] = entry.value;
@@ -410,14 +412,24 @@ const syncSecretsAzureAppConfig = async ({
}
// create or update secrets on Azure App Config
for await (const key of Object.keys(secrets)) {
if (!(key in azureAppConfigSecrets) || secrets[key]?.value !== azureAppConfigSecrets[key]) {
await request.put(
`${integration.app}/kv/${key}?api-version=2023-11-01`,
{
value: secrets[key]?.value
value: secrets[key]?.value,
...(isAzureKeyVaultReference(secrets[key]?.value || "") && {
content_type: "application/vnd.microsoft.appconfig.keyvaultref+json;charset=utf-8"
})
},
{
...(metadata.azureLabel && {
params: {
label: metadata.azureLabel
}
}),
headers: {
Authorization: `Bearer ${accessToken}`
},
@@ -437,6 +449,11 @@ const syncSecretsAzureAppConfig = async ({
headers: {
Authorization: `Bearer ${accessToken}`
},
...(metadata.azureLabel && {
params: {
label: metadata.azureLabel
}
}),
// we force IPV4 because docker setup fails with ipv6
httpsAgent: new https.Agent({
family: 4
@@ -35,6 +35,8 @@ export const IntegrationMetadataSchema = z.object({
.optional()
.describe(INTEGRATION.CREATE.metadata.secretAWSTag),
azureLabel: z.string().optional().describe(INTEGRATION.CREATE.metadata.azureLabel),
githubVisibility: z
.union([z.literal("selected"), z.literal("private"), z.literal("all")])
.optional()
+16
View File
@@ -0,0 +1,16 @@
import { OrganizationsSchema } from "@app/db/schemas";
export const sanitizedOrganizationSchema = OrganizationsSchema.pick({
id: true,
name: true,
customerId: true,
slug: true,
createdAt: true,
updatedAt: true,
authEnforced: true,
scimEnabled: true,
kmsDefaultKeyId: true,
defaultMembershipRole: true,
enforceMfa: true,
selectedMfaMethod: true
});
+94 -14
View File
@@ -31,11 +31,13 @@ import { BadRequestError, ForbiddenRequestError, NotFoundError, UnauthorizedErro
import { groupBy } from "@app/lib/fn";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { isDisposableEmail } from "@app/lib/validator";
import { TQueueServiceFactory } from "@app/queue";
import { getDefaultOrgMembershipRoleForUpdateOrg } from "@app/services/org/org-role-fns";
import { TOrgMembershipDALFactory } from "@app/services/org-membership/org-membership-dal";
import { TUserAliasDALFactory } from "@app/services/user-alias/user-alias-dal";
import { ActorAuthMethod, ActorType, AuthMethod, AuthTokenType } from "../auth/auth-type";
import { TAuthLoginFactory } from "../auth/auth-login-service";
import { ActorAuthMethod, ActorType, AuthMethod, AuthModeJwtTokenPayload, AuthTokenType } from "../auth/auth-type";
import { TAuthTokenServiceFactory } from "../auth-token/auth-token-service";
import { TokenType } from "../auth-token/auth-token-types";
import { TIdentityMetadataDALFactory } from "../identity/identity-metadata-dal";
@@ -47,6 +49,10 @@ import { TProjectKeyDALFactory } from "../project-key/project-key-dal";
import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal";
import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal";
import { TProjectRoleDALFactory } from "../project-role/project-role-dal";
import { TSecretDALFactory } from "../secret/secret-dal";
import { fnDeleteProjectSecretReminders } from "../secret/secret-fns";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service";
import { TUserDALFactory } from "../user/user-dal";
import { TIncidentContactsDALFactory } from "./incident-contacts-dal";
@@ -69,6 +75,9 @@ import {
type TOrgServiceFactoryDep = {
userAliasDAL: Pick<TUserAliasDALFactory, "delete">;
secretDAL: Pick<TSecretDALFactory, "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find">;
folderDAL: Pick<TSecretFolderDALFactory, "findByProjectId">;
orgDAL: TOrgDALFactory;
orgBotDAL: TOrgBotDALFactory;
orgRoleDAL: TOrgRoleDALFactory;
@@ -97,6 +106,8 @@ type TOrgServiceFactoryDep = {
projectBotDAL: Pick<TProjectBotDALFactory, "findOne" | "updateById">;
projectUserMembershipRoleDAL: Pick<TProjectUserMembershipRoleDALFactory, "insertMany" | "create">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
queueService: Pick<TQueueServiceFactory, "stopRepeatableJob">;
loginService: Pick<TAuthLoginFactory, "generateUserTokens">;
};
export type TOrgServiceFactory = ReturnType<typeof orgServiceFactory>;
@@ -104,6 +115,9 @@ export type TOrgServiceFactory = ReturnType<typeof orgServiceFactory>;
export const orgServiceFactory = ({
userAliasDAL,
orgDAL,
secretDAL,
secretV2BridgeDAL,
folderDAL,
userDAL,
groupDAL,
orgRoleDAL,
@@ -124,7 +138,9 @@ export const orgServiceFactory = ({
projectBotDAL,
projectUserMembershipRoleDAL,
identityMetadataDAL,
projectBotService
projectBotService,
queueService,
loginService
}: TOrgServiceFactoryDep) => {
/*
* Get organization details by the organization id
@@ -419,24 +435,88 @@ export const orgServiceFactory = ({
/*
* Delete organization by id
* */
const deleteOrganizationById = async (
userId: string,
orgId: string,
actorAuthMethod: ActorAuthMethod,
actorOrgId: string | undefined
) => {
const deleteOrganizationById = async ({
userId,
authorizationHeader,
userAgentHeader,
ipAddress,
orgId,
actorAuthMethod,
actorOrgId
}: {
userId: string;
authorizationHeader?: string;
userAgentHeader?: string;
ipAddress: string;
orgId: string;
actorAuthMethod: ActorAuthMethod;
actorOrgId: string | undefined;
}) => {
const { membership } = await permissionService.getUserOrgPermission(userId, orgId, actorAuthMethod, actorOrgId);
if ((membership.role as OrgMembershipRole) !== OrgMembershipRole.Admin)
if ((membership.role as OrgMembershipRole) !== OrgMembershipRole.Admin) {
throw new ForbiddenRequestError({
name: "DeleteOrganizationById",
message: "Insufficient privileges"
});
const organization = await orgDAL.deleteById(orgId);
if (organization.customerId) {
await licenseService.removeOrgCustomer(organization.customerId);
}
return organization;
if (!authorizationHeader) {
throw new UnauthorizedError({ name: "Authorization header not set on request." });
}
if (!userAgentHeader) {
throw new BadRequestError({ name: "User agent not set on request." });
}
const cfg = getConfig();
const authToken = authorizationHeader.replace("Bearer ", "");
const decodedToken = jwt.verify(authToken, cfg.AUTH_SECRET) as AuthModeJwtTokenPayload;
if (!decodedToken.authMethod) throw new UnauthorizedError({ name: "Auth method not found on existing token" });
const response = await orgDAL.transaction(async (tx) => {
const projects = await projectDAL.find({ orgId }, { tx });
for await (const project of projects) {
await fnDeleteProjectSecretReminders(project.id, {
secretDAL,
secretV2BridgeDAL,
queueService,
projectBotService,
folderDAL
});
}
const deletedOrg = await orgDAL.deleteById(orgId, tx);
if (deletedOrg.customerId) {
await licenseService.removeOrgCustomer(deletedOrg.customerId);
}
// Generate new tokens without the organization ID present
const user = await userDAL.findById(userId, tx);
const { access: accessToken, refresh: refreshToken } = await loginService.generateUserTokens(
{
user,
authMethod: decodedToken.authMethod,
ip: ipAddress,
userAgent: userAgentHeader,
isMfaVerified: decodedToken.isMfaVerified,
mfaMethod: decodedToken.mfaMethod
},
tx
);
return {
organization: deletedOrg,
tokens: {
accessToken,
refreshToken
}
};
});
return response;
};
/*
* Org membership management
+156 -2
View File
@@ -8,12 +8,16 @@ import { TPermissionServiceFactory } from "@app/ee/services/permission/permissio
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
import { TProjectTemplateServiceFactory } from "@app/ee/services/project-template/project-template-service";
import { InfisicalProjectTemplate } from "@app/ee/services/project-template/project-template-types";
import { TSshCertificateAuthorityDALFactory } from "@app/ee/services/ssh/ssh-certificate-authority-dal";
import { TSshCertificateDALFactory } from "@app/ee/services/ssh-certificate/ssh-certificate-dal";
import { TSshCertificateTemplateDALFactory } from "@app/ee/services/ssh-certificate-template/ssh-certificate-template-dal";
import { TKeyStoreFactory } from "@app/keystore/keystore";
import { infisicalSymmetricEncypt } from "@app/lib/crypto/encryption";
import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/errors";
import { groupBy } from "@app/lib/fn";
import { alphaNumericNanoId } from "@app/lib/nanoid";
import { TProjectPermission } from "@app/lib/types";
import { TQueueServiceFactory } from "@app/queue";
import { ActorType } from "../auth/auth-type";
import { TCertificateDALFactory } from "../certificate/certificate-dal";
@@ -28,13 +32,17 @@ import { TOrgServiceFactory } from "../org/org-service";
import { TPkiAlertDALFactory } from "../pki-alert/pki-alert-dal";
import { TPkiCollectionDALFactory } from "../pki-collection/pki-collection-dal";
import { TProjectBotDALFactory } from "../project-bot/project-bot-dal";
import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { TProjectKeyDALFactory } from "../project-key/project-key-dal";
import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal";
import { TProjectUserMembershipRoleDALFactory } from "../project-membership/project-user-membership-role-dal";
import { TProjectRoleDALFactory } from "../project-role/project-role-dal";
import { getPredefinedRoles } from "../project-role/project-role-fns";
import { TSecretDALFactory } from "../secret/secret-dal";
import { fnDeleteProjectSecretReminders } from "../secret/secret-fns";
import { ROOT_FOLDER_NAME, TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { TProjectSlackConfigDALFactory } from "../slack/project-slack-config-dal";
import { TSlackIntegrationDALFactory } from "../slack/slack-integration-dal";
import { TUserDALFactory } from "../user/user-dal";
@@ -52,6 +60,9 @@ import {
TListProjectCertificateTemplatesDTO,
TListProjectCertsDTO,
TListProjectsDTO,
TListProjectSshCasDTO,
TListProjectSshCertificatesDTO,
TListProjectSshCertificateTemplatesDTO,
TLoadProjectKmsBackupDTO,
TToggleProjectAutoCapitalizationDTO,
TUpdateAuditLogsRetentionDTO,
@@ -74,7 +85,10 @@ type TProjectServiceFactoryDep = {
projectDAL: TProjectDALFactory;
projectQueue: TProjectQueueFactory;
userDAL: TUserDALFactory;
folderDAL: TSecretFolderDALFactory;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
folderDAL: Pick<TSecretFolderDALFactory, "insertMany" | "findByProjectId">;
secretDAL: Pick<TSecretDALFactory, "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find">;
projectEnvDAL: Pick<TProjectEnvDALFactory, "insertMany" | "find">;
identityOrgMembershipDAL: TIdentityOrgDALFactory;
identityProjectDAL: TIdentityProjectDALFactory;
@@ -89,9 +103,14 @@ type TProjectServiceFactoryDep = {
certificateTemplateDAL: Pick<TCertificateTemplateDALFactory, "getCertTemplatesByProjectId">;
pkiAlertDAL: Pick<TPkiAlertDALFactory, "find">;
pkiCollectionDAL: Pick<TPkiCollectionDALFactory, "find">;
sshCertificateAuthorityDAL: Pick<TSshCertificateAuthorityDALFactory, "find">;
sshCertificateDAL: Pick<TSshCertificateDALFactory, "find" | "countSshCertificatesInProject">;
sshCertificateTemplateDAL: Pick<TSshCertificateTemplateDALFactory, "find">;
permissionService: TPermissionServiceFactory;
orgService: Pick<TOrgServiceFactory, "addGhostUser">;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
queueService: Pick<TQueueServiceFactory, "stopRepeatableJob">;
orgDAL: Pick<TOrgDALFactory, "findOne">;
keyStore: Pick<TKeyStoreFactory, "deleteItem">;
projectBotDAL: Pick<TProjectBotDALFactory, "create">;
@@ -112,9 +131,13 @@ export type TProjectServiceFactory = ReturnType<typeof projectServiceFactory>;
export const projectServiceFactory = ({
projectDAL,
secretDAL,
secretV2BridgeDAL,
projectQueue,
projectKeyDAL,
permissionService,
queueService,
projectBotService,
orgDAL,
userDAL,
folderDAL,
@@ -132,6 +155,9 @@ export const projectServiceFactory = ({
certificateTemplateDAL,
pkiCollectionDAL,
pkiAlertDAL,
sshCertificateAuthorityDAL,
sshCertificateDAL,
sshCertificateTemplateDAL,
keyStore,
kmsService,
projectBotDAL,
@@ -424,6 +450,14 @@ export const projectServiceFactory = ({
await userDAL.deleteById(projectGhostUser.id, tx);
}
await fnDeleteProjectSecretReminders(project.id, {
secretDAL,
secretV2BridgeDAL,
queueService,
projectBotService,
folderDAL
});
return delProject;
});
@@ -441,7 +475,12 @@ export const projectServiceFactory = ({
const workspaces = await projectDAL.findAllProjects(actorId, actorOrgId, type);
if (includeRoles) {
const { permission } = await permissionService.getUserOrgPermission(actorId, actorOrgId, actorAuthMethod);
const { permission } = await permissionService.getUserOrgPermission(
actorId,
actorOrgId,
actorAuthMethod,
actorOrgId
);
// `includeRoles` is specifically used by organization admins when inviting new users to the organizations to avoid looping redundant api calls.
ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Create, OrgPermissionSubjects.Member);
@@ -896,6 +935,118 @@ export const projectServiceFactory = ({
};
};
/**
* Return list of SSH CAs for project
*/
const listProjectSshCas = async ({
actorId,
actorOrgId,
actorAuthMethod,
actor,
projectId
}: TListProjectSshCasDTO) => {
const { permission, ForbidOnInvalidProjectType } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
ForbidOnInvalidProjectType(ProjectType.SSH);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
ProjectPermissionSub.SshCertificateAuthorities
);
const cas = await sshCertificateAuthorityDAL.find(
{
projectId
},
{ sort: [["updatedAt", "desc"]] }
);
return cas;
};
/**
* Return list of SSH certificates for project
*/
const listProjectSshCertificates = async ({
limit = 25,
offset = 0,
actorId,
actorOrgId,
actorAuthMethod,
actor,
projectId
}: TListProjectSshCertificatesDTO) => {
const { permission, ForbidOnInvalidProjectType } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
ForbidOnInvalidProjectType(ProjectType.SSH);
ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.SshCertificates);
const cas = await sshCertificateAuthorityDAL.find({
projectId
});
const certificates = await sshCertificateDAL.find(
{
$in: {
sshCaId: cas.map((ca) => ca.id)
}
},
{ offset, limit, sort: [["updatedAt", "desc"]] }
);
const count = await sshCertificateDAL.countSshCertificatesInProject(projectId);
return { certificates, totalCount: count };
};
/**
* Return list of SSH certificate templates for project
*/
const listProjectSshCertificateTemplates = async ({
actorId,
actorOrgId,
actorAuthMethod,
actor,
projectId
}: TListProjectSshCertificateTemplatesDTO) => {
const { permission, ForbidOnInvalidProjectType } = await permissionService.getProjectPermission(
actor,
actorId,
projectId,
actorAuthMethod,
actorOrgId
);
ForbidOnInvalidProjectType(ProjectType.SSH);
ForbiddenError.from(permission).throwUnlessCan(
ProjectPermissionActions.Read,
ProjectPermissionSub.SshCertificateTemplates
);
const cas = await sshCertificateAuthorityDAL.find({
projectId
});
const certificateTemplates = await sshCertificateTemplateDAL.find({
$in: {
sshCaId: cas.map((ca) => ca.id)
}
});
return { certificateTemplates };
};
const updateProjectKmsKey = async ({
projectId,
kms,
@@ -1129,6 +1280,9 @@ export const projectServiceFactory = ({
listProjectAlerts,
listProjectPkiCollections,
listProjectCertificateTemplates,
listProjectSshCas,
listProjectSshCertificates,
listProjectSshCertificateTemplates,
updateVersionLimit,
updateAuditLogsRetention,
updateProjectKmsKey,
@@ -132,6 +132,13 @@ export type TGetProjectKmsKey = TProjectPermission;
export type TListProjectCertificateTemplatesDTO = TProjectPermission;
export type TListProjectSshCasDTO = TProjectPermission;
export type TListProjectSshCertificateTemplatesDTO = TProjectPermission;
export type TListProjectSshCertificatesDTO = {
offset: number;
limit: number;
} & TProjectPermission;
export type TGetProjectSlackConfig = TProjectPermission;
export type TUpdateProjectSlackConfig = {
@@ -5,6 +5,7 @@ import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TIdentityAccessTokenDALFactory } from "../identity-access-token/identity-access-token-dal";
import { TIdentityUaClientSecretDALFactory } from "../identity-ua/identity-ua-client-secret-dal";
import { TSecretDALFactory } from "../secret/secret-dal";
import { TSecretVersionDALFactory } from "../secret/secret-version-dal";
import { TSecretFolderVersionDALFactory } from "../secret-folder/secret-folder-version-dal";
import { TSecretSharingDALFactory } from "../secret-sharing/secret-sharing-dal";
@@ -16,6 +17,7 @@ type TDailyResourceCleanUpQueueServiceFactoryDep = {
identityUniversalAuthClientSecretDAL: Pick<TIdentityUaClientSecretDALFactory, "removeExpiredClientSecrets">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "pruneExcessVersions">;
secretVersionV2DAL: Pick<TSecretVersionV2DALFactory, "pruneExcessVersions">;
secretDAL: Pick<TSecretDALFactory, "pruneSecretReminders">;
secretFolderVersionDAL: Pick<TSecretFolderVersionDALFactory, "pruneExcessVersions">;
snapshotDAL: Pick<TSnapshotDALFactory, "pruneExcessSnapshots">;
secretSharingDAL: Pick<TSecretSharingDALFactory, "pruneExpiredSharedSecrets">;
@@ -30,6 +32,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
snapshotDAL,
secretVersionDAL,
secretFolderVersionDAL,
secretDAL,
identityAccessTokenDAL,
secretSharingDAL,
secretVersionV2DAL,
@@ -37,6 +40,7 @@ export const dailyResourceCleanUpQueueServiceFactory = ({
}: TDailyResourceCleanUpQueueServiceFactoryDep) => {
queueService.start(QueueName.DailyResourceCleanUp, async () => {
logger.info(`${QueueName.DailyResourceCleanUp}: queue task started`);
await secretDAL.pruneSecretReminders(queueService);
await auditLogDAL.pruneAuditLog();
await identityAccessTokenDAL.removeExpiredTokens();
await identityUniversalAuthClientSecretDAL.removeExpiredClientSecrets();
+92 -1
View File
@@ -5,6 +5,8 @@ import { TDbClient } from "@app/db";
import { SecretsSchema, SecretType, TableName, TSecrets, TSecretsUpdate } from "@app/db/schemas";
import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
import { logger } from "@app/lib/logger";
import { QueueName, TQueueServiceFactory } from "@app/queue";
export type TSecretDALFactory = ReturnType<typeof secretDALFactory>;
@@ -339,6 +341,94 @@ export const secretDALFactory = (db: TDbClient) => {
}
};
const pruneSecretReminders = async (queueService: TQueueServiceFactory) => {
const REMINDER_PRUNE_BATCH_SIZE = 5_000;
const MAX_RETRY_ON_FAILURE = 3;
let numberOfRetryOnFailure = 0;
let deletedReminderCount = 0;
logger.info(`${QueueName.DailyResourceCleanUp}: secret reminders started`);
try {
const repeatableJobs = await queueService.getRepeatableJobs(QueueName.SecretReminder);
const reminderJobs = repeatableJobs
.map((job) => ({ secretId: job.id?.replace("reminder-", "") as string, jobKey: job.key }))
.filter(Boolean);
if (reminderJobs.length === 0) {
logger.info(`${QueueName.DailyResourceCleanUp}: no reminder jobs found`);
return;
}
for (let offset = 0; offset < reminderJobs.length; offset += REMINDER_PRUNE_BATCH_SIZE) {
try {
const batchIds = reminderJobs.slice(offset, offset + REMINDER_PRUNE_BATCH_SIZE).map((r) => r.secretId);
const payload = {
$in: {
id: batchIds
}
};
const opts = {
limit: REMINDER_PRUNE_BATCH_SIZE
};
// Find existing secrets with pagination
// eslint-disable-next-line no-await-in-loop
const [secrets, secretsV2] = await Promise.all([
ormify(db, TableName.Secret).find(payload, opts),
ormify(db, TableName.SecretV2).find(payload, opts)
]);
const foundSecretIds = new Set([
...secrets.map((secret) => secret.id),
...secretsV2.map((secret) => secret.id)
]);
// Find IDs that don't exist in either table
const secretIdsNotFound = batchIds.filter((secretId) => !foundSecretIds.has(secretId));
// Delete reminders for non-existent secrets
for (const secretId of secretIdsNotFound) {
const jobKey = reminderJobs.find((r) => r.secretId === secretId)?.jobKey;
if (jobKey) {
// eslint-disable-next-line no-await-in-loop
await queueService.stopRepeatableJobByKey(QueueName.SecretReminder, jobKey);
deletedReminderCount += 1;
}
}
numberOfRetryOnFailure = 0;
} catch (error) {
numberOfRetryOnFailure += 1;
logger.error(error, `Failed to process batch at offset ${offset}`);
if (numberOfRetryOnFailure >= MAX_RETRY_ON_FAILURE) {
break;
}
// Retry the current batch
offset -= REMINDER_PRUNE_BATCH_SIZE;
// eslint-disable-next-line no-promise-executor-return, @typescript-eslint/no-loop-func, no-await-in-loop
await new Promise((resolve) => setTimeout(resolve, 500 * numberOfRetryOnFailure));
}
// Small delay between batches
// eslint-disable-next-line no-promise-executor-return, @typescript-eslint/no-loop-func, no-await-in-loop
await new Promise((resolve) => setTimeout(resolve, 10));
}
} catch (error) {
logger.error(error, "Failed to complete secret reminder pruning");
} finally {
logger.info(
`${QueueName.DailyResourceCleanUp}: secret reminders completed. Deleted ${deletedReminderCount} reminders`
);
}
};
return {
...secretOrm,
update,
@@ -352,6 +442,7 @@ export const secretDALFactory = (db: TDbClient) => {
findByBlindIndexes,
upsertSecretReferences,
findReferencedSecretReferences,
findAllProjectSecretValues
findAllProjectSecretValues,
pruneSecretReminders
};
};
+50
View File
@@ -19,9 +19,11 @@ import {
decryptSymmetric128BitHexKeyUTF8,
encryptSymmetric128BitHexKeyUTF8
} from "@app/lib/crypto";
import { daysToMillisecond, secondsToMillis } from "@app/lib/dates";
import { BadRequestError, NotFoundError } from "@app/lib/errors";
import { groupBy, unique } from "@app/lib/fn";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import {
fnSecretBulkInsert as fnSecretV2BridgeBulkInsert,
fnSecretBulkUpdate as fnSecretV2BridgeBulkUpdate,
@@ -31,8 +33,10 @@ import {
import { ActorAuthMethod, ActorType } from "../auth/auth-type";
import { KmsDataKey } from "../kms/kms-types";
import { getBotKeyFnFactory } from "../project-bot/project-bot-fns";
import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { TSecretDALFactory } from "./secret-dal";
import {
TCreateManySecretsRawFn,
@@ -1138,3 +1142,49 @@ export const decryptSecretWithBot = (
secretComment
};
};
type TFnDeleteProjectSecretReminders = {
secretDAL: Pick<TSecretDALFactory, "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find">;
queueService: Pick<TQueueServiceFactory, "stopRepeatableJob">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
folderDAL: Pick<TSecretFolderDALFactory, "findByProjectId">;
};
export const fnDeleteProjectSecretReminders = async (
projectId: string,
{ secretDAL, secretV2BridgeDAL, queueService, projectBotService, folderDAL }: TFnDeleteProjectSecretReminders
) => {
const projectFolders = await folderDAL.findByProjectId(projectId);
const { shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId, false);
const projectSecrets = shouldUseSecretV2Bridge
? await secretV2BridgeDAL.find({
$in: { folderId: projectFolders.map((folder) => folder.id) },
$notNull: ["reminderRepeatDays"]
})
: await secretDAL.find({
$in: { folderId: projectFolders.map((folder) => folder.id) },
$notNull: ["secretReminderRepeatDays"]
});
const appCfg = getConfig();
for await (const secret of projectSecrets) {
const repeatDays = shouldUseSecretV2Bridge
? (secret as { reminderRepeatDays: number }).reminderRepeatDays
: (secret as { secretReminderRepeatDays: number }).secretReminderRepeatDays;
// We're using the queue service directly to get around conflicting imports.
if (repeatDays) {
await queueService.stopRepeatableJob(
QueueName.SecretReminder,
QueueJobs.SecretReminder,
{
// on prod it this will be in days, in development this will be second
every: appCfg.NODE_ENV === "development" ? secondsToMillis(repeatDays) : daysToMillisecond(repeatDays)
},
`reminder-${secret.id}`
);
}
}
};
+3 -1
View File
@@ -248,7 +248,9 @@ export const secretQueueFactory = ({
? secondsToMillis(newSecret.secretReminderRepeatDays)
: daysToMillisecond(newSecret.secretReminderRepeatDays),
immediately: true
}
},
removeOnComplete: true,
removeOnFail: true
}
);
} catch (err) {
+30 -8
View File
@@ -491,8 +491,8 @@ export const secretServiceFactory = ({
secretDAL
});
const deletedSecret = await secretDAL.transaction(async (tx) =>
fnSecretBulkDelete({
const deletedSecret = await secretDAL.transaction(async (tx) => {
const secrets = await fnSecretBulkDelete({
projectId,
folderId,
actorId,
@@ -505,8 +505,19 @@ export const secretServiceFactory = ({
}
],
tx
})
);
});
for await (const secret of secrets) {
if (secret.secretReminderRepeatDays !== null && secret.secretReminderRepeatDays !== undefined) {
await secretQueueService.removeSecretReminder({
repeatDays: secret.secretReminderRepeatDays,
secretId: secret.id
});
}
}
return secrets;
});
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
@@ -971,8 +982,8 @@ export const secretServiceFactory = ({
secretDAL
});
const secretsDeleted = await secretDAL.transaction(async (tx) =>
fnSecretBulkDelete({
const secretsDeleted = await secretDAL.transaction(async (tx) => {
const secrets = await fnSecretBulkDelete({
secretDAL,
secretQueueService,
inputSecrets: inputSecrets.map(({ type, secretName }) => ({
@@ -983,8 +994,19 @@ export const secretServiceFactory = ({
folderId,
actorId,
tx
})
);
});
for await (const secret of secrets) {
if (secret.secretReminderRepeatDays !== null && secret.secretReminderRepeatDays !== undefined) {
await secretQueueService.removeSecretReminder({
repeatDays: secret.secretReminderRepeatDays,
secretId: secret.id
});
}
}
return secrets;
});
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({