awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-06 07:26:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

misc: updated processes to run after db migration

Browse Source
This commit is contained in:
Sheen Capadngan
2025-10-28 23:08:39 +08:00
parent 1cff5cd7d7
commit 9d61b06eec
6 changed files with 675 additions and 647 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-queue.ts
+152 -148
View File
@@ -42,168 +42,172 @@ export const secretRotationV2QueueServiceFactory = async ({
smtpService, smtpService,
notificationService notificationService
}: TSecretRotationV2QueueServiceFactoryDep) => { }: TSecretRotationV2QueueServiceFactoryDep) => {
const appCfg = getConfig(); const init = async () => {
const appCfg = getConfig();
if (appCfg.isRotationDevelopmentMode) { if (appCfg.isRotationDevelopmentMode) {
logger.warn("Secret Rotation V2 is in development mode."); logger.warn("Secret Rotation V2 is in development mode.");
} }
await queueService.startPg<QueueName.SecretRotationV2>( await queueService.startPg<QueueName.SecretRotationV2>(
QueueJobs.SecretRotationV2QueueRotations, QueueJobs.SecretRotationV2QueueRotations,
async () => { async () => {
try { try {
const rotateBy = getNextUtcRotationInterval(); const rotateBy = getNextUtcRotationInterval();
const currentTime = new Date(); const currentTime = new Date();
const secretRotations = await secretRotationV2DAL.findSecretRotationsToQueue(rotateBy); const secretRotations = await secretRotationV2DAL.findSecretRotationsToQueue(rotateBy);
logger.info(
`secretRotationV2Queue: Queue Rotations [currentTime=${currentTime.toISOString()}] [rotateBy=${rotateBy.toISOString()}] [count=${
secretRotations.length
}]`
);
for await (const rotation of secretRotations) {
logger.info( logger.info(
`secretRotationV2Queue: Queue Rotation [rotationId=${rotation.id}] [lastRotatedAt=${new Date( `secretRotationV2Queue: Queue Rotations [currentTime=${currentTime.toISOString()}] [rotateBy=${rotateBy.toISOString()}] [count=${
rotation.lastRotatedAt secretRotations.length
).toISOString()}] [rotateAt=${new Date(rotation.nextRotationAt!).toISOString()}]` }]`
); );
const data = { for await (const rotation of secretRotations) {
rotationId: rotation.id, logger.info(
queuedAt: currentTime `secretRotationV2Queue: Queue Rotation [rotationId=${rotation.id}] [lastRotatedAt=${new Date(
} as TSecretRotationRotateSecretsJobPayload; rotation.lastRotatedAt
).toISOString()}] [rotateAt=${new Date(rotation.nextRotationAt!).toISOString()}]`
if (appCfg.isTestMode) {
logger.warn("secretRotationV2Queue: Manually rotating secrets for test mode");
await rotateSecretsFns({
job: {
id: uuidv4(),
data,
retryCount: 0,
retryLimit: 0
},
secretRotationV2DAL,
secretRotationV2Service
});
} else {
await queueService.queuePg(
QueueJobs.SecretRotationV2RotateSecrets,
{
rotationId: rotation.id,
queuedAt: currentTime
},
getSecretRotationRotateSecretJobOptions(rotation)
); );
const data = {
rotationId: rotation.id,
queuedAt: currentTime
} as TSecretRotationRotateSecretsJobPayload;
if (appCfg.isTestMode) {
logger.warn("secretRotationV2Queue: Manually rotating secrets for test mode");
await rotateSecretsFns({
job: {
id: uuidv4(),
data,
retryCount: 0,
retryLimit: 0
},
secretRotationV2DAL,
secretRotationV2Service
});
} else {
await queueService.queuePg(
QueueJobs.SecretRotationV2RotateSecrets,
{
rotationId: rotation.id,
queuedAt: currentTime
},
getSecretRotationRotateSecretJobOptions(rotation)
);
}
} }
} catch (error) {
logger.error(error, "secretRotationV2Queue: Queue Rotations Error:");
throw error;
} }
} catch (error) { },
logger.error(error, "secretRotationV2Queue: Queue Rotations Error:"); {
throw error; batchSize: 1,
workerCount: 1,
pollingIntervalSeconds: appCfg.isRotationDevelopmentMode ? 0.5 : 30
} }
}, );
{
batchSize: 1,
workerCount: 1,
pollingIntervalSeconds: appCfg.isRotationDevelopmentMode ? 0.5 : 30
}
);
await queueService.startPg<QueueName.SecretRotationV2>( await queueService.startPg<QueueName.SecretRotationV2>(
QueueJobs.SecretRotationV2RotateSecrets, QueueJobs.SecretRotationV2RotateSecrets,
async ([job]) => { async ([job]) => {
await rotateSecretsFns({ await rotateSecretsFns({
job: { job: {
...job, ...job,
data: job.data as TSecretRotationRotateSecretsJobPayload data: job.data as TSecretRotationRotateSecretsJobPayload
}, },
secretRotationV2DAL, secretRotationV2DAL,
secretRotationV2Service secretRotationV2Service
});
},
{
batchSize: 1,
workerCount: 2,
pollingIntervalSeconds: 0.5
}
);
await queueService.startPg<QueueName.SecretRotationV2>(
QueueJobs.SecretRotationV2SendNotification,
async ([job]) => {
const { secretRotation } = job.data as TSecretRotationSendNotificationJobPayload;
try {
const {
name: rotationName,
type,
projectId,
lastRotationAttemptedAt,
folder,
environment,
id: rotationId
} = secretRotation;
logger.info(`secretRotationV2Queue: Sending Status Notification [rotationId=${rotationId}]`);
const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
const project = await projectDAL.findById(projectId);
const projectAdmins = projectMembers.filter((member) =>
member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
);
const rotationType = SECRET_ROTATION_NAME_MAP[type as SecretRotation];
const rotationPath = `/projects/secret-management/${projectId}/secrets/${environment.slug}`;
await notificationService.createUserNotifications(
projectAdmins.map((admin) => ({
userId: admin.userId,
orgId: project.orgId,
type: NotificationType.SECRET_ROTATION_FAILED,
title: "Secret Rotation Failed",
body: `Your **${rotationType}** rotation **${rotationName}** failed to rotate.`,
link: rotationPath
}))
);
await smtpService.sendMail({
recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
template: SmtpTemplates.SecretRotationFailed,
subjectLine: `Secret Rotation Failed`,
substitutions: {
rotationName,
rotationType,
content: `Your ${rotationType} Rotation failed to rotate during it's scheduled rotation. The last rotation attempt occurred at ${new Date(
lastRotationAttemptedAt
).toISOString()}. Please check the rotation status in Infisical for more details.`,
secretPath: folder.path,
environment: environment.name,
projectName: project.name,
rotationUrl: encodeURI(`${appCfg.SITE_URL}${rotationPath}`)
}
}); });
} catch (error) { },
logger.error( {
error, batchSize: 1,
`secretRotationV2Queue: Failed to Send Status Notification [rotationId=${secretRotation.id}]` workerCount: 2,
); pollingIntervalSeconds: 0.5
throw error;
} }
}, );
{
batchSize: 1,
workerCount: 2,
pollingIntervalSeconds: 1
}
);
await queueService.schedulePg( await queueService.startPg<QueueName.SecretRotationV2>(
QueueJobs.SecretRotationV2QueueRotations, QueueJobs.SecretRotationV2SendNotification,
appCfg.isRotationDevelopmentMode ? "* * * * *" : "0 0 * * *", async ([job]) => {
undefined, const { secretRotation } = job.data as TSecretRotationSendNotificationJobPayload;
{ tz: "UTC" } try {
); const {
name: rotationName,
type,
projectId,
lastRotationAttemptedAt,
folder,
environment,
id: rotationId
} = secretRotation;
logger.info(`secretRotationV2Queue: Sending Status Notification [rotationId=${rotationId}]`);
const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
const project = await projectDAL.findById(projectId);
const projectAdmins = projectMembers.filter((member) =>
member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
);
const rotationType = SECRET_ROTATION_NAME_MAP[type as SecretRotation];
const rotationPath = `/projects/secret-management/${projectId}/secrets/${environment.slug}`;
await notificationService.createUserNotifications(
projectAdmins.map((admin) => ({
userId: admin.userId,
orgId: project.orgId,
type: NotificationType.SECRET_ROTATION_FAILED,
title: "Secret Rotation Failed",
body: `Your **${rotationType}** rotation **${rotationName}** failed to rotate.`,
link: rotationPath
}))
);
await smtpService.sendMail({
recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
template: SmtpTemplates.SecretRotationFailed,
subjectLine: `Secret Rotation Failed`,
substitutions: {
rotationName,
rotationType,
content: `Your ${rotationType} Rotation failed to rotate during it's scheduled rotation. The last rotation attempt occurred at ${new Date(
lastRotationAttemptedAt
).toISOString()}. Please check the rotation status in Infisical for more details.`,
secretPath: folder.path,
environment: environment.name,
projectName: project.name,
rotationUrl: encodeURI(`${appCfg.SITE_URL}${rotationPath}`)
}
});
} catch (error) {
logger.error(
error,
`secretRotationV2Queue: Failed to Send Status Notification [rotationId=${secretRotation.id}]`
);
throw error;
}
},
{
batchSize: 1,
workerCount: 2,
pollingIntervalSeconds: 1
}
);
await queueService.schedulePg(
QueueJobs.SecretRotationV2QueueRotations,
appCfg.isRotationDevelopmentMode ? "* * * * *" : "0 0 * * *",
undefined,
{ tz: "UTC" }
);
};
return { init };
}; };
backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
+414 -411
View File
@@ -141,202 +141,6 @@ export const secretScanningV2QueueServiceFactory = async ({
} }
}; };
await queueService.startPg<QueueName.SecretScanningV2>(
QueueJobs.SecretScanningV2FullScan,
async ([job]) => {
const { scanId, resourceId, dataSourceId } = job.data as TQueueSecretScanningDataSourceFullScan;
const { retryCount, retryLimit } = job;
const logDetails = `[scanId=${scanId}] [resourceId=${resourceId}] [dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
const tempFolder = await createTempFolder();
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
const resource = await secretScanningV2DAL.resources.findById(resourceId);
if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;
try {
try {
lock = await keyStore.acquireLock(
[KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId)],
60 * 1000 * 5
);
} catch (e) {
throw new Error("Failed to acquire scanning lock.");
}
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Scanning
}
);
let connection: TAppConnection | null = null;
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
kmsService,
appConnectionDAL
});
const findingsPath = join(tempFolder, "findings.json");
const scanPath = await factory.getFullScanPath({
dataSource: {
...dataSource,
connection
} as TSecretScanningDataSourceWithConnection,
resourceName: resource.name,
tempFolder
});
const config = await secretScanningV2DAL.configs.findOne({
projectId: dataSource.projectId
});
let configPath: string | undefined;
if (config && config.content) {
configPath = join(tempFolder, "infisical-scan.toml");
await writeTextToFile(configPath, config.content);
}
let findingsPayload: TFindingsPayload;
switch (resource.type) {
case SecretScanningResource.Repository:
case SecretScanningResource.Project:
findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath, configPath);
break;
default:
throw new Error("Unhandled resource type");
}
const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
let findings: TSecretScanningFindings[] = [];
if (findingsPayload.length) {
findings = await secretScanningV2DAL.findings.upsert(
findingsPayload.map((finding) => ({
...finding,
projectId: dataSource.projectId,
dataSourceName: dataSource.name,
dataSourceType: dataSource.type,
resourceName: resource.name,
resourceType: resource.type,
scanId
})),
["projectId", "fingerprint"],
tx,
["resourceName", "dataSourceName"]
);
}
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Completed,
statusMessage: null
}
);
return findings;
});
const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
if (newFindings.length) {
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Completed,
resourceName: resource.name,
isDiffScan: false,
dataSource,
numberOfSecrets: newFindings.length,
scanId
});
}
await auditLogService.createAuditLog({
projectId: dataSource.projectId,
actor: {
type: ActorType.PLATFORM,
metadata: {}
},
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
metadata: {
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceId: resource.id,
resourceType: resource.type,
scanId,
scanStatus: SecretScanningScanStatus.Completed,
scanType: SecretScanningScanType.FullScan,
numberOfSecretsDetected: findingsPayload.length
}
}
});
logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails} findings=[${findingsPayload.length}]`);
} catch (error) {
if (retryCount === retryLimit) {
const errorMessage = parseScanErrorMessage(error);
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Failed,
statusMessage: errorMessage
}
);
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Failed,
resourceName: resource.name,
dataSource,
errorMessage
});
await auditLogService.createAuditLog({
projectId: dataSource.projectId,
actor: {
type: ActorType.PLATFORM,
metadata: {}
},
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
metadata: {
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceId: resource.id,
resourceType: resource.type,
scanId,
scanStatus: SecretScanningScanStatus.Failed,
scanType: SecretScanningScanType.FullScan
}
}
});
}
logger.error(error, `secretScanningV2Queue: Full Scan Failed ${logDetails}`);
throw error;
} finally {
await deleteTempFolder(tempFolder);
await lock?.release();
}
},
{
batchSize: 1,
workerCount: 2,
pollingIntervalSeconds: 1
}
);
const queueResourceDiffScan = async ({ const queueResourceDiffScan = async ({
payload, payload,
dataSourceId, dataSourceId,
@@ -391,148 +195,127 @@ export const secretScanningV2QueueServiceFactory = async ({
} }
}; };
await queueService.startPg<QueueName.SecretScanningV2>( const init = async () => {
QueueJobs.SecretScanningV2DiffScan, await queueService.startPg<QueueName.SecretScanningV2>(
async ([job]) => { QueueJobs.SecretScanningV2FullScan,
const { payload, dataSourceId, resourceId, scanId } = job.data as TQueueSecretScanningResourceDiffScan; async ([job]) => {
const { retryCount, retryLimit } = job; const { scanId, resourceId, dataSourceId } = job.data as TQueueSecretScanningDataSourceFullScan;
const { retryCount, retryLimit } = job;
const logDetails = `[dataSourceId=${dataSourceId}] [scanId=${scanId}] [resourceId=${resourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`; const logDetails = `[scanId=${scanId}] [resourceId=${resourceId}] [dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId); const tempFolder = await createTempFolder();
if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`); const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
const resource = await secretScanningV2DAL.resources.findById(resourceId); if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`); const resource = await secretScanningV2DAL.resources.findById(resourceId);
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({ if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
kmsService,
appConnectionDAL
});
const tempFolder = await createTempFolder(); let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;
try { try {
await secretScanningV2DAL.scans.update( try {
{ id: scanId }, lock = await keyStore.acquireLock(
{ [KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId)],
status: SecretScanningScanStatus.Scanning 60 * 1000 * 5
}
);
let connection: TAppConnection | null = null;
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
const config = await secretScanningV2DAL.configs.findOne({
projectId: dataSource.projectId
});
let configPath: string | undefined;
if (config && config.content) {
configPath = join(tempFolder, "infisical-scan.toml");
await writeTextToFile(configPath, config.content);
}
const findingsPayload = await factory.getDiffScanFindingsPayload({
dataSource: {
...dataSource,
connection
} as TSecretScanningDataSourceWithConnection,
resourceName: resource.name,
payload,
configPath
});
const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
let findings: TSecretScanningFindings[] = [];
if (findingsPayload.length) {
findings = await secretScanningV2DAL.findings.upsert(
findingsPayload.map((finding) => ({
...finding,
projectId: dataSource.projectId,
dataSourceName: dataSource.name,
dataSourceType: dataSource.type,
resourceName: resource.name,
resourceType: resource.type,
scanId
})),
["projectId", "fingerprint"],
tx,
["resourceName", "dataSourceName"]
); );
} catch (e) {
throw new Error("Failed to acquire scanning lock.");
} }
await secretScanningV2DAL.scans.update( await secretScanningV2DAL.scans.update(
{ id: scanId }, { id: scanId },
{ {
status: SecretScanningScanStatus.Completed status: SecretScanningScanStatus.Scanning
} }
); );
return findings; let connection: TAppConnection | null = null;
}); if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
const newFindings = allFindings.filter((finding) => finding.scanId === scanId); const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
kmsService,
if (newFindings.length) { appConnectionDAL
const finding = newFindings[0] as TSecretScanningFinding;
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Completed,
resourceName: resource.name,
isDiffScan: true,
dataSource,
numberOfSecrets: newFindings.length,
scanId,
authorName: finding?.details?.author,
authorEmail: finding?.details?.email
}); });
}
await auditLogService.createAuditLog({ const findingsPath = join(tempFolder, "findings.json");
projectId: dataSource.projectId,
actor: { const scanPath = await factory.getFullScanPath({
type: ActorType.PLATFORM, dataSource: {
metadata: {} ...dataSource,
}, connection
event: { } as TSecretScanningDataSourceWithConnection,
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN, resourceName: resource.name,
metadata: { tempFolder
dataSourceId: dataSource.id, });
dataSourceType: dataSource.type,
resourceId, const config = await secretScanningV2DAL.configs.findOne({
resourceType: resource.type, projectId: dataSource.projectId
scanId, });
scanStatus: SecretScanningScanStatus.Completed,
scanType: SecretScanningScanType.DiffScan, let configPath: string | undefined;
numberOfSecretsDetected: findingsPayload.length
} if (config && config.content) {
configPath = join(tempFolder, "infisical-scan.toml");
await writeTextToFile(configPath, config.content);
} }
});
logger.info(`secretScanningV2Queue: Diff Scan Complete ${logDetails}`); let findingsPayload: TFindingsPayload;
} catch (error) { switch (resource.type) {
if (retryCount === retryLimit) { case SecretScanningResource.Repository:
const errorMessage = parseScanErrorMessage(error); case SecretScanningResource.Project:
findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath, configPath);
break;
default:
throw new Error("Unhandled resource type");
}
await secretScanningV2DAL.scans.update( const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
{ id: scanId }, let findings: TSecretScanningFindings[] = [];
{ if (findingsPayload.length) {
status: SecretScanningScanStatus.Failed, findings = await secretScanningV2DAL.findings.upsert(
statusMessage: errorMessage findingsPayload.map((finding) => ({
...finding,
projectId: dataSource.projectId,
dataSourceName: dataSource.name,
dataSourceType: dataSource.type,
resourceName: resource.name,
resourceType: resource.type,
scanId
})),
["projectId", "fingerprint"],
tx,
["resourceName", "dataSourceName"]
);
} }
);
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, { await secretScanningV2DAL.scans.update(
status: SecretScanningScanStatus.Failed, { id: scanId },
resourceName: resource.name, {
dataSource, status: SecretScanningScanStatus.Completed,
errorMessage statusMessage: null
}
);
return findings;
}); });
const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
if (newFindings.length) {
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Completed,
resourceName: resource.name,
isDiffScan: false,
dataSource,
numberOfSecrets: newFindings.length,
scanId
});
}
await auditLogService.createAuditLog({ await auditLogService.createAuditLog({
projectId: dataSource.projectId, projectId: dataSource.projectId,
actor: { actor: {
@@ -547,128 +330,348 @@ export const secretScanningV2QueueServiceFactory = async ({
resourceId: resource.id, resourceId: resource.id,
resourceType: resource.type, resourceType: resource.type,
scanId, scanId,
scanStatus: SecretScanningScanStatus.Failed, scanStatus: SecretScanningScanStatus.Completed,
scanType: SecretScanningScanType.DiffScan scanType: SecretScanningScanType.FullScan,
numberOfSecretsDetected: findingsPayload.length
} }
} }
}); });
logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails} findings=[${findingsPayload.length}]`);
} catch (error) {
if (retryCount === retryLimit) {
const errorMessage = parseScanErrorMessage(error);
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Failed,
statusMessage: errorMessage
}
);
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Failed,
resourceName: resource.name,
dataSource,
errorMessage
});
await auditLogService.createAuditLog({
projectId: dataSource.projectId,
actor: {
type: ActorType.PLATFORM,
metadata: {}
},
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
metadata: {
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceId: resource.id,
resourceType: resource.type,
scanId,
scanStatus: SecretScanningScanStatus.Failed,
scanType: SecretScanningScanType.FullScan
}
}
});
}
logger.error(error, `secretScanningV2Queue: Full Scan Failed ${logDetails}`);
throw error;
} finally {
await deleteTempFolder(tempFolder);
await lock?.release();
} }
},
logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`); {
throw error; batchSize: 1,
} finally { workerCount: 2,
await deleteTempFolder(tempFolder); pollingIntervalSeconds: 1
} }
}, );
{
batchSize: 1,
workerCount: 2,
pollingIntervalSeconds: 1
}
);
await queueService.startPg<QueueName.SecretScanningV2>( await queueService.startPg<QueueName.SecretScanningV2>(
QueueJobs.SecretScanningV2SendNotification, QueueJobs.SecretScanningV2DiffScan,
async ([job]) => { async ([job]) => {
const { dataSource, resourceName, ...payload } = job.data as TQueueSecretScanningSendNotification; const { payload, dataSourceId, resourceId, scanId } = job.data as TQueueSecretScanningResourceDiffScan;
const { retryCount, retryLimit } = job;
const appCfg = getConfig(); const logDetails = `[dataSourceId=${dataSourceId}] [scanId=${scanId}] [resourceId=${resourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
if (!appCfg.isSmtpConfigured) return; const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
try { if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
const { projectId } = dataSource;
logger.info( const resource = await secretScanningV2DAL.resources.findById(resourceId);
`secretScanningV2Queue: Sending Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
);
const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId); if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
const project = await projectDAL.findById(projectId);
const recipients = projectMembers.filter((member) => { const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
const isAdmin = member.roles.some((role) => role.role === ProjectMembershipRole.Admin); kmsService,
const isCompleted = payload.status === SecretScanningScanStatus.Completed; appConnectionDAL
// We assume that the committer is one of the project members
const isCommitter = isCompleted && payload.authorEmail === member.user.email;
return isAdmin || isCommitter;
}); });
const timestamp = new Date().toISOString(); const tempFolder = await createTempFolder();
const subjectLine = try {
payload.status === SecretScanningScanStatus.Completed await secretScanningV2DAL.scans.update(
? "Incident Alert: Secret(s) Leaked" { id: scanId },
: `Secret Scanning Failed`; {
status: SecretScanningScanStatus.Scanning
}
);
await notificationService.createUserNotifications( let connection: TAppConnection | null = null;
recipients.map((member) => ({ if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
userId: member.userId,
orgId: project.orgId,
type:
payload.status === SecretScanningScanStatus.Completed
? NotificationType.SECRET_SCANNING_SECRETS_DETECTED
: NotificationType.SECRET_SCANNING_SCAN_FAILED,
title: subjectLine,
body:
payload.status === SecretScanningScanStatus.Completed
? `Uncovered **${payload.numberOfSecrets}** secret(s) ${payload.isDiffScan ? " from a recent commit to" : " in"} **${resourceName}**.`
: `Encountered an error while attempting to scan the resource **${resourceName}**: ${payload.errorMessage}`,
link:
payload.status === SecretScanningScanStatus.Completed
? `/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
: `/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
}))
);
await smtpService.sendMail({ const config = await secretScanningV2DAL.configs.findOne({
recipients: recipients.map((member) => member.user.email!).filter(Boolean), projectId: dataSource.projectId
template: });
payload.status === SecretScanningScanStatus.Completed
? SmtpTemplates.SecretScanningV2SecretsDetected let configPath: string | undefined;
: SmtpTemplates.SecretScanningV2ScanFailed,
subjectLine, if (config && config.content) {
substitutions: configPath = join(tempFolder, "infisical-scan.toml");
payload.status === SecretScanningScanStatus.Completed await writeTextToFile(configPath, config.content);
? { }
authorName: payload.authorName,
authorEmail: payload.authorEmail, const findingsPayload = await factory.getDiffScanFindingsPayload({
resourceName, dataSource: {
numberOfSecrets: payload.numberOfSecrets, ...dataSource,
isDiffScan: payload.isDiffScan, connection
url: encodeURI( } as TSecretScanningDataSourceWithConnection,
`${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}` resourceName: resource.name,
), payload,
timestamp configPath
} });
: {
const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
let findings: TSecretScanningFindings[] = [];
if (findingsPayload.length) {
findings = await secretScanningV2DAL.findings.upsert(
findingsPayload.map((finding) => ({
...finding,
projectId: dataSource.projectId,
dataSourceName: dataSource.name, dataSourceName: dataSource.name,
resourceName, dataSourceType: dataSource.type,
projectName: project.name, resourceName: resource.name,
timestamp, resourceType: resource.type,
errorMessage: payload.errorMessage, scanId
url: encodeURI( })),
`${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}` ["projectId", "fingerprint"],
) tx,
["resourceName", "dataSourceName"]
);
}
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Completed
}
);
return findings;
});
const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
if (newFindings.length) {
const finding = newFindings[0] as TSecretScanningFinding;
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Completed,
resourceName: resource.name,
isDiffScan: true,
dataSource,
numberOfSecrets: newFindings.length,
scanId,
authorName: finding?.details?.author,
authorEmail: finding?.details?.email
});
}
await auditLogService.createAuditLog({
projectId: dataSource.projectId,
actor: {
type: ActorType.PLATFORM,
metadata: {}
},
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
metadata: {
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceId,
resourceType: resource.type,
scanId,
scanStatus: SecretScanningScanStatus.Completed,
scanType: SecretScanningScanType.DiffScan,
numberOfSecretsDetected: findingsPayload.length
}
}
});
logger.info(`secretScanningV2Queue: Diff Scan Complete ${logDetails}`);
} catch (error) {
if (retryCount === retryLimit) {
const errorMessage = parseScanErrorMessage(error);
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
status: SecretScanningScanStatus.Failed,
statusMessage: errorMessage
}
);
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
status: SecretScanningScanStatus.Failed,
resourceName: resource.name,
dataSource,
errorMessage
});
await auditLogService.createAuditLog({
projectId: dataSource.projectId,
actor: {
type: ActorType.PLATFORM,
metadata: {}
},
event: {
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
metadata: {
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceId: resource.id,
resourceType: resource.type,
scanId,
scanStatus: SecretScanningScanStatus.Failed,
scanType: SecretScanningScanType.DiffScan
} }
}); }
} catch (error) { });
logger.error( }
error,
`secretScanningV2Queue: Failed to Send Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]` logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`);
); throw error;
throw error; } finally {
await deleteTempFolder(tempFolder);
}
},
{
batchSize: 1,
workerCount: 2,
pollingIntervalSeconds: 1
} }
}, );
{
batchSize: 1, await queueService.startPg<QueueName.SecretScanningV2>(
workerCount: 2, QueueJobs.SecretScanningV2SendNotification,
pollingIntervalSeconds: 1 async ([job]) => {
} const { dataSource, resourceName, ...payload } = job.data as TQueueSecretScanningSendNotification;
);
const appCfg = getConfig();
if (!appCfg.isSmtpConfigured) return;
try {
const { projectId } = dataSource;
logger.info(
`secretScanningV2Queue: Sending Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
);
const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
const project = await projectDAL.findById(projectId);
const recipients = projectMembers.filter((member) => {
const isAdmin = member.roles.some((role) => role.role === ProjectMembershipRole.Admin);
const isCompleted = payload.status === SecretScanningScanStatus.Completed;
// We assume that the committer is one of the project members
const isCommitter = isCompleted && payload.authorEmail === member.user.email;
return isAdmin || isCommitter;
});
const timestamp = new Date().toISOString();
const subjectLine =
payload.status === SecretScanningScanStatus.Completed
? "Incident Alert: Secret(s) Leaked"
: `Secret Scanning Failed`;
await notificationService.createUserNotifications(
recipients.map((member) => ({
userId: member.userId,
orgId: project.orgId,
type:
payload.status === SecretScanningScanStatus.Completed
? NotificationType.SECRET_SCANNING_SECRETS_DETECTED
: NotificationType.SECRET_SCANNING_SCAN_FAILED,
title: subjectLine,
body:
payload.status === SecretScanningScanStatus.Completed
? `Uncovered **${payload.numberOfSecrets}** secret(s) ${payload.isDiffScan ? " from a recent commit to" : " in"} **${resourceName}**.`
: `Encountered an error while attempting to scan the resource **${resourceName}**: ${payload.errorMessage}`,
link:
payload.status === SecretScanningScanStatus.Completed
? `/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
: `/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
}))
);
await smtpService.sendMail({
recipients: recipients.map((member) => member.user.email!).filter(Boolean),
template:
payload.status === SecretScanningScanStatus.Completed
? SmtpTemplates.SecretScanningV2SecretsDetected
: SmtpTemplates.SecretScanningV2ScanFailed,
subjectLine,
substitutions:
payload.status === SecretScanningScanStatus.Completed
? {
authorName: payload.authorName,
authorEmail: payload.authorEmail,
resourceName,
numberOfSecrets: payload.numberOfSecrets,
isDiffScan: payload.isDiffScan,
url: encodeURI(
`${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
),
timestamp
}
: {
dataSourceName: dataSource.name,
resourceName,
projectName: project.name,
timestamp,
errorMessage: payload.errorMessage,
url: encodeURI(
`${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
)
}
});
} catch (error) {
logger.error(
error,
`secretScanningV2Queue: Failed to Send Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
);
throw error;
}
},
{
batchSize: 1,
workerCount: 2,
pollingIntervalSeconds: 1
}
);
};
return { return {
queueDataSourceFullScan, queueDataSourceFullScan,
queueResourceDiffScan queueResourceDiffScan,
init
}; };
}; };
backend/src/main.ts
+13 -2
View File
@@ -72,7 +72,7 @@ const run = async () => {
const keyStore = keyStoreFactory(envConfig, keyValueStoreDAL); const keyStore = keyStoreFactory(envConfig, keyValueStoreDAL);
const redis = buildRedisFromConfig(envConfig); const redis = buildRedisFromConfig(envConfig);
const server = await main({ const { server, completeServerInitialization } = await main({
db, db,
auditLogDb, auditLogDb,
superAdminDAL, superAdminDAL,
@@ -140,7 +140,18 @@ const run = async () => {
} }
}); });
logger.info("Migrations complete. Marking server as READY..."); logger.info("Migrations complete. Completing server initialization...");
try {
await completeServerInitialization();
} catch (error) {
logger.error(error, "Failed to complete server initialization");
await server.close();
await queue.shutdown();
process.exit(1);
}
logger.info("Server initialization complete. Marking server as READY...");
markServerReady(); markServerReady();
backend/src/server/app.ts
+2 -2
View File
@@ -221,7 +221,7 @@ export const main = async ({
}; };
}); });
await server.register(registerRoutes, { const completeServerInitialization = await registerRoutes(server, {
smtp, smtp,
queue, queue,
db, db,
@@ -240,7 +240,7 @@ export const main = async ({
await server.ready(); await server.ready();
server.swagger(); server.swagger();
return server; return { server, completeServerInitialization };
} catch (err) { } catch (err) {
server.log.error(err); server.log.error(err);
await queue.shutdown(); await queue.shutdown();
backend/src/server/routes/index.ts
+77 -71
View File
@@ -2206,7 +2206,7 @@ export const registerRoutes = async (
internalCaFns internalCaFns
}); });
await secretRotationV2QueueServiceFactory({ const secretRotationV2Queue = await secretRotationV2QueueServiceFactory({
secretRotationV2Service, secretRotationV2Service,
secretRotationV2DAL, secretRotationV2DAL,
queueService, queueService,
@@ -2300,50 +2300,87 @@ export const registerRoutes = async (
// setup the communication with license key server // setup the communication with license key server
await licenseService.init(); await licenseService.init();
// If FIPS is enabled, we check to ensure that the users license includes FIPS mode. const completeServerInitialization = async () => {
crypto.verifyFipsLicense(licenseService); await superAdminService.initServerCfg();
await superAdminService.initServerCfg(); // If FIPS is enabled, we check to ensure that the users license includes FIPS mode.
crypto.verifyFipsLicense(licenseService);
// Start HSM service if it's configured/enabled. // Start HSM service if it's configured/enabled.
await hsmService.startService(); await hsmService.startService();
const hsmStatus = await isHsmActiveAndEnabled({ const hsmStatus = await isHsmActiveAndEnabled({
hsmService, hsmService,
kmsRootConfigDAL, kmsRootConfigDAL,
licenseService licenseService
}); });
// if the encryption strategy is software - user needs to provide an encryption key // if the encryption strategy is software - user needs to provide an encryption key
// if the encryption strategy is null AND the hsm is not configured - user needs to provide an encryption key // if the encryption strategy is null AND the hsm is not configured - user needs to provide an encryption key
const needsEncryptionKey = const needsEncryptionKey =
hsmStatus.rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.Software || hsmStatus.rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.Software ||
(hsmStatus.rootKmsConfigEncryptionStrategy === null && !hsmStatus.isHsmConfigured); (hsmStatus.rootKmsConfigEncryptionStrategy === null && !hsmStatus.isHsmConfigured);
if (needsEncryptionKey) { if (needsEncryptionKey) {
if (!envConfig.ROOT_ENCRYPTION_KEY && !envConfig.ENCRYPTION_KEY) { if (!envConfig.ROOT_ENCRYPTION_KEY && !envConfig.ENCRYPTION_KEY) {
throw new BadRequestError({ throw new BadRequestError({
message: message:
"Root KMS encryption strategy is set to software. Please set the ENCRYPTION_KEY environment variable and restart your deployment.\nYou can enable HSM encryption in the Server Console." "Root KMS encryption strategy is set to software. Please set the ENCRYPTION_KEY environment variable and restart your deployment.\nYou can enable HSM encryption in the Server Console."
}); });
}
} }
}
await telemetryQueue.startTelemetryCheck(); await telemetryQueue.startTelemetryCheck();
await telemetryQueue.startAggregatedEventsJob(); await telemetryQueue.startAggregatedEventsJob();
await dailyResourceCleanUp.init(); await dailyResourceCleanUp.init();
await healthAlert.init(); await healthAlert.init();
await pkiSyncCleanup.init(); await pkiSyncCleanup.init();
await pamAccountRotation.init(); await pamAccountRotation.init();
await dailyReminderQueueService.startDailyRemindersJob(); await dailyReminderQueueService.startDailyRemindersJob();
await dailyReminderQueueService.startSecretReminderMigrationJob(); await dailyReminderQueueService.startSecretReminderMigrationJob();
await dailyExpiringPkiItemAlert.startSendingAlerts(); await dailyExpiringPkiItemAlert.startSendingAlerts();
await pkiSubscriberQueue.startDailyAutoRenewalJob(); await pkiSubscriberQueue.startDailyAutoRenewalJob();
await certificateV3Queue.init(); await certificateV3Queue.init();
await kmsService.startService(hsmStatus); await kmsService.startService(hsmStatus);
await microsoftTeamsService.start(); await microsoftTeamsService.start();
await dynamicSecretQueueService.init(); await dynamicSecretQueueService.init();
await eventBusService.init(); await secretScanningV2Queue.init();
await secretRotationV2Queue.init();
await notificationQueue.init();
await eventBusService.init();
const cronJobs: CronJob[] = [];
if (appCfg.isProductionMode) {
const rateLimitSyncJob = await rateLimitService.initializeBackgroundSync();
if (rateLimitSyncJob) {
cronJobs.push(rateLimitSyncJob);
}
const licenseSyncJob = await licenseService.initializeBackgroundSync();
if (licenseSyncJob) {
cronJobs.push(licenseSyncJob);
}
const microsoftTeamsSyncJob = await microsoftTeamsService.initializeBackgroundSync();
if (microsoftTeamsSyncJob) {
cronJobs.push(microsoftTeamsSyncJob);
}
const adminIntegrationsSyncJob = await superAdminService.initializeAdminIntegrationConfigSync();
if (adminIntegrationsSyncJob) {
cronJobs.push(adminIntegrationsSyncJob);
}
}
const configSyncJob = await superAdminService.initializeEnvConfigSync();
if (configSyncJob) {
cronJobs.push(configSyncJob);
}
const oauthConfigSyncJob = await initializeOauthConfigSync();
if (oauthConfigSyncJob) {
cronJobs.push(oauthConfigSyncJob);
}
};
// inject all services // inject all services
server.decorate<FastifyZodProvider["services"]>("services", { server.decorate<FastifyZodProvider["services"]>("services", {
@@ -2473,38 +2510,6 @@ export const registerRoutes = async (
convertor: convertorService convertor: convertorService
}); });
const cronJobs: CronJob[] = [];
if (appCfg.isProductionMode) {
const rateLimitSyncJob = await rateLimitService.initializeBackgroundSync();
if (rateLimitSyncJob) {
cronJobs.push(rateLimitSyncJob);
}
const licenseSyncJob = await licenseService.initializeBackgroundSync();
if (licenseSyncJob) {
cronJobs.push(licenseSyncJob);
}
const microsoftTeamsSyncJob = await microsoftTeamsService.initializeBackgroundSync();
if (microsoftTeamsSyncJob) {
cronJobs.push(microsoftTeamsSyncJob);
}
const adminIntegrationsSyncJob = await superAdminService.initializeAdminIntegrationConfigSync();
if (adminIntegrationsSyncJob) {
cronJobs.push(adminIntegrationsSyncJob);
}
}
const configSyncJob = await superAdminService.initializeEnvConfigSync();
if (configSyncJob) {
cronJobs.push(configSyncJob);
}
const oauthConfigSyncJob = await initializeOauthConfigSync();
if (oauthConfigSyncJob) {
cronJobs.push(oauthConfigSyncJob);
}
server.decorate<FastifyZodProvider["store"]>("store", { server.decorate<FastifyZodProvider["store"]>("store", {
user: userDAL, user: userDAL,
kmipClient: kmipClientDAL kmipClient: kmipClientDAL
@@ -2593,9 +2598,10 @@ export const registerRoutes = async (
await server.register(registerV4Routes, { prefix: "/api/v4" }); await server.register(registerV4Routes, { prefix: "/api/v4" });
server.addHook("onClose", async () => { server.addHook("onClose", async () => {
cronJobs.forEach((job) => job.stop());
await telemetryService.flushAll(); await telemetryService.flushAll();
await eventBusService.close(); await eventBusService.close();
sseService.close(); sseService.close();
}); });
return completeServerInitialization;
}; };
backend/src/services/notification/notification-queue.ts
+17 -13
View File
@@ -10,6 +10,7 @@ type TNotificationQueueServiceFactoryDep = {
export type TNotificationQueueServiceFactory = { export type TNotificationQueueServiceFactory = {
pushUserNotifications: (data: TCreateUserNotificationDTO[]) => Promise<void>; pushUserNotifications: (data: TCreateUserNotificationDTO[]) => Promise<void>;
init: () => Promise<void>;
}; };
export const notificationQueueServiceFactory = async ({ export const notificationQueueServiceFactory = async ({
@@ -20,20 +21,23 @@ export const notificationQueueServiceFactory = async ({
await queueService.queuePg(QueueJobs.UserNotification, { notifications: data }); await queueService.queuePg(QueueJobs.UserNotification, { notifications: data });
}; };
await queueService.startPg( const init = async () => {
QueueJobs.UserNotification, await queueService.startPg(
async ([job]) => { QueueJobs.UserNotification,
const { notifications } = job.data as { notifications: TCreateUserNotificationDTO[] }; async ([job]) => {
await userNotificationDAL.batchInsert(notifications); const { notifications } = job.data as { notifications: TCreateUserNotificationDTO[] };
}, await userNotificationDAL.batchInsert(notifications);
{ },
batchSize: 1, {
workerCount: 2, batchSize: 1,
pollingIntervalSeconds: 1 workerCount: 2,
} pollingIntervalSeconds: 1
); }
);
};
return { return {
pushUserNotifications pushUserNotifications,
init
}; };
}; };