mirror of
https://github.com/awatertrevi/infisical.git
synced 2026-09-22 13:39:35 +00:00
misc: updated processes to run after db migration
This commit is contained in:
Sheen Capadngan
@@ -42,168 +42,172 @@ export const secretRotationV2QueueServiceFactory = async ({
|
||||
smtpService,
|
||||
notificationService
|
||||
}: TSecretRotationV2QueueServiceFactoryDep) => {
|
||||
const appCfg = getConfig();
|
||||
const init = async () => {
|
||||
const appCfg = getConfig();
|
||||
|
||||
if (appCfg.isRotationDevelopmentMode) {
|
||||
logger.warn("Secret Rotation V2 is in development mode.");
|
||||
}
|
||||
if (appCfg.isRotationDevelopmentMode) {
|
||||
logger.warn("Secret Rotation V2 is in development mode.");
|
||||
}
|
||||
|
||||
await queueService.startPg<QueueName.SecretRotationV2>(
|
||||
QueueJobs.SecretRotationV2QueueRotations,
|
||||
async () => {
|
||||
try {
|
||||
const rotateBy = getNextUtcRotationInterval();
|
||||
await queueService.startPg<QueueName.SecretRotationV2>(
|
||||
QueueJobs.SecretRotationV2QueueRotations,
|
||||
async () => {
|
||||
try {
|
||||
const rotateBy = getNextUtcRotationInterval();
|
||||
|
||||
const currentTime = new Date();
|
||||
const currentTime = new Date();
|
||||
|
||||
const secretRotations = await secretRotationV2DAL.findSecretRotationsToQueue(rotateBy);
|
||||
const secretRotations = await secretRotationV2DAL.findSecretRotationsToQueue(rotateBy);
|
||||
|
||||
logger.info(
|
||||
`secretRotationV2Queue: Queue Rotations [currentTime=${currentTime.toISOString()}] [rotateBy=${rotateBy.toISOString()}] [count=${
|
||||
secretRotations.length
|
||||
}]`
|
||||
);
|
||||
|
||||
for await (const rotation of secretRotations) {
|
||||
logger.info(
|
||||
`secretRotationV2Queue: Queue Rotation [rotationId=${rotation.id}] [lastRotatedAt=${new Date(
|
||||
rotation.lastRotatedAt
|
||||
).toISOString()}] [rotateAt=${new Date(rotation.nextRotationAt!).toISOString()}]`
|
||||
`secretRotationV2Queue: Queue Rotations [currentTime=${currentTime.toISOString()}] [rotateBy=${rotateBy.toISOString()}] [count=${
|
||||
secretRotations.length
|
||||
}]`
|
||||
);
|
||||
|
||||
const data = {
|
||||
rotationId: rotation.id,
|
||||
queuedAt: currentTime
|
||||
} as TSecretRotationRotateSecretsJobPayload;
|
||||
|
||||
if (appCfg.isTestMode) {
|
||||
logger.warn("secretRotationV2Queue: Manually rotating secrets for test mode");
|
||||
await rotateSecretsFns({
|
||||
job: {
|
||||
id: uuidv4(),
|
||||
data,
|
||||
retryCount: 0,
|
||||
retryLimit: 0
|
||||
},
|
||||
secretRotationV2DAL,
|
||||
secretRotationV2Service
|
||||
});
|
||||
} else {
|
||||
await queueService.queuePg(
|
||||
QueueJobs.SecretRotationV2RotateSecrets,
|
||||
{
|
||||
rotationId: rotation.id,
|
||||
queuedAt: currentTime
|
||||
},
|
||||
getSecretRotationRotateSecretJobOptions(rotation)
|
||||
for await (const rotation of secretRotations) {
|
||||
logger.info(
|
||||
`secretRotationV2Queue: Queue Rotation [rotationId=${rotation.id}] [lastRotatedAt=${new Date(
|
||||
rotation.lastRotatedAt
|
||||
).toISOString()}] [rotateAt=${new Date(rotation.nextRotationAt!).toISOString()}]`
|
||||
);
|
||||
|
||||
const data = {
|
||||
rotationId: rotation.id,
|
||||
queuedAt: currentTime
|
||||
} as TSecretRotationRotateSecretsJobPayload;
|
||||
|
||||
if (appCfg.isTestMode) {
|
||||
logger.warn("secretRotationV2Queue: Manually rotating secrets for test mode");
|
||||
await rotateSecretsFns({
|
||||
job: {
|
||||
id: uuidv4(),
|
||||
data,
|
||||
retryCount: 0,
|
||||
retryLimit: 0
|
||||
},
|
||||
secretRotationV2DAL,
|
||||
secretRotationV2Service
|
||||
});
|
||||
} else {
|
||||
await queueService.queuePg(
|
||||
QueueJobs.SecretRotationV2RotateSecrets,
|
||||
{
|
||||
rotationId: rotation.id,
|
||||
queuedAt: currentTime
|
||||
},
|
||||
getSecretRotationRotateSecretJobOptions(rotation)
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(error, "secretRotationV2Queue: Queue Rotations Error:");
|
||||
throw error;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(error, "secretRotationV2Queue: Queue Rotations Error:");
|
||||
throw error;
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 1,
|
||||
pollingIntervalSeconds: appCfg.isRotationDevelopmentMode ? 0.5 : 30
|
||||
}
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 1,
|
||||
pollingIntervalSeconds: appCfg.isRotationDevelopmentMode ? 0.5 : 30
|
||||
}
|
||||
);
|
||||
);
|
||||
|
||||
await queueService.startPg<QueueName.SecretRotationV2>(
|
||||
QueueJobs.SecretRotationV2RotateSecrets,
|
||||
async ([job]) => {
|
||||
await rotateSecretsFns({
|
||||
job: {
|
||||
...job,
|
||||
data: job.data as TSecretRotationRotateSecretsJobPayload
|
||||
},
|
||||
secretRotationV2DAL,
|
||||
secretRotationV2Service
|
||||
});
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 2,
|
||||
pollingIntervalSeconds: 0.5
|
||||
}
|
||||
);
|
||||
|
||||
await queueService.startPg<QueueName.SecretRotationV2>(
|
||||
QueueJobs.SecretRotationV2SendNotification,
|
||||
async ([job]) => {
|
||||
const { secretRotation } = job.data as TSecretRotationSendNotificationJobPayload;
|
||||
try {
|
||||
const {
|
||||
name: rotationName,
|
||||
type,
|
||||
projectId,
|
||||
lastRotationAttemptedAt,
|
||||
folder,
|
||||
environment,
|
||||
id: rotationId
|
||||
} = secretRotation;
|
||||
|
||||
logger.info(`secretRotationV2Queue: Sending Status Notification [rotationId=${rotationId}]`);
|
||||
|
||||
const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
|
||||
const project = await projectDAL.findById(projectId);
|
||||
|
||||
const projectAdmins = projectMembers.filter((member) =>
|
||||
member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
|
||||
);
|
||||
|
||||
const rotationType = SECRET_ROTATION_NAME_MAP[type as SecretRotation];
|
||||
|
||||
const rotationPath = `/projects/secret-management/${projectId}/secrets/${environment.slug}`;
|
||||
|
||||
await notificationService.createUserNotifications(
|
||||
projectAdmins.map((admin) => ({
|
||||
userId: admin.userId,
|
||||
orgId: project.orgId,
|
||||
type: NotificationType.SECRET_ROTATION_FAILED,
|
||||
title: "Secret Rotation Failed",
|
||||
body: `Your **${rotationType}** rotation **${rotationName}** failed to rotate.`,
|
||||
link: rotationPath
|
||||
}))
|
||||
);
|
||||
|
||||
await smtpService.sendMail({
|
||||
recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
|
||||
template: SmtpTemplates.SecretRotationFailed,
|
||||
subjectLine: `Secret Rotation Failed`,
|
||||
substitutions: {
|
||||
rotationName,
|
||||
rotationType,
|
||||
content: `Your ${rotationType} Rotation failed to rotate during it's scheduled rotation. The last rotation attempt occurred at ${new Date(
|
||||
lastRotationAttemptedAt
|
||||
).toISOString()}. Please check the rotation status in Infisical for more details.`,
|
||||
secretPath: folder.path,
|
||||
environment: environment.name,
|
||||
projectName: project.name,
|
||||
rotationUrl: encodeURI(`${appCfg.SITE_URL}${rotationPath}`)
|
||||
}
|
||||
await queueService.startPg<QueueName.SecretRotationV2>(
|
||||
QueueJobs.SecretRotationV2RotateSecrets,
|
||||
async ([job]) => {
|
||||
await rotateSecretsFns({
|
||||
job: {
|
||||
...job,
|
||||
data: job.data as TSecretRotationRotateSecretsJobPayload
|
||||
},
|
||||
secretRotationV2DAL,
|
||||
secretRotationV2Service
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
error,
|
||||
`secretRotationV2Queue: Failed to Send Status Notification [rotationId=${secretRotation.id}]`
|
||||
);
|
||||
throw error;
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 2,
|
||||
pollingIntervalSeconds: 0.5
|
||||
}
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 2,
|
||||
pollingIntervalSeconds: 1
|
||||
}
|
||||
);
|
||||
);
|
||||
|
||||
await queueService.schedulePg(
|
||||
QueueJobs.SecretRotationV2QueueRotations,
|
||||
appCfg.isRotationDevelopmentMode ? "* * * * *" : "0 0 * * *",
|
||||
undefined,
|
||||
{ tz: "UTC" }
|
||||
);
|
||||
await queueService.startPg<QueueName.SecretRotationV2>(
|
||||
QueueJobs.SecretRotationV2SendNotification,
|
||||
async ([job]) => {
|
||||
const { secretRotation } = job.data as TSecretRotationSendNotificationJobPayload;
|
||||
try {
|
||||
const {
|
||||
name: rotationName,
|
||||
type,
|
||||
projectId,
|
||||
lastRotationAttemptedAt,
|
||||
folder,
|
||||
environment,
|
||||
id: rotationId
|
||||
} = secretRotation;
|
||||
|
||||
logger.info(`secretRotationV2Queue: Sending Status Notification [rotationId=${rotationId}]`);
|
||||
|
||||
const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
|
||||
const project = await projectDAL.findById(projectId);
|
||||
|
||||
const projectAdmins = projectMembers.filter((member) =>
|
||||
member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
|
||||
);
|
||||
|
||||
const rotationType = SECRET_ROTATION_NAME_MAP[type as SecretRotation];
|
||||
|
||||
const rotationPath = `/projects/secret-management/${projectId}/secrets/${environment.slug}`;
|
||||
|
||||
await notificationService.createUserNotifications(
|
||||
projectAdmins.map((admin) => ({
|
||||
userId: admin.userId,
|
||||
orgId: project.orgId,
|
||||
type: NotificationType.SECRET_ROTATION_FAILED,
|
||||
title: "Secret Rotation Failed",
|
||||
body: `Your **${rotationType}** rotation **${rotationName}** failed to rotate.`,
|
||||
link: rotationPath
|
||||
}))
|
||||
);
|
||||
|
||||
await smtpService.sendMail({
|
||||
recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
|
||||
template: SmtpTemplates.SecretRotationFailed,
|
||||
subjectLine: `Secret Rotation Failed`,
|
||||
substitutions: {
|
||||
rotationName,
|
||||
rotationType,
|
||||
content: `Your ${rotationType} Rotation failed to rotate during it's scheduled rotation. The last rotation attempt occurred at ${new Date(
|
||||
lastRotationAttemptedAt
|
||||
).toISOString()}. Please check the rotation status in Infisical for more details.`,
|
||||
secretPath: folder.path,
|
||||
environment: environment.name,
|
||||
projectName: project.name,
|
||||
rotationUrl: encodeURI(`${appCfg.SITE_URL}${rotationPath}`)
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
error,
|
||||
`secretRotationV2Queue: Failed to Send Status Notification [rotationId=${secretRotation.id}]`
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 2,
|
||||
pollingIntervalSeconds: 1
|
||||
}
|
||||
);
|
||||
|
||||
await queueService.schedulePg(
|
||||
QueueJobs.SecretRotationV2QueueRotations,
|
||||
appCfg.isRotationDevelopmentMode ? "* * * * *" : "0 0 * * *",
|
||||
undefined,
|
||||
{ tz: "UTC" }
|
||||
);
|
||||
};
|
||||
|
||||
return { init };
|
||||
};
|
||||
|
||||
@@ -141,202 +141,6 @@ export const secretScanningV2QueueServiceFactory = async ({
|
||||
}
|
||||
};
|
||||
|
||||
await queueService.startPg<QueueName.SecretScanningV2>(
|
||||
QueueJobs.SecretScanningV2FullScan,
|
||||
async ([job]) => {
|
||||
const { scanId, resourceId, dataSourceId } = job.data as TQueueSecretScanningDataSourceFullScan;
|
||||
const { retryCount, retryLimit } = job;
|
||||
|
||||
const logDetails = `[scanId=${scanId}] [resourceId=${resourceId}] [dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
|
||||
|
||||
const tempFolder = await createTempFolder();
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
|
||||
|
||||
const resource = await secretScanningV2DAL.resources.findById(resourceId);
|
||||
|
||||
if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
|
||||
|
||||
let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;
|
||||
|
||||
try {
|
||||
try {
|
||||
lock = await keyStore.acquireLock(
|
||||
[KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId)],
|
||||
60 * 1000 * 5
|
||||
);
|
||||
} catch (e) {
|
||||
throw new Error("Failed to acquire scanning lock.");
|
||||
}
|
||||
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Scanning
|
||||
}
|
||||
);
|
||||
|
||||
let connection: TAppConnection | null = null;
|
||||
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
|
||||
|
||||
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
|
||||
kmsService,
|
||||
appConnectionDAL
|
||||
});
|
||||
|
||||
const findingsPath = join(tempFolder, "findings.json");
|
||||
|
||||
const scanPath = await factory.getFullScanPath({
|
||||
dataSource: {
|
||||
...dataSource,
|
||||
connection
|
||||
} as TSecretScanningDataSourceWithConnection,
|
||||
resourceName: resource.name,
|
||||
tempFolder
|
||||
});
|
||||
|
||||
const config = await secretScanningV2DAL.configs.findOne({
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
let configPath: string | undefined;
|
||||
|
||||
if (config && config.content) {
|
||||
configPath = join(tempFolder, "infisical-scan.toml");
|
||||
await writeTextToFile(configPath, config.content);
|
||||
}
|
||||
|
||||
let findingsPayload: TFindingsPayload;
|
||||
switch (resource.type) {
|
||||
case SecretScanningResource.Repository:
|
||||
case SecretScanningResource.Project:
|
||||
findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath, configPath);
|
||||
break;
|
||||
default:
|
||||
throw new Error("Unhandled resource type");
|
||||
}
|
||||
|
||||
const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
|
||||
let findings: TSecretScanningFindings[] = [];
|
||||
if (findingsPayload.length) {
|
||||
findings = await secretScanningV2DAL.findings.upsert(
|
||||
findingsPayload.map((finding) => ({
|
||||
...finding,
|
||||
projectId: dataSource.projectId,
|
||||
dataSourceName: dataSource.name,
|
||||
dataSourceType: dataSource.type,
|
||||
resourceName: resource.name,
|
||||
resourceType: resource.type,
|
||||
scanId
|
||||
})),
|
||||
["projectId", "fingerprint"],
|
||||
tx,
|
||||
["resourceName", "dataSourceName"]
|
||||
);
|
||||
}
|
||||
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Completed,
|
||||
statusMessage: null
|
||||
}
|
||||
);
|
||||
|
||||
return findings;
|
||||
});
|
||||
|
||||
const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
|
||||
|
||||
if (newFindings.length) {
|
||||
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
|
||||
status: SecretScanningScanStatus.Completed,
|
||||
resourceName: resource.name,
|
||||
isDiffScan: false,
|
||||
dataSource,
|
||||
numberOfSecrets: newFindings.length,
|
||||
scanId
|
||||
});
|
||||
}
|
||||
|
||||
await auditLogService.createAuditLog({
|
||||
projectId: dataSource.projectId,
|
||||
actor: {
|
||||
type: ActorType.PLATFORM,
|
||||
metadata: {}
|
||||
},
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
|
||||
metadata: {
|
||||
dataSourceId: dataSource.id,
|
||||
dataSourceType: dataSource.type,
|
||||
resourceId: resource.id,
|
||||
resourceType: resource.type,
|
||||
scanId,
|
||||
scanStatus: SecretScanningScanStatus.Completed,
|
||||
scanType: SecretScanningScanType.FullScan,
|
||||
numberOfSecretsDetected: findingsPayload.length
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails} findings=[${findingsPayload.length}]`);
|
||||
} catch (error) {
|
||||
if (retryCount === retryLimit) {
|
||||
const errorMessage = parseScanErrorMessage(error);
|
||||
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Failed,
|
||||
statusMessage: errorMessage
|
||||
}
|
||||
);
|
||||
|
||||
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
|
||||
status: SecretScanningScanStatus.Failed,
|
||||
resourceName: resource.name,
|
||||
dataSource,
|
||||
errorMessage
|
||||
});
|
||||
|
||||
await auditLogService.createAuditLog({
|
||||
projectId: dataSource.projectId,
|
||||
actor: {
|
||||
type: ActorType.PLATFORM,
|
||||
metadata: {}
|
||||
},
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
|
||||
metadata: {
|
||||
dataSourceId: dataSource.id,
|
||||
dataSourceType: dataSource.type,
|
||||
resourceId: resource.id,
|
||||
resourceType: resource.type,
|
||||
scanId,
|
||||
scanStatus: SecretScanningScanStatus.Failed,
|
||||
scanType: SecretScanningScanType.FullScan
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
logger.error(error, `secretScanningV2Queue: Full Scan Failed ${logDetails}`);
|
||||
throw error;
|
||||
} finally {
|
||||
await deleteTempFolder(tempFolder);
|
||||
await lock?.release();
|
||||
}
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 2,
|
||||
pollingIntervalSeconds: 1
|
||||
}
|
||||
);
|
||||
|
||||
const queueResourceDiffScan = async ({
|
||||
payload,
|
||||
dataSourceId,
|
||||
@@ -391,148 +195,127 @@ export const secretScanningV2QueueServiceFactory = async ({
|
||||
}
|
||||
};
|
||||
|
||||
await queueService.startPg<QueueName.SecretScanningV2>(
|
||||
QueueJobs.SecretScanningV2DiffScan,
|
||||
async ([job]) => {
|
||||
const { payload, dataSourceId, resourceId, scanId } = job.data as TQueueSecretScanningResourceDiffScan;
|
||||
const { retryCount, retryLimit } = job;
|
||||
const init = async () => {
|
||||
await queueService.startPg<QueueName.SecretScanningV2>(
|
||||
QueueJobs.SecretScanningV2FullScan,
|
||||
async ([job]) => {
|
||||
const { scanId, resourceId, dataSourceId } = job.data as TQueueSecretScanningDataSourceFullScan;
|
||||
const { retryCount, retryLimit } = job;
|
||||
|
||||
const logDetails = `[dataSourceId=${dataSourceId}] [scanId=${scanId}] [resourceId=${resourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
|
||||
const logDetails = `[scanId=${scanId}] [resourceId=${resourceId}] [dataSourceId=${dataSourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
|
||||
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
const tempFolder = await createTempFolder();
|
||||
|
||||
if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
const resource = await secretScanningV2DAL.resources.findById(resourceId);
|
||||
if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
|
||||
|
||||
if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
|
||||
const resource = await secretScanningV2DAL.resources.findById(resourceId);
|
||||
|
||||
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
|
||||
kmsService,
|
||||
appConnectionDAL
|
||||
});
|
||||
if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
|
||||
|
||||
const tempFolder = await createTempFolder();
|
||||
let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;
|
||||
|
||||
try {
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Scanning
|
||||
}
|
||||
);
|
||||
|
||||
let connection: TAppConnection | null = null;
|
||||
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
|
||||
|
||||
const config = await secretScanningV2DAL.configs.findOne({
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
let configPath: string | undefined;
|
||||
|
||||
if (config && config.content) {
|
||||
configPath = join(tempFolder, "infisical-scan.toml");
|
||||
await writeTextToFile(configPath, config.content);
|
||||
}
|
||||
|
||||
const findingsPayload = await factory.getDiffScanFindingsPayload({
|
||||
dataSource: {
|
||||
...dataSource,
|
||||
connection
|
||||
} as TSecretScanningDataSourceWithConnection,
|
||||
resourceName: resource.name,
|
||||
payload,
|
||||
configPath
|
||||
});
|
||||
|
||||
const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
|
||||
let findings: TSecretScanningFindings[] = [];
|
||||
|
||||
if (findingsPayload.length) {
|
||||
findings = await secretScanningV2DAL.findings.upsert(
|
||||
findingsPayload.map((finding) => ({
|
||||
...finding,
|
||||
projectId: dataSource.projectId,
|
||||
dataSourceName: dataSource.name,
|
||||
dataSourceType: dataSource.type,
|
||||
resourceName: resource.name,
|
||||
resourceType: resource.type,
|
||||
scanId
|
||||
})),
|
||||
["projectId", "fingerprint"],
|
||||
tx,
|
||||
["resourceName", "dataSourceName"]
|
||||
try {
|
||||
try {
|
||||
lock = await keyStore.acquireLock(
|
||||
[KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId)],
|
||||
60 * 1000 * 5
|
||||
);
|
||||
} catch (e) {
|
||||
throw new Error("Failed to acquire scanning lock.");
|
||||
}
|
||||
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Completed
|
||||
status: SecretScanningScanStatus.Scanning
|
||||
}
|
||||
);
|
||||
|
||||
return findings;
|
||||
});
|
||||
let connection: TAppConnection | null = null;
|
||||
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
|
||||
|
||||
const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
|
||||
|
||||
if (newFindings.length) {
|
||||
const finding = newFindings[0] as TSecretScanningFinding;
|
||||
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
|
||||
status: SecretScanningScanStatus.Completed,
|
||||
resourceName: resource.name,
|
||||
isDiffScan: true,
|
||||
dataSource,
|
||||
numberOfSecrets: newFindings.length,
|
||||
scanId,
|
||||
authorName: finding?.details?.author,
|
||||
authorEmail: finding?.details?.email
|
||||
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
|
||||
kmsService,
|
||||
appConnectionDAL
|
||||
});
|
||||
}
|
||||
|
||||
await auditLogService.createAuditLog({
|
||||
projectId: dataSource.projectId,
|
||||
actor: {
|
||||
type: ActorType.PLATFORM,
|
||||
metadata: {}
|
||||
},
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
|
||||
metadata: {
|
||||
dataSourceId: dataSource.id,
|
||||
dataSourceType: dataSource.type,
|
||||
resourceId,
|
||||
resourceType: resource.type,
|
||||
scanId,
|
||||
scanStatus: SecretScanningScanStatus.Completed,
|
||||
scanType: SecretScanningScanType.DiffScan,
|
||||
numberOfSecretsDetected: findingsPayload.length
|
||||
}
|
||||
const findingsPath = join(tempFolder, "findings.json");
|
||||
|
||||
const scanPath = await factory.getFullScanPath({
|
||||
dataSource: {
|
||||
...dataSource,
|
||||
connection
|
||||
} as TSecretScanningDataSourceWithConnection,
|
||||
resourceName: resource.name,
|
||||
tempFolder
|
||||
});
|
||||
|
||||
const config = await secretScanningV2DAL.configs.findOne({
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
let configPath: string | undefined;
|
||||
|
||||
if (config && config.content) {
|
||||
configPath = join(tempFolder, "infisical-scan.toml");
|
||||
await writeTextToFile(configPath, config.content);
|
||||
}
|
||||
});
|
||||
|
||||
logger.info(`secretScanningV2Queue: Diff Scan Complete ${logDetails}`);
|
||||
} catch (error) {
|
||||
if (retryCount === retryLimit) {
|
||||
const errorMessage = parseScanErrorMessage(error);
|
||||
let findingsPayload: TFindingsPayload;
|
||||
switch (resource.type) {
|
||||
case SecretScanningResource.Repository:
|
||||
case SecretScanningResource.Project:
|
||||
findingsPayload = await scanGitRepositoryAndGetFindings(scanPath, findingsPath, configPath);
|
||||
break;
|
||||
default:
|
||||
throw new Error("Unhandled resource type");
|
||||
}
|
||||
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Failed,
|
||||
statusMessage: errorMessage
|
||||
const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
|
||||
let findings: TSecretScanningFindings[] = [];
|
||||
if (findingsPayload.length) {
|
||||
findings = await secretScanningV2DAL.findings.upsert(
|
||||
findingsPayload.map((finding) => ({
|
||||
...finding,
|
||||
projectId: dataSource.projectId,
|
||||
dataSourceName: dataSource.name,
|
||||
dataSourceType: dataSource.type,
|
||||
resourceName: resource.name,
|
||||
resourceType: resource.type,
|
||||
scanId
|
||||
})),
|
||||
["projectId", "fingerprint"],
|
||||
tx,
|
||||
["resourceName", "dataSourceName"]
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
|
||||
status: SecretScanningScanStatus.Failed,
|
||||
resourceName: resource.name,
|
||||
dataSource,
|
||||
errorMessage
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Completed,
|
||||
statusMessage: null
|
||||
}
|
||||
);
|
||||
|
||||
return findings;
|
||||
});
|
||||
|
||||
const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
|
||||
|
||||
if (newFindings.length) {
|
||||
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
|
||||
status: SecretScanningScanStatus.Completed,
|
||||
resourceName: resource.name,
|
||||
isDiffScan: false,
|
||||
dataSource,
|
||||
numberOfSecrets: newFindings.length,
|
||||
scanId
|
||||
});
|
||||
}
|
||||
|
||||
await auditLogService.createAuditLog({
|
||||
projectId: dataSource.projectId,
|
||||
actor: {
|
||||
@@ -547,128 +330,348 @@ export const secretScanningV2QueueServiceFactory = async ({
|
||||
resourceId: resource.id,
|
||||
resourceType: resource.type,
|
||||
scanId,
|
||||
scanStatus: SecretScanningScanStatus.Failed,
|
||||
scanType: SecretScanningScanType.DiffScan
|
||||
scanStatus: SecretScanningScanStatus.Completed,
|
||||
scanType: SecretScanningScanType.FullScan,
|
||||
numberOfSecretsDetected: findingsPayload.length
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
logger.info(`secretScanningV2Queue: Full Scan Complete ${logDetails} findings=[${findingsPayload.length}]`);
|
||||
} catch (error) {
|
||||
if (retryCount === retryLimit) {
|
||||
const errorMessage = parseScanErrorMessage(error);
|
||||
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Failed,
|
||||
statusMessage: errorMessage
|
||||
}
|
||||
);
|
||||
|
||||
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
|
||||
status: SecretScanningScanStatus.Failed,
|
||||
resourceName: resource.name,
|
||||
dataSource,
|
||||
errorMessage
|
||||
});
|
||||
|
||||
await auditLogService.createAuditLog({
|
||||
projectId: dataSource.projectId,
|
||||
actor: {
|
||||
type: ActorType.PLATFORM,
|
||||
metadata: {}
|
||||
},
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
|
||||
metadata: {
|
||||
dataSourceId: dataSource.id,
|
||||
dataSourceType: dataSource.type,
|
||||
resourceId: resource.id,
|
||||
resourceType: resource.type,
|
||||
scanId,
|
||||
scanStatus: SecretScanningScanStatus.Failed,
|
||||
scanType: SecretScanningScanType.FullScan
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
logger.error(error, `secretScanningV2Queue: Full Scan Failed ${logDetails}`);
|
||||
throw error;
|
||||
} finally {
|
||||
await deleteTempFolder(tempFolder);
|
||||
await lock?.release();
|
||||
}
|
||||
|
||||
logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`);
|
||||
throw error;
|
||||
} finally {
|
||||
await deleteTempFolder(tempFolder);
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 2,
|
||||
pollingIntervalSeconds: 1
|
||||
}
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 2,
|
||||
pollingIntervalSeconds: 1
|
||||
}
|
||||
);
|
||||
);
|
||||
|
||||
await queueService.startPg<QueueName.SecretScanningV2>(
|
||||
QueueJobs.SecretScanningV2SendNotification,
|
||||
async ([job]) => {
|
||||
const { dataSource, resourceName, ...payload } = job.data as TQueueSecretScanningSendNotification;
|
||||
await queueService.startPg<QueueName.SecretScanningV2>(
|
||||
QueueJobs.SecretScanningV2DiffScan,
|
||||
async ([job]) => {
|
||||
const { payload, dataSourceId, resourceId, scanId } = job.data as TQueueSecretScanningResourceDiffScan;
|
||||
const { retryCount, retryLimit } = job;
|
||||
|
||||
const appCfg = getConfig();
|
||||
const logDetails = `[dataSourceId=${dataSourceId}] [scanId=${scanId}] [resourceId=${resourceId}] [jobId=${job.id}] retryCount=[${retryCount}/${retryLimit}]`;
|
||||
|
||||
if (!appCfg.isSmtpConfigured) return;
|
||||
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
|
||||
|
||||
try {
|
||||
const { projectId } = dataSource;
|
||||
if (!dataSource) throw new Error(`Data source with ID "${dataSourceId}" not found`);
|
||||
|
||||
logger.info(
|
||||
`secretScanningV2Queue: Sending Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
|
||||
);
|
||||
const resource = await secretScanningV2DAL.resources.findById(resourceId);
|
||||
|
||||
const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
|
||||
const project = await projectDAL.findById(projectId);
|
||||
if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
|
||||
|
||||
const recipients = projectMembers.filter((member) => {
|
||||
const isAdmin = member.roles.some((role) => role.role === ProjectMembershipRole.Admin);
|
||||
const isCompleted = payload.status === SecretScanningScanStatus.Completed;
|
||||
// We assume that the committer is one of the project members
|
||||
const isCommitter = isCompleted && payload.authorEmail === member.user.email;
|
||||
return isAdmin || isCommitter;
|
||||
const factory = SECRET_SCANNING_FACTORY_MAP[dataSource.type as SecretScanningDataSource]({
|
||||
kmsService,
|
||||
appConnectionDAL
|
||||
});
|
||||
|
||||
const timestamp = new Date().toISOString();
|
||||
const tempFolder = await createTempFolder();
|
||||
|
||||
const subjectLine =
|
||||
payload.status === SecretScanningScanStatus.Completed
|
||||
? "Incident Alert: Secret(s) Leaked"
|
||||
: `Secret Scanning Failed`;
|
||||
try {
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Scanning
|
||||
}
|
||||
);
|
||||
|
||||
await notificationService.createUserNotifications(
|
||||
recipients.map((member) => ({
|
||||
userId: member.userId,
|
||||
orgId: project.orgId,
|
||||
type:
|
||||
payload.status === SecretScanningScanStatus.Completed
|
||||
? NotificationType.SECRET_SCANNING_SECRETS_DETECTED
|
||||
: NotificationType.SECRET_SCANNING_SCAN_FAILED,
|
||||
title: subjectLine,
|
||||
body:
|
||||
payload.status === SecretScanningScanStatus.Completed
|
||||
? `Uncovered **${payload.numberOfSecrets}** secret(s) ${payload.isDiffScan ? " from a recent commit to" : " in"} **${resourceName}**.`
|
||||
: `Encountered an error while attempting to scan the resource **${resourceName}**: ${payload.errorMessage}`,
|
||||
link:
|
||||
payload.status === SecretScanningScanStatus.Completed
|
||||
? `/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
|
||||
: `/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
|
||||
}))
|
||||
);
|
||||
let connection: TAppConnection | null = null;
|
||||
if (dataSource.connection) connection = await decryptAppConnection(dataSource.connection, kmsService);
|
||||
|
||||
await smtpService.sendMail({
|
||||
recipients: recipients.map((member) => member.user.email!).filter(Boolean),
|
||||
template:
|
||||
payload.status === SecretScanningScanStatus.Completed
|
||||
? SmtpTemplates.SecretScanningV2SecretsDetected
|
||||
: SmtpTemplates.SecretScanningV2ScanFailed,
|
||||
subjectLine,
|
||||
substitutions:
|
||||
payload.status === SecretScanningScanStatus.Completed
|
||||
? {
|
||||
authorName: payload.authorName,
|
||||
authorEmail: payload.authorEmail,
|
||||
resourceName,
|
||||
numberOfSecrets: payload.numberOfSecrets,
|
||||
isDiffScan: payload.isDiffScan,
|
||||
url: encodeURI(
|
||||
`${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
|
||||
),
|
||||
timestamp
|
||||
}
|
||||
: {
|
||||
const config = await secretScanningV2DAL.configs.findOne({
|
||||
projectId: dataSource.projectId
|
||||
});
|
||||
|
||||
let configPath: string | undefined;
|
||||
|
||||
if (config && config.content) {
|
||||
configPath = join(tempFolder, "infisical-scan.toml");
|
||||
await writeTextToFile(configPath, config.content);
|
||||
}
|
||||
|
||||
const findingsPayload = await factory.getDiffScanFindingsPayload({
|
||||
dataSource: {
|
||||
...dataSource,
|
||||
connection
|
||||
} as TSecretScanningDataSourceWithConnection,
|
||||
resourceName: resource.name,
|
||||
payload,
|
||||
configPath
|
||||
});
|
||||
|
||||
const allFindings = await secretScanningV2DAL.findings.transaction(async (tx) => {
|
||||
let findings: TSecretScanningFindings[] = [];
|
||||
|
||||
if (findingsPayload.length) {
|
||||
findings = await secretScanningV2DAL.findings.upsert(
|
||||
findingsPayload.map((finding) => ({
|
||||
...finding,
|
||||
projectId: dataSource.projectId,
|
||||
dataSourceName: dataSource.name,
|
||||
resourceName,
|
||||
projectName: project.name,
|
||||
timestamp,
|
||||
errorMessage: payload.errorMessage,
|
||||
url: encodeURI(
|
||||
`${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
|
||||
)
|
||||
dataSourceType: dataSource.type,
|
||||
resourceName: resource.name,
|
||||
resourceType: resource.type,
|
||||
scanId
|
||||
})),
|
||||
["projectId", "fingerprint"],
|
||||
tx,
|
||||
["resourceName", "dataSourceName"]
|
||||
);
|
||||
}
|
||||
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Completed
|
||||
}
|
||||
);
|
||||
|
||||
return findings;
|
||||
});
|
||||
|
||||
const newFindings = allFindings.filter((finding) => finding.scanId === scanId);
|
||||
|
||||
if (newFindings.length) {
|
||||
const finding = newFindings[0] as TSecretScanningFinding;
|
||||
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
|
||||
status: SecretScanningScanStatus.Completed,
|
||||
resourceName: resource.name,
|
||||
isDiffScan: true,
|
||||
dataSource,
|
||||
numberOfSecrets: newFindings.length,
|
||||
scanId,
|
||||
authorName: finding?.details?.author,
|
||||
authorEmail: finding?.details?.email
|
||||
});
|
||||
}
|
||||
|
||||
await auditLogService.createAuditLog({
|
||||
projectId: dataSource.projectId,
|
||||
actor: {
|
||||
type: ActorType.PLATFORM,
|
||||
metadata: {}
|
||||
},
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
|
||||
metadata: {
|
||||
dataSourceId: dataSource.id,
|
||||
dataSourceType: dataSource.type,
|
||||
resourceId,
|
||||
resourceType: resource.type,
|
||||
scanId,
|
||||
scanStatus: SecretScanningScanStatus.Completed,
|
||||
scanType: SecretScanningScanType.DiffScan,
|
||||
numberOfSecretsDetected: findingsPayload.length
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
logger.info(`secretScanningV2Queue: Diff Scan Complete ${logDetails}`);
|
||||
} catch (error) {
|
||||
if (retryCount === retryLimit) {
|
||||
const errorMessage = parseScanErrorMessage(error);
|
||||
|
||||
await secretScanningV2DAL.scans.update(
|
||||
{ id: scanId },
|
||||
{
|
||||
status: SecretScanningScanStatus.Failed,
|
||||
statusMessage: errorMessage
|
||||
}
|
||||
);
|
||||
|
||||
await queueService.queuePg(QueueJobs.SecretScanningV2SendNotification, {
|
||||
status: SecretScanningScanStatus.Failed,
|
||||
resourceName: resource.name,
|
||||
dataSource,
|
||||
errorMessage
|
||||
});
|
||||
|
||||
await auditLogService.createAuditLog({
|
||||
projectId: dataSource.projectId,
|
||||
actor: {
|
||||
type: ActorType.PLATFORM,
|
||||
metadata: {}
|
||||
},
|
||||
event: {
|
||||
type: EventType.SECRET_SCANNING_DATA_SOURCE_SCAN,
|
||||
metadata: {
|
||||
dataSourceId: dataSource.id,
|
||||
dataSourceType: dataSource.type,
|
||||
resourceId: resource.id,
|
||||
resourceType: resource.type,
|
||||
scanId,
|
||||
scanStatus: SecretScanningScanStatus.Failed,
|
||||
scanType: SecretScanningScanType.DiffScan
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
error,
|
||||
`secretScanningV2Queue: Failed to Send Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
logger.error(error, `secretScanningV2Queue: Diff Scan Failed ${logDetails}`);
|
||||
throw error;
|
||||
} finally {
|
||||
await deleteTempFolder(tempFolder);
|
||||
}
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 2,
|
||||
pollingIntervalSeconds: 1
|
||||
}
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 2,
|
||||
pollingIntervalSeconds: 1
|
||||
}
|
||||
);
|
||||
);
|
||||
|
||||
await queueService.startPg<QueueName.SecretScanningV2>(
|
||||
QueueJobs.SecretScanningV2SendNotification,
|
||||
async ([job]) => {
|
||||
const { dataSource, resourceName, ...payload } = job.data as TQueueSecretScanningSendNotification;
|
||||
|
||||
const appCfg = getConfig();
|
||||
|
||||
if (!appCfg.isSmtpConfigured) return;
|
||||
|
||||
try {
|
||||
const { projectId } = dataSource;
|
||||
|
||||
logger.info(
|
||||
`secretScanningV2Queue: Sending Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
|
||||
);
|
||||
|
||||
const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
|
||||
const project = await projectDAL.findById(projectId);
|
||||
|
||||
const recipients = projectMembers.filter((member) => {
|
||||
const isAdmin = member.roles.some((role) => role.role === ProjectMembershipRole.Admin);
|
||||
const isCompleted = payload.status === SecretScanningScanStatus.Completed;
|
||||
// We assume that the committer is one of the project members
|
||||
const isCommitter = isCompleted && payload.authorEmail === member.user.email;
|
||||
return isAdmin || isCommitter;
|
||||
});
|
||||
|
||||
const timestamp = new Date().toISOString();
|
||||
|
||||
const subjectLine =
|
||||
payload.status === SecretScanningScanStatus.Completed
|
||||
? "Incident Alert: Secret(s) Leaked"
|
||||
: `Secret Scanning Failed`;
|
||||
|
||||
await notificationService.createUserNotifications(
|
||||
recipients.map((member) => ({
|
||||
userId: member.userId,
|
||||
orgId: project.orgId,
|
||||
type:
|
||||
payload.status === SecretScanningScanStatus.Completed
|
||||
? NotificationType.SECRET_SCANNING_SECRETS_DETECTED
|
||||
: NotificationType.SECRET_SCANNING_SCAN_FAILED,
|
||||
title: subjectLine,
|
||||
body:
|
||||
payload.status === SecretScanningScanStatus.Completed
|
||||
? `Uncovered **${payload.numberOfSecrets}** secret(s) ${payload.isDiffScan ? " from a recent commit to" : " in"} **${resourceName}**.`
|
||||
: `Encountered an error while attempting to scan the resource **${resourceName}**: ${payload.errorMessage}`,
|
||||
link:
|
||||
payload.status === SecretScanningScanStatus.Completed
|
||||
? `/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
|
||||
: `/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
|
||||
}))
|
||||
);
|
||||
|
||||
await smtpService.sendMail({
|
||||
recipients: recipients.map((member) => member.user.email!).filter(Boolean),
|
||||
template:
|
||||
payload.status === SecretScanningScanStatus.Completed
|
||||
? SmtpTemplates.SecretScanningV2SecretsDetected
|
||||
: SmtpTemplates.SecretScanningV2ScanFailed,
|
||||
subjectLine,
|
||||
substitutions:
|
||||
payload.status === SecretScanningScanStatus.Completed
|
||||
? {
|
||||
authorName: payload.authorName,
|
||||
authorEmail: payload.authorEmail,
|
||||
resourceName,
|
||||
numberOfSecrets: payload.numberOfSecrets,
|
||||
isDiffScan: payload.isDiffScan,
|
||||
url: encodeURI(
|
||||
`${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/findings?search=scanId:${payload.scanId}`
|
||||
),
|
||||
timestamp
|
||||
}
|
||||
: {
|
||||
dataSourceName: dataSource.name,
|
||||
resourceName,
|
||||
projectName: project.name,
|
||||
timestamp,
|
||||
errorMessage: payload.errorMessage,
|
||||
url: encodeURI(
|
||||
`${appCfg.SITE_URL}/projects/secret-scanning/${projectId}/data-sources/${dataSource.type}/${dataSource.id}`
|
||||
)
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
error,
|
||||
`secretScanningV2Queue: Failed to Send Status Notification [dataSourceId=${dataSource.id}] [resourceName=${resourceName}] [status=${payload.status}]`
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 2,
|
||||
pollingIntervalSeconds: 1
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
return {
|
||||
queueDataSourceFullScan,
|
||||
queueResourceDiffScan
|
||||
queueResourceDiffScan,
|
||||
init
|
||||
};
|
||||
};
|
||||
|
||||
@@ -72,7 +72,7 @@ const run = async () => {
|
||||
const keyStore = keyStoreFactory(envConfig, keyValueStoreDAL);
|
||||
const redis = buildRedisFromConfig(envConfig);
|
||||
|
||||
const server = await main({
|
||||
const { server, completeServerInitialization } = await main({
|
||||
db,
|
||||
auditLogDb,
|
||||
superAdminDAL,
|
||||
@@ -140,7 +140,18 @@ const run = async () => {
|
||||
}
|
||||
});
|
||||
|
||||
logger.info("Migrations complete. Marking server as READY...");
|
||||
logger.info("Migrations complete. Completing server initialization...");
|
||||
|
||||
try {
|
||||
await completeServerInitialization();
|
||||
} catch (error) {
|
||||
logger.error(error, "Failed to complete server initialization");
|
||||
await server.close();
|
||||
await queue.shutdown();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
logger.info("Server initialization complete. Marking server as READY...");
|
||||
|
||||
markServerReady();
|
||||
|
||||
|
||||
@@ -221,7 +221,7 @@ export const main = async ({
|
||||
};
|
||||
});
|
||||
|
||||
await server.register(registerRoutes, {
|
||||
const completeServerInitialization = await registerRoutes(server, {
|
||||
smtp,
|
||||
queue,
|
||||
db,
|
||||
@@ -240,7 +240,7 @@ export const main = async ({
|
||||
|
||||
await server.ready();
|
||||
server.swagger();
|
||||
return server;
|
||||
return { server, completeServerInitialization };
|
||||
} catch (err) {
|
||||
server.log.error(err);
|
||||
await queue.shutdown();
|
||||
|
||||
@@ -2206,7 +2206,7 @@ export const registerRoutes = async (
|
||||
internalCaFns
|
||||
});
|
||||
|
||||
await secretRotationV2QueueServiceFactory({
|
||||
const secretRotationV2Queue = await secretRotationV2QueueServiceFactory({
|
||||
secretRotationV2Service,
|
||||
secretRotationV2DAL,
|
||||
queueService,
|
||||
@@ -2300,50 +2300,87 @@ export const registerRoutes = async (
|
||||
// setup the communication with license key server
|
||||
await licenseService.init();
|
||||
|
||||
// If FIPS is enabled, we check to ensure that the users license includes FIPS mode.
|
||||
crypto.verifyFipsLicense(licenseService);
|
||||
const completeServerInitialization = async () => {
|
||||
await superAdminService.initServerCfg();
|
||||
|
||||
await superAdminService.initServerCfg();
|
||||
// If FIPS is enabled, we check to ensure that the users license includes FIPS mode.
|
||||
crypto.verifyFipsLicense(licenseService);
|
||||
|
||||
// Start HSM service if it's configured/enabled.
|
||||
await hsmService.startService();
|
||||
// Start HSM service if it's configured/enabled.
|
||||
await hsmService.startService();
|
||||
|
||||
const hsmStatus = await isHsmActiveAndEnabled({
|
||||
hsmService,
|
||||
kmsRootConfigDAL,
|
||||
licenseService
|
||||
});
|
||||
const hsmStatus = await isHsmActiveAndEnabled({
|
||||
hsmService,
|
||||
kmsRootConfigDAL,
|
||||
licenseService
|
||||
});
|
||||
|
||||
// if the encryption strategy is software - user needs to provide an encryption key
|
||||
// if the encryption strategy is null AND the hsm is not configured - user needs to provide an encryption key
|
||||
const needsEncryptionKey =
|
||||
hsmStatus.rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.Software ||
|
||||
(hsmStatus.rootKmsConfigEncryptionStrategy === null && !hsmStatus.isHsmConfigured);
|
||||
// if the encryption strategy is software - user needs to provide an encryption key
|
||||
// if the encryption strategy is null AND the hsm is not configured - user needs to provide an encryption key
|
||||
const needsEncryptionKey =
|
||||
hsmStatus.rootKmsConfigEncryptionStrategy === RootKeyEncryptionStrategy.Software ||
|
||||
(hsmStatus.rootKmsConfigEncryptionStrategy === null && !hsmStatus.isHsmConfigured);
|
||||
|
||||
if (needsEncryptionKey) {
|
||||
if (!envConfig.ROOT_ENCRYPTION_KEY && !envConfig.ENCRYPTION_KEY) {
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Root KMS encryption strategy is set to software. Please set the ENCRYPTION_KEY environment variable and restart your deployment.\nYou can enable HSM encryption in the Server Console."
|
||||
});
|
||||
if (needsEncryptionKey) {
|
||||
if (!envConfig.ROOT_ENCRYPTION_KEY && !envConfig.ENCRYPTION_KEY) {
|
||||
throw new BadRequestError({
|
||||
message:
|
||||
"Root KMS encryption strategy is set to software. Please set the ENCRYPTION_KEY environment variable and restart your deployment.\nYou can enable HSM encryption in the Server Console."
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await telemetryQueue.startTelemetryCheck();
|
||||
await telemetryQueue.startAggregatedEventsJob();
|
||||
await dailyResourceCleanUp.init();
|
||||
await healthAlert.init();
|
||||
await pkiSyncCleanup.init();
|
||||
await pamAccountRotation.init();
|
||||
await dailyReminderQueueService.startDailyRemindersJob();
|
||||
await dailyReminderQueueService.startSecretReminderMigrationJob();
|
||||
await dailyExpiringPkiItemAlert.startSendingAlerts();
|
||||
await pkiSubscriberQueue.startDailyAutoRenewalJob();
|
||||
await certificateV3Queue.init();
|
||||
await kmsService.startService(hsmStatus);
|
||||
await microsoftTeamsService.start();
|
||||
await dynamicSecretQueueService.init();
|
||||
await eventBusService.init();
|
||||
await telemetryQueue.startTelemetryCheck();
|
||||
await telemetryQueue.startAggregatedEventsJob();
|
||||
await dailyResourceCleanUp.init();
|
||||
await healthAlert.init();
|
||||
await pkiSyncCleanup.init();
|
||||
await pamAccountRotation.init();
|
||||
await dailyReminderQueueService.startDailyRemindersJob();
|
||||
await dailyReminderQueueService.startSecretReminderMigrationJob();
|
||||
await dailyExpiringPkiItemAlert.startSendingAlerts();
|
||||
await pkiSubscriberQueue.startDailyAutoRenewalJob();
|
||||
await certificateV3Queue.init();
|
||||
await kmsService.startService(hsmStatus);
|
||||
await microsoftTeamsService.start();
|
||||
await dynamicSecretQueueService.init();
|
||||
await secretScanningV2Queue.init();
|
||||
await secretRotationV2Queue.init();
|
||||
await notificationQueue.init();
|
||||
await eventBusService.init();
|
||||
|
||||
const cronJobs: CronJob[] = [];
|
||||
if (appCfg.isProductionMode) {
|
||||
const rateLimitSyncJob = await rateLimitService.initializeBackgroundSync();
|
||||
if (rateLimitSyncJob) {
|
||||
cronJobs.push(rateLimitSyncJob);
|
||||
}
|
||||
const licenseSyncJob = await licenseService.initializeBackgroundSync();
|
||||
if (licenseSyncJob) {
|
||||
cronJobs.push(licenseSyncJob);
|
||||
}
|
||||
|
||||
const microsoftTeamsSyncJob = await microsoftTeamsService.initializeBackgroundSync();
|
||||
if (microsoftTeamsSyncJob) {
|
||||
cronJobs.push(microsoftTeamsSyncJob);
|
||||
}
|
||||
|
||||
const adminIntegrationsSyncJob = await superAdminService.initializeAdminIntegrationConfigSync();
|
||||
if (adminIntegrationsSyncJob) {
|
||||
cronJobs.push(adminIntegrationsSyncJob);
|
||||
}
|
||||
}
|
||||
|
||||
const configSyncJob = await superAdminService.initializeEnvConfigSync();
|
||||
if (configSyncJob) {
|
||||
cronJobs.push(configSyncJob);
|
||||
}
|
||||
|
||||
const oauthConfigSyncJob = await initializeOauthConfigSync();
|
||||
if (oauthConfigSyncJob) {
|
||||
cronJobs.push(oauthConfigSyncJob);
|
||||
}
|
||||
};
|
||||
|
||||
// inject all services
|
||||
server.decorate<FastifyZodProvider["services"]>("services", {
|
||||
@@ -2473,38 +2510,6 @@ export const registerRoutes = async (
|
||||
convertor: convertorService
|
||||
});
|
||||
|
||||
const cronJobs: CronJob[] = [];
|
||||
if (appCfg.isProductionMode) {
|
||||
const rateLimitSyncJob = await rateLimitService.initializeBackgroundSync();
|
||||
if (rateLimitSyncJob) {
|
||||
cronJobs.push(rateLimitSyncJob);
|
||||
}
|
||||
const licenseSyncJob = await licenseService.initializeBackgroundSync();
|
||||
if (licenseSyncJob) {
|
||||
cronJobs.push(licenseSyncJob);
|
||||
}
|
||||
|
||||
const microsoftTeamsSyncJob = await microsoftTeamsService.initializeBackgroundSync();
|
||||
if (microsoftTeamsSyncJob) {
|
||||
cronJobs.push(microsoftTeamsSyncJob);
|
||||
}
|
||||
|
||||
const adminIntegrationsSyncJob = await superAdminService.initializeAdminIntegrationConfigSync();
|
||||
if (adminIntegrationsSyncJob) {
|
||||
cronJobs.push(adminIntegrationsSyncJob);
|
||||
}
|
||||
}
|
||||
|
||||
const configSyncJob = await superAdminService.initializeEnvConfigSync();
|
||||
if (configSyncJob) {
|
||||
cronJobs.push(configSyncJob);
|
||||
}
|
||||
|
||||
const oauthConfigSyncJob = await initializeOauthConfigSync();
|
||||
if (oauthConfigSyncJob) {
|
||||
cronJobs.push(oauthConfigSyncJob);
|
||||
}
|
||||
|
||||
server.decorate<FastifyZodProvider["store"]>("store", {
|
||||
user: userDAL,
|
||||
kmipClient: kmipClientDAL
|
||||
@@ -2593,9 +2598,10 @@ export const registerRoutes = async (
|
||||
await server.register(registerV4Routes, { prefix: "/api/v4" });
|
||||
|
||||
server.addHook("onClose", async () => {
|
||||
cronJobs.forEach((job) => job.stop());
|
||||
await telemetryService.flushAll();
|
||||
await eventBusService.close();
|
||||
sseService.close();
|
||||
});
|
||||
|
||||
return completeServerInitialization;
|
||||
};
|
||||
|
||||
@@ -10,6 +10,7 @@ type TNotificationQueueServiceFactoryDep = {
|
||||
|
||||
export type TNotificationQueueServiceFactory = {
|
||||
pushUserNotifications: (data: TCreateUserNotificationDTO[]) => Promise<void>;
|
||||
init: () => Promise<void>;
|
||||
};
|
||||
|
||||
export const notificationQueueServiceFactory = async ({
|
||||
@@ -20,20 +21,23 @@ export const notificationQueueServiceFactory = async ({
|
||||
await queueService.queuePg(QueueJobs.UserNotification, { notifications: data });
|
||||
};
|
||||
|
||||
await queueService.startPg(
|
||||
QueueJobs.UserNotification,
|
||||
async ([job]) => {
|
||||
const { notifications } = job.data as { notifications: TCreateUserNotificationDTO[] };
|
||||
await userNotificationDAL.batchInsert(notifications);
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 2,
|
||||
pollingIntervalSeconds: 1
|
||||
}
|
||||
);
|
||||
const init = async () => {
|
||||
await queueService.startPg(
|
||||
QueueJobs.UserNotification,
|
||||
async ([job]) => {
|
||||
const { notifications } = job.data as { notifications: TCreateUserNotificationDTO[] };
|
||||
await userNotificationDAL.batchInsert(notifications);
|
||||
},
|
||||
{
|
||||
batchSize: 1,
|
||||
workerCount: 2,
|
||||
pollingIntervalSeconds: 1
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
return {
|
||||
pushUserNotifications
|
||||
pushUserNotifications,
|
||||
init
|
||||
};
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user