Add offline reports

This commit is contained in:
Carlos Monastyrski
2025-09-01 18:50:54 -03:00
parent 01fba20872
commit 4063cf5294
12 changed files with 571 additions and 5 deletions
+2
View File
@@ -115,6 +115,7 @@ import { TUserServiceFactory } from "@app/services/user/user-service";
import { TUserEngagementServiceFactory } from "@app/services/user-engagement/user-engagement-service";
import { TWebhookServiceFactory } from "@app/services/webhook/webhook-service";
import { TWorkflowIntegrationServiceFactory } from "@app/services/workflow-integration/workflow-integration-service";
import { TOfflineUsageReportServiceFactory } from "@app/services/offline-usage-report/offline-usage-report-service";
declare module "@fastify/request-context" {
interface RequestContextData {
@@ -303,6 +304,7 @@ declare module "fastify" {
bus: TEventBusService;
sse: TServerSentEventsService;
identityAuthTemplate: TIdentityAuthTemplateServiceFactory;
offlineUsageReport: TOfflineUsageReportServiceFactory;
};
// this is exclusive use for middlewares in which we need to inject data
// everywhere else access using service layer
@@ -722,6 +722,16 @@ export const licenseServiceFactory = ({
await keyStore.deleteItem(FEATURE_CACHE_KEY(orgId));
};
const getCustomerId = () => {
if (!selfHostedLicense) return "unknown";
return selfHostedLicense?.customerId;
};
const getLicenseId = () => {
if (!selfHostedLicense) return "unknown";
return selfHostedLicense?.licenseId;
};
return {
generateOrgCustomerId,
removeOrgCustomer,
@@ -736,6 +746,8 @@ export const licenseServiceFactory = ({
return onPremFeatures;
},
getPlan,
getCustomerId,
getLicenseId,
invalidateGetPlan,
updateSubscriptionOrgMemberCount,
refreshPlan,
+9
View File
@@ -291,6 +291,8 @@ import { TSmtpService } from "@app/services/smtp/smtp-service";
import { invalidateCacheQueueFactory } from "@app/services/super-admin/invalidate-cache-queue";
import { TSuperAdminDALFactory } from "@app/services/super-admin/super-admin-dal";
import { getServerCfg, superAdminServiceFactory } from "@app/services/super-admin/super-admin-service";
import { offlineUsageReportDALFactory } from "@app/services/offline-usage-report/offline-usage-report-dal";
import { offlineUsageReportServiceFactory } from "@app/services/offline-usage-report/offline-usage-report-service";
import { telemetryDALFactory } from "@app/services/telemetry/telemetry-dal";
import { telemetryQueueServiceFactory } from "@app/services/telemetry/telemetry-queue";
import { telemetryServiceFactory } from "@app/services/telemetry/telemetry-service";
@@ -385,6 +387,7 @@ export const registerRoutes = async (
const reminderRecipientDAL = reminderRecipientDALFactory(db);
const integrationDAL = integrationDALFactory(db);
const offlineUsageReportDAL = offlineUsageReportDALFactory(db);
const integrationAuthDAL = integrationAuthDALFactory(db);
const webhookDAL = webhookDALFactory(db);
const serviceTokenDAL = serviceTokenDALFactory(db);
@@ -844,6 +847,11 @@ export const registerRoutes = async (
invalidateCacheQueue
});
const offlineUsageReportService = offlineUsageReportServiceFactory({
offlineUsageReportDAL,
licenseService
});
const orgAdminService = orgAdminServiceFactory({
smtpService,
projectDAL,
@@ -1999,6 +2007,7 @@ export const registerRoutes = async (
apiKey: apiKeyService,
authToken: tokenService,
superAdmin: superAdminService,
offlineUsageReport: offlineUsageReportService,
project: projectService,
projectMembership: projectMembershipService,
projectKey: projectKeyService,
+43 -2
View File
@@ -53,7 +53,8 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => {
defaultAuthOrgAuthMethod: z.string().nullish(),
isSecretScanningDisabled: z.boolean(),
kubernetesAutoFetchServiceAccountToken: z.boolean(),
paramsFolderSecretDetectionEnabled: z.boolean()
paramsFolderSecretDetectionEnabled: z.boolean(),
isOfflineUsageReportsEnabled: z.boolean()
})
})
}
@@ -69,7 +70,8 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => {
isMigrationModeOn: serverEnvs.MAINTENANCE_MODE,
isSecretScanningDisabled: serverEnvs.DISABLE_SECRET_SCANNING,
kubernetesAutoFetchServiceAccountToken: serverEnvs.KUBERNETES_AUTO_FETCH_SERVICE_ACCOUNT_TOKEN,
paramsFolderSecretDetectionEnabled: serverEnvs.PARAMS_FOLDER_SECRET_DETECTION_ENABLED
paramsFolderSecretDetectionEnabled: serverEnvs.PARAMS_FOLDER_SECRET_DETECTION_ENABLED,
isOfflineUsageReportsEnabled: !!serverEnvs.LICENSE_KEY_OFFLINE
}
};
}
@@ -838,4 +840,43 @@ export const registerAdminRouter = async (server: FastifyZodProvider) => {
};
}
});
// Usage Report Routes
server.route({
method: "POST",
url: "/usage-report/generate",
config: {
rateLimit: writeLimit
},
schema: {
response: {
200: z.object({
csvContent: z.string(),
signature: z.string(),
filename: z.string()
})
}
},
onRequest: (req, res, done) => {
verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN])(req, res, () => {
verifySuperAdmin(req, res, done);
});
},
handler: async () => {
const cfg = getConfig();
if (!cfg.LICENSE_KEY_OFFLINE) {
throw new BadRequestError({
message: "Offline usage reports are not enabled. LICENSE_KEY_OFFLINE must be configured."
});
}
const result = await server.services.offlineUsageReport.generateUsageReportCSV();
return {
csvContent: result.csvContent,
signature: result.signature,
filename: result.filename
};
}
});
};
@@ -0,0 +1,202 @@
import { TDbClient } from "@app/db";
import { TableName } from "@app/db/schemas";
export type TOfflineUsageReportDALFactory = ReturnType<typeof offlineUsageReportDALFactory>;
export const offlineUsageReportDALFactory = (db: TDbClient) => {
const getUserMetrics = async () => {
// Get total users and admin users
const userMetrics = (await db
.from(TableName.Users)
.select(
db.raw(
`
COUNT(*) as total_users,
COUNT(CASE WHEN "superAdmin" = true THEN 1 END) as admin_users
`
)
)
.where({ isGhost: false })
.first()) as { total_users: string; admin_users: string } | undefined;
// Get users by auth method
const authMethodStats = (await db
.from(TableName.Users)
.select(
db.raw(`
unnest("authMethods") as auth_method,
COUNT(*) as count
`)
)
.where({ isGhost: false })
.whereNotNull("authMethods")
.groupBy(db.raw('unnest("authMethods")'))) as Array<{ auth_method: string; count: string }>;
const usersByAuthMethod = authMethodStats.reduce(
(acc: Record<string, number>, row: { auth_method: string; count: string }) => {
acc[row.auth_method] = parseInt(row.count, 10);
return acc;
},
{} as Record<string, number>
);
return {
totalUsers: parseInt(userMetrics?.total_users || "0", 10),
adminUsers: parseInt(userMetrics?.admin_users || "0", 10),
usersByAuthMethod
};
};
const getMachineIdentityMetrics = async () => {
// Get total machine identities
const identityMetrics = (await db
.from(TableName.Identity)
.select(
db.raw(
`
COUNT(*) as total_identities
`
)
)
.first()) as { total_identities: string } | undefined;
// Get identities by auth method
const authMethodStats = (await db
.from(TableName.Identity)
.select("authMethod")
.count("* as count")
.whereNotNull("authMethod")
.groupBy("authMethod")) as Array<{ authMethod: string; count: string }>;
const machineIdentitiesByAuthMethod = authMethodStats.reduce(
(acc: Record<string, number>, row: { authMethod: string; count: string }) => {
acc[row.authMethod] = parseInt(row.count, 10);
return acc;
},
{} as Record<string, number>
);
return {
totalMachineIdentities: parseInt(identityMetrics?.total_identities || "0", 10),
machineIdentitiesByAuthMethod
};
};
const getProjectMetrics = async () => {
// Get total projects and projects by type
const projectMetrics = (await db
.from(TableName.Project)
.select("type")
.count("* as count")
.groupBy("type")) as Array<{ type: string; count: string }>;
const totalProjects = projectMetrics.reduce(
(sum, row: { type: string; count: string }) => sum + parseInt(row.count, 10),
0
);
const projectsByType = projectMetrics.reduce(
(acc: Record<string, number>, row: { type: string; count: string }) => {
acc[row.type] = parseInt(row.count, 10);
return acc;
},
{} as Record<string, number>
);
// Calculate average secrets per project
const secretsPerProject = (await db
.from(TableName.SecretV2)
.select("folderId")
.count("* as count")
.groupBy("folderId")) as Array<{ folderId: string; count: string }>;
const averageSecretsPerProject =
totalProjects > 0
? secretsPerProject.reduce(
(sum, row: { folderId: string; count: string }) => sum + parseInt(row.count, 10),
0
) / totalProjects
: 0;
return {
totalProjects,
projectsByType,
averageSecretsPerProject: Math.round(averageSecretsPerProject * 100) / 100
};
};
const getSecretMetrics = async () => {
// Get total secrets
const totalSecretsResult = (await db.from(TableName.SecretV2).count("* as count").first()) as
| { count: string }
| undefined;
const totalSecrets = parseInt(totalSecretsResult?.count || "0", 10);
// Get secrets by project
const secretsByProject = (await db
.from(`${TableName.SecretV2} as s`)
.select("p.id as projectId", "p.name as projectName")
.count("s.id as secretCount")
.leftJoin(`${TableName.SecretFolder} as sf`, "s.folderId", "sf.id")
.leftJoin(`${TableName.Environment} as e`, "sf.envId", "e.id")
.leftJoin(`${TableName.Project} as p`, "e.projectId", "p.id")
.groupBy("p.id", "p.name")
.whereNotNull("p.id")) as Array<{ projectId: string; projectName: string; secretCount: string }>;
return {
totalSecrets,
secretsByProject: secretsByProject.map(
(row: { projectId: string; projectName: string; secretCount: string }) => ({
projectId: row.projectId,
projectName: row.projectName,
secretCount: parseInt(row.secretCount, 10)
})
)
};
};
const getSecretSyncMetrics = async () => {
const totalSecretSyncsResult = (await db.from(TableName.SecretSync).count("* as count").first()) as
| { count: string }
| undefined;
return {
totalSecretSyncs: parseInt(totalSecretSyncsResult?.count || "0", 10)
};
};
const getDynamicSecretMetrics = async () => {
const totalDynamicSecretsResult = (await db.from(TableName.DynamicSecret).count("* as count").first()) as
| { count: string }
| undefined;
return {
totalDynamicSecrets: parseInt(totalDynamicSecretsResult?.count || "0", 10)
};
};
const getSecretRotationMetrics = async () => {
// Check both v1 and v2 secret rotation tables
const [v1RotationsResult, v2RotationsResult] = await Promise.all([
db.from(TableName.SecretRotation).count("* as count").first() as Promise<{ count: string } | undefined>,
db.from(TableName.SecretRotationV2).count("* as count").first() as Promise<{ count: string } | undefined>
]);
const totalV1Rotations = parseInt(v1RotationsResult?.count || "0", 10);
const totalV2Rotations = parseInt(v2RotationsResult?.count || "0", 10);
return {
totalSecretRotations: totalV1Rotations + totalV2Rotations
};
};
return {
getUserMetrics,
getMachineIdentityMetrics,
getProjectMetrics,
getSecretMetrics,
getSecretSyncMetrics,
getDynamicSecretMetrics,
getSecretRotationMetrics
};
};
@@ -0,0 +1,138 @@
import crypto from "crypto";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import { getConfig } from "@app/lib/config/env";
import { BadRequestError } from "@app/lib/errors";
import { TOfflineUsageReportDALFactory } from "./offline-usage-report-dal";
type TOfflineUsageReportServiceFactoryDep = {
offlineUsageReportDAL: TOfflineUsageReportDALFactory;
licenseService: Pick<TLicenseServiceFactory, "getCustomerId" | "getLicenseId">;
};
export type TOfflineUsageReportServiceFactory = ReturnType<typeof offlineUsageReportServiceFactory>;
export const offlineUsageReportServiceFactory = ({
offlineUsageReportDAL,
licenseService
}: TOfflineUsageReportServiceFactoryDep) => {
const signReportContent = (content: string, licenseId: string): string => {
const contentHash = crypto.createHash("sha256").update(content).digest("hex");
const hmac = crypto.createHmac("sha256", licenseId);
hmac.update(contentHash);
return hmac.digest("hex");
};
const verifyReportContent = (content: string, signature: string, licenseId: string): boolean => {
const expectedSignature = signReportContent(content, licenseId);
return signature === expectedSignature;
};
const generateUsageReportCSV = async () => {
const cfg = getConfig();
if (!cfg.LICENSE_KEY_OFFLINE) {
throw new BadRequestError({
message: "Offline usage reports are not enabled. LICENSE_KEY_OFFLINE must be configured."
});
}
const customerId = licenseService.getCustomerId() as string;
const licenseId = licenseService.getLicenseId();
const [
userMetrics,
machineIdentityMetrics,
projectMetrics,
secretMetrics,
secretSyncMetrics,
dynamicSecretMetrics,
secretRotationMetrics
] = await Promise.all([
offlineUsageReportDAL.getUserMetrics(),
offlineUsageReportDAL.getMachineIdentityMetrics(),
offlineUsageReportDAL.getProjectMetrics(),
offlineUsageReportDAL.getSecretMetrics(),
offlineUsageReportDAL.getSecretSyncMetrics(),
offlineUsageReportDAL.getDynamicSecretMetrics(),
offlineUsageReportDAL.getSecretRotationMetrics()
]);
const csvLines: string[] = [];
const headers = [
"Total Users",
"Admin Users",
"Total Identities",
"Total Projects",
"Total Secrets",
"Total Secret Syncs",
"Total Dynamic Secrets",
"Total Secret Rotations",
"Avg Secrets Per Project"
];
const allUserAuthMethods = Object.keys(userMetrics.usersByAuthMethod);
allUserAuthMethods.forEach((method) => {
headers.push(`Users Auth ${method}`);
});
const allIdentityAuthMethods = Object.keys(machineIdentityMetrics.machineIdentitiesByAuthMethod);
allIdentityAuthMethods.forEach((method) => {
headers.push(`Identities Auth ${method}`);
});
const allProjectTypes = Object.keys(projectMetrics.projectsByType);
allProjectTypes.forEach((type) => {
headers.push(`Projects ${type}`);
});
headers.push("Signature");
csvLines.push(headers.slice(0, -1).join(","));
const dataRow: (string | number)[] = [
userMetrics.totalUsers,
userMetrics.adminUsers,
machineIdentityMetrics.totalMachineIdentities,
projectMetrics.totalProjects,
secretMetrics.totalSecrets,
secretSyncMetrics.totalSecretSyncs,
dynamicSecretMetrics.totalDynamicSecrets,
secretRotationMetrics.totalSecretRotations,
projectMetrics.averageSecretsPerProject
];
allUserAuthMethods.forEach((method) => {
dataRow.push(userMetrics.usersByAuthMethod[method] || 0);
});
allIdentityAuthMethods.forEach((method) => {
dataRow.push(machineIdentityMetrics.machineIdentitiesByAuthMethod[method] || 0);
});
allProjectTypes.forEach((type) => {
dataRow.push(projectMetrics.projectsByType[type] || 0);
});
const tempCsvLines = [headers.slice(0, -1).join(","), dataRow.join(",")];
const contentWithoutSignature = tempCsvLines.join("\n");
const signature = signReportContent(contentWithoutSignature, licenseId);
dataRow.push(signature);
const finalCsvLines = [headers.join(","), dataRow.join(",")];
const csvContent = finalCsvLines.join("\n");
return {
csvContent,
signature,
filename: `infisical-usage-report-${customerId}-${new Date().toISOString().split("T")[0]}.csv`
};
};
return {
generateUsageReportCSV,
verifyReportSignature: (csvContent: string, signature: string, licenseId: string) =>
verifyReportContent(csvContent, signature, licenseId)
};
};
@@ -0,0 +1,42 @@
export interface TUsageMetrics {
// User metrics
totalUsers: number;
usersByAuthMethod: Record<string, number>;
adminUsers: number;
// Machine identity metrics
totalMachineIdentities: number;
machineIdentitiesByAuthMethod: Record<string, number>;
// Project metrics
totalProjects: number;
projectsByType: Record<string, number>;
averageSecretsPerProject: number;
// Secret metrics
totalSecrets: number;
totalSecretSyncs: number;
totalDynamicSecrets: number;
totalSecretRotations: number;
}
export interface TUsageReportMetadata {
generatedAt: string;
instanceId: string;
reportVersion: string;
}
export interface TUsageReport {
metadata: TUsageReportMetadata;
metrics: TUsageMetrics;
signature?: string;
}
export interface TGenerateUsageReportDTO {
includeSignature?: boolean;
}
export interface TVerifyUsageReportDTO {
reportData: string;
signature: string;
}