misc: initial working setup for metrics observabilit

backend/src/lib/config/env.ts

@@ -119,7 +119,8 @@ const envSchema = z
       .transform((val) => val === "true")
       .optional(),
     INFISICAL_CLOUD: zodStrBool.default("false"),
-    MAINTENANCE_MODE: zodStrBool.default("false")
+    MAINTENANCE_MODE: zodStrBool.default("false"),
+    TELEMETRY_EXPORT_URL: zpStr(z.string().optional())
   })
   .transform((data) => ({
     ...data,

backend/src/lib/telemetry/instrumentation.ts

@@ -0,0 +1,37 @@
+import opentelemetry from "@opentelemetry/api";
+import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
+import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-proto";
+import { registerInstrumentations } from "@opentelemetry/instrumentation";
+import { Resource } from "@opentelemetry/resources";
+import { AggregationTemporality, MeterProvider, PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
+import { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
+
+export const initTelemetry = (exportURL: string) => {
+  const resource = Resource.default().merge(
+    new Resource({
+      [SEMRESATTRS_SERVICE_NAME]: "infisical-server",
+      [SEMRESATTRS_SERVICE_VERSION]: "0.1.0"
+    })
+  );
+
+  const metricExporter = new OTLPMetricExporter({
+    url: `${exportURL}/v1/metrics`,
+    temporalityPreference: AggregationTemporality.DELTA
+  });
+
+  const metricReader = new PeriodicExportingMetricReader({
+    exporter: metricExporter,
+    exportIntervalMillis: 30000
+  });
+
+  const myServiceMeterProvider = new MeterProvider({
+    resource,
+    readers: [metricReader]
+  });
+
+  opentelemetry.metrics.setGlobalMeterProvider(myServiceMeterProvider);
+
+  registerInstrumentations({
+    instrumentations: [getNodeAutoInstrumentations()]
+  });
+};

backend/src/main.ts

@@ -4,6 +4,7 @@ import { initDbConnection } from "./db";
 import { keyStoreFactory } from "./keystore/keystore";
 import { formatSmtpConfig, initEnvConfig } from "./lib/config/env";
 import { initLogger } from "./lib/logger";
+import { initTelemetry } from "./lib/telemetry/instrumentation";
 import { queueServiceFactory } from "./queue";
 import { main } from "./server/app";
 import { bootstrapCheck } from "./server/boot-strap-check";
@@ -13,6 +14,11 @@ dotenv.config();
 const run = async () => {
   const logger = await initLogger();
   const appCfg = initEnvConfig(logger);
+
+  if (appCfg.TELEMETRY_EXPORT_URL) {
+    initTelemetry(appCfg.TELEMETRY_EXPORT_URL);
+  }
+
   const db = initDbConnection({
     dbConnectionUri: appCfg.DB_CONNECTION_URI,
     dbRootCert: appCfg.DB_ROOT_CERT

backend/src/server/app.ts

@@ -20,6 +20,7 @@ import { TQueueServiceFactory } from "@app/queue";
 import { TSmtpService } from "@app/services/smtp/smtp-service";
 
 import { globalRateLimiterCfg } from "./config/rateLimiter";
+import { apiMetrics } from "./plugins/api-metrics";
 import { fastifyErrHandler } from "./plugins/error-handler";
 import { registerExternalNextjs } from "./plugins/external-nextjs";
 import { serializerCompiler, validatorCompiler, ZodTypeProvider } from "./plugins/fastify-zod";
@@ -62,7 +63,7 @@ export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => {
     });
     // pull ip based on various proxy headers
     await server.register(fastifyIp);
-
+    await server.register(apiMetrics);
     await server.register(fastifySwagger);
     await server.register(fastifyFormBody);
     await server.register(fastifyErrHandler);

backend/src/server/plugins/api-metrics.ts

@@ -0,0 +1,21 @@
+import opentelemetry from "@opentelemetry/api";
+import fp from "fastify-plugin";
+
+export const apiMetrics = fp(async (fastify) => {
+  const apiMeter = opentelemetry.metrics.getMeter("API");
+  const latencyHistogram = apiMeter.createHistogram("API latency", {
+    unit: "ms"
+  });
+
+  fastify.addHook("onResponse", async (request, reply) => {
+    const { method } = request;
+    const route = request.routerPath;
+    const { statusCode } = reply;
+
+    latencyHistogram.record(reply.elapsedTime, {
+      route,
+      method,
+      statusCode
+    });
+  });
+});