misc: finalized instrumentation setup

Makefile

@@ -10,6 +10,9 @@ up-dev:
 up-dev-ldap:
 	docker compose -f docker-compose.dev.yml --profile ldap up --build
 
+up-dev-metrics:
+	docker compose -f docker-compose.dev.yml --profile metrics up --build
+
 up-prod:
 	docker-compose -f docker-compose.prod.yml up --build
 
@@ -27,4 +30,3 @@ reviewable-api:
 	npm run type:check
 
 reviewable: reviewable-ui reviewable-api
-

backend/package.json

@@ -138,11 +138,11 @@
     "@octokit/plugin-retry": "^5.0.5",
     "@octokit/rest": "^20.0.2",
     "@octokit/webhooks-types": "^7.3.1",
-    "@opentelemetry/api": "^1.8.0",
+    "@opentelemetry/api": "^1.9.0",
-    "@opentelemetry/auto-instrumentations-node": "^0.46.1",
+    "@opentelemetry/auto-instrumentations-node": "^0.53.0",
-    "@opentelemetry/exporter-metrics-otlp-proto": "^0.51.1",
+    "@opentelemetry/exporter-metrics-otlp-proto": "^0.55.0",
-    "@opentelemetry/exporter-prometheus": "^0.51.1",
+    "@opentelemetry/exporter-prometheus": "^0.55.0",
-    "@opentelemetry/instrumentation": "^0.51.1",
+    "@opentelemetry/instrumentation": "^0.55.0",
     "@opentelemetry/resources": "^1.28.0",
     "@opentelemetry/sdk-metrics": "^1.28.0",
     "@opentelemetry/semantic-conventions": "^1.27.0",

backend/src/lib/config/env.ts

@@ -212,11 +212,11 @@ let envCfg: Readonly<z.infer<typeof envSchema>>;
 
 export const getConfig = () => envCfg;
 // cannot import singleton logger directly as it needs config to load various transport
-export const initEnvConfig = (logger: Logger) => {
+export const initEnvConfig = (logger?: Logger) => {
   const parsedEnv = envSchema.safeParse(process.env);
   if (!parsedEnv.success) {
-    logger.error("Invalid environment variables. Check the error below");
+    (logger ?? console).error("Invalid environment variables. Check the error below");
-    logger.error(parsedEnv.error.issues);
+    (logger ?? console).error(parsedEnv.error.issues);
     process.exit(-1);
   }
 

backend/src/lib/telemetry/instrumentation.ts

@@ -5,9 +5,14 @@ import { PrometheusExporter } from "@opentelemetry/exporter-prometheus";
 import { registerInstrumentations } from "@opentelemetry/instrumentation";
 import { Resource } from "@opentelemetry/resources";
 import { AggregationTemporality, MeterProvider, PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
-import { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
+import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
+import dotenv from "dotenv";
 
-export const initTelemetryInstrumentation = async ({
+import { initEnvConfig } from "../config/env";
+
+dotenv.config();
+
+const initTelemetryInstrumentation = ({
   exportType,
   otlpURL,
   otlpUser,
@@ -24,8 +29,8 @@ export const initTelemetryInstrumentation = async ({
 
   const resource = Resource.default().merge(
     new Resource({
-      [SEMRESATTRS_SERVICE_NAME]: "infisical-server",
+      [ATTR_SERVICE_NAME]: "infisical-server",
-      [SEMRESATTRS_SERVICE_VERSION]: "0.1.0"
+      [ATTR_SERVICE_VERSION]: "0.1.0"
     })
   );
 
@@ -67,3 +72,20 @@ export const initTelemetryInstrumentation = async ({
     instrumentations: [getNodeAutoInstrumentations()]
   });
 };
+
+const setupTelemetry = () => {
+  const appCfg = initEnvConfig();
+
+  if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
+    console.log("Initializing telemetry instrumentation");
+    initTelemetryInstrumentation({
+      otlpURL: appCfg.OTEL_EXPORT_OTLP_ENDPOINT,
+      otlpUser: appCfg.OTEL_COLLECTOR_BASIC_AUTH_USERNAME,
+      otlpPassword: appCfg.OTEL_COLLECTOR_BASIC_AUTH_PASSWORD,
+      otlpPushInterval: appCfg.OTEL_OTLP_PUSH_INTERVAL,
+      exportType: appCfg.OTEL_EXPORT_TYPE
+    });
+  }
+};
+
+void setupTelemetry();

backend/src/main.ts

@@ -1,3 +1,5 @@
+import "./lib/telemetry/instrumentation";
+
 import dotenv from "dotenv";
 import path from "path";
 
@@ -8,7 +10,6 @@ import { keyStoreFactory } from "./keystore/keystore";
 import { formatSmtpConfig, initEnvConfig, IS_PACKAGED } from "./lib/config/env";
 import { isMigrationMode } from "./lib/fn";
 import { initLogger } from "./lib/logger";
-import { initTelemetryInstrumentation } from "./lib/telemetry/instrumentation";
 import { queueServiceFactory } from "./queue";
 import { main } from "./server/app";
 import { bootstrapCheck } from "./server/boot-strap-check";
@@ -20,16 +21,6 @@ const run = async () => {
   const logger = await initLogger();
   const appCfg = initEnvConfig(logger);
 
-  if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
-    await initTelemetryInstrumentation({
-      otlpURL: appCfg.OTEL_EXPORT_OTLP_ENDPOINT,
-      otlpUser: appCfg.OTEL_COLLECTOR_BASIC_AUTH_USERNAME,
-      otlpPassword: appCfg.OTEL_COLLECTOR_BASIC_AUTH_PASSWORD,
-      otlpPushInterval: appCfg.OTEL_OTLP_PUSH_INTERVAL,
-      exportType: appCfg.OTEL_EXPORT_TYPE
-    });
-  }
-
   const db = initDbConnection({
     dbConnectionUri: appCfg.DB_CONNECTION_URI,
     dbRootCert: appCfg.DB_ROOT_CERT,

backend/src/server/plugins/api-metrics.ts

@@ -3,7 +3,7 @@ import fp from "fastify-plugin";
 
 export const apiMetrics = fp(async (fastify) => {
   const apiMeter = opentelemetry.metrics.getMeter("API");
-  const latencyHistogram = apiMeter.createHistogram("API latency", {
+  const latencyHistogram = apiMeter.createHistogram("API_latency", {
     unit: "ms"
   });
 

docker-compose.dev.yml

@@ -86,6 +86,7 @@ services:
       - .env
     ports:
       - 4000:4000
+      - 9464:9464 # for OTEL collection of Prometheus metrics
     environment:
       - NODE_ENV=development
       - DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable
@@ -95,6 +96,28 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
+  prometheus:
+    image: prom/prometheus
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    ports:
+      - "9090:9090"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+    profiles: [metrics]
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana
+    restart: unless-stopped
+    environment:
+      - GF_LOG_LEVEL=debug
+    ports:
+      - "3005:3000"
+    volumes:
+      - "grafana_storage:/var/lib/grafana"
+    profiles: [metrics]
+
   frontend:
     container_name: infisical-dev-frontend
     restart: unless-stopped
@@ -166,3 +189,4 @@ volumes:
     driver: local
   ldap_data:
   ldap_config:
+  grafana_storage:

prometheus.yml

@@ -1,8 +1,5 @@
 scrape_configs:
-  - job_name: "otel-collector"
+  - job_name: "metric-collector"
     scrape_interval: 30s
     static_configs:
-      - targets: ["otel-collector:8889"]
+      - targets: ["backend:9464"]
-    basic_auth:
-      username: infisical
-      password: infisical