feat: add prometheus report support

pnpm-lock.yaml

@@ -540,6 +540,9 @@ importers:
       ping:
         specifier: ^0.4.4
         version: 0.4.4
+      prom-client:
+        specifier: ^15.1.3
+        version: 15.1.3
       puppeteer:
         specifier: 23.4.1
         version: 23.4.1(typescript@5.5.4)
@@ -4911,6 +4914,9 @@ packages:
   bing-translate-api@4.0.2:
     resolution: {integrity: sha512-JJ8XUehnxzOhHU91oy86xEtp8OOMjVEjCZJX042fKxoO19NNvxJ5omeCcxQNFoPbDqVpBJwqiGVquL0oPdQm1Q==}
 
+  bintrees@1.0.2:
+    resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==}
+
   bl@4.1.0:
     resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}
 
@@ -9926,6 +9932,10 @@ packages:
     resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==}
     engines: {node: '>=0.4.0'}
 
+  prom-client@15.1.3:
+    resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==}
+    engines: {node: ^16 || ^18 || >=20}
+
   promise-coalesce@1.1.2:
     resolution: {integrity: sha512-zLaJ9b8hnC564fnJH6NFSOGZYYdzrAJn2JUUIwzoQb32fG2QAakpDNM+CZo1km6keXkRXRM+hml1BFAPVnPkxg==}
     engines: {node: '>=16'}
@@ -11582,6 +11592,9 @@ packages:
   tcp-ping@0.1.1:
     resolution: {integrity: sha512-7Ed10Ds0hYnF+O1lfiZ2iSZ1bCAj+96Madctebmq7Y1ALPWlBY4YI8C6pCL+UTlshFY5YogixKLpgDP/4BlHrw==}
 
+  tdigest@0.1.2:
+    resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==}
+
   teex@1.0.1:
     resolution: {integrity: sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==}
 
@@ -14173,7 +14186,7 @@ snapshots:
       '@babel/helper-split-export-declaration': 7.22.6
       '@babel/parser': 7.24.0
       '@babel/types': 7.24.0
-      debug: 4.3.6
+      debug: 4.3.7
       globals: 11.12.0
     transitivePeerDependencies:
       - supports-color
@@ -14188,7 +14201,7 @@ snapshots:
       '@babel/helper-split-export-declaration': 7.22.6
       '@babel/parser': 7.24.0
       '@babel/types': 7.24.0
-      debug: 4.3.6
+      debug: 4.3.7
       globals: 11.12.0
     transitivePeerDependencies:
       - supports-color
@@ -18273,13 +18286,13 @@ snapshots:
 
   agent-base@6.0.2:
     dependencies:
-      debug: 4.3.6
+      debug: 4.3.7
     transitivePeerDependencies:
       - supports-color
 
   agent-base@7.1.0:
     dependencies:
-      debug: 4.3.6
+      debug: 4.3.7
     transitivePeerDependencies:
       - supports-color
 
@@ -18790,6 +18803,8 @@ snapshots:
     dependencies:
       got: 11.8.6
 
+  bintrees@1.0.2: {}
+
   bl@4.1.0:
     dependencies:
       buffer: 5.7.1
@@ -20714,7 +20729,7 @@ snapshots:
 
   extract-zip@2.0.1:
     dependencies:
-      debug: 4.3.6
+      debug: 4.3.7
       get-stream: 5.2.0
       yauzl: 2.10.0
     optionalDependencies:
@@ -21065,7 +21080,7 @@ snapshots:
     dependencies:
       basic-ftp: 5.0.3
       data-uri-to-buffer: 6.0.1
-      debug: 4.3.6
+      debug: 4.3.7
       fs-extra: 8.1.0
     transitivePeerDependencies:
       - supports-color
@@ -21659,14 +21674,14 @@ snapshots:
   http-proxy-agent@7.0.0:
     dependencies:
       agent-base: 7.1.0
-      debug: 4.3.6
+      debug: 4.3.7
     transitivePeerDependencies:
       - supports-color
 
   http-proxy-agent@7.0.2:
     dependencies:
       agent-base: 7.1.0
-      debug: 4.3.6
+      debug: 4.3.7
     transitivePeerDependencies:
       - supports-color
 
@@ -21707,7 +21722,7 @@ snapshots:
   https-proxy-agent@5.0.1:
     dependencies:
       agent-base: 6.0.2
-      debug: 4.3.6
+      debug: 4.3.7
     transitivePeerDependencies:
       - supports-color
 
@@ -21721,21 +21736,21 @@ snapshots:
   https-proxy-agent@7.0.0:
     dependencies:
       agent-base: 7.1.0
-      debug: 4.3.6
+      debug: 4.3.7
     transitivePeerDependencies:
       - supports-color
 
   https-proxy-agent@7.0.2:
     dependencies:
       agent-base: 7.1.0
-      debug: 4.3.6
+      debug: 4.3.7
     transitivePeerDependencies:
       - supports-color
 
   https-proxy-agent@7.0.5:
     dependencies:
       agent-base: 7.1.0
-      debug: 4.3.6
+      debug: 4.3.7
     transitivePeerDependencies:
       - supports-color
 
@@ -23475,7 +23490,7 @@ snapshots:
   micromark@3.2.0:
     dependencies:
       '@types/debug': 4.1.12
-      debug: 4.3.6
+      debug: 4.3.7
       decode-named-character-reference: 1.0.2
       micromark-core-commonmark: 1.1.0
       micromark-factory-space: 1.1.0
@@ -24200,7 +24215,7 @@ snapshots:
     dependencies:
       '@tootallnate/quickjs-emscripten': 0.23.0
       agent-base: 7.1.0
-      debug: 4.3.6
+      debug: 4.3.7
       get-uri: 6.0.2
       http-proxy-agent: 7.0.2
       https-proxy-agent: 7.0.5
@@ -24908,6 +24923,11 @@ snapshots:
 
   progress@2.0.3: {}
 
+  prom-client@15.1.3:
+    dependencies:
+      '@opentelemetry/api': 1.4.1
+      tdigest: 0.1.2
+
   promise-coalesce@1.1.2: {}
 
   promise-retry@2.0.1:
@@ -24966,7 +24986,7 @@ snapshots:
   proxy-agent@6.4.0:
     dependencies:
       agent-base: 7.1.0
-      debug: 4.3.6
+      debug: 4.3.7
       http-proxy-agent: 7.0.2
       https-proxy-agent: 7.0.5
       lru-cache: 7.18.3
@@ -26954,7 +26974,7 @@ snapshots:
   socks-proxy-agent@8.0.2:
     dependencies:
       agent-base: 7.1.0
-      debug: 4.3.6
+      debug: 4.3.7
       socks: 2.7.1
     transitivePeerDependencies:
       - supports-color
@@ -27459,6 +27479,10 @@ snapshots:
 
   tcp-ping@0.1.1: {}
 
+  tdigest@0.1.2:
+    dependencies:
+      bintrees: 1.0.2
+
   teex@1.0.1:
     dependencies:
       streamx: 2.20.1

src/server/app.ts

@@ -22,10 +22,12 @@ import path from 'path';
 import { monitorPageManager } from './model/monitor/page/manager.js';
 import { ExpressAuth } from '@auth/express';
 import { authConfig } from './model/auth.js';
+import { prometheusApiVersion } from './middleware/prometheus/index.js';
 
 const app = express();
 
 app.set('trust proxy', true);
+app.use(prometheusApiVersion());
 app.use(compression());
 app.use(
   express.json({

src/server/main.ts

@@ -9,6 +9,7 @@ import { initCronjob } from './cronjob/index.js';
 import { logger } from './utils/logger.js';
 import { app } from './app.js';
 import { runMQWorker } from './mq/worker.js';
+import { initCounter } from './utils/prometheus/index.js';
 
 const port = env.port;
 
@@ -20,6 +21,8 @@ initSocketio(httpServer);
 
 initCronjob();
 
+initCounter();
+
 runMQWorker();
 
 monitorManager.startAll();

src/server/middleware/prometheus/README.md

@@ -0,0 +1 @@
+This folder is fork from https://github.com/PayU/prometheus-api-metrics

src/server/middleware/prometheus/index.ts

@@ -0,0 +1,132 @@
+import Prometheus from 'prom-client';
+import * as utils from './utils.js';
+import { SetupOptions, ApiMetricsOpts } from './types.js';
+import { ExpressMiddleware } from './middleware.js';
+
+export function prometheusApiVersion(options: ApiMetricsOpts = {}) {
+  const appVersion = '1.0.0';
+  const projectName = 'tianji';
+
+  const {
+    metricsPath,
+    defaultMetricsInterval = 10000,
+    durationBuckets,
+    requestSizeBuckets,
+    responseSizeBuckets,
+    useUniqueHistogramName,
+    metricsPrefix,
+    excludeRoutes,
+    includeQueryParams,
+    additionalLabels = [],
+    extractAdditionalLabelValuesFn,
+  } = options;
+
+  const setupOptions: SetupOptions = {};
+
+  setupOptions.metricsRoute = utils.validateInput({
+    input: metricsPath,
+    isValidInputFn: utils.isString,
+    defaultValue: '/_prom/metrics',
+    errorMessage: 'metricsPath should be an string',
+  });
+
+  setupOptions.excludeRoutes = utils.validateInput({
+    input: excludeRoutes,
+    isValidInputFn: utils.isArray,
+    defaultValue: [],
+    errorMessage: 'excludeRoutes should be an array',
+  });
+
+  setupOptions.includeQueryParams = includeQueryParams;
+  setupOptions.defaultMetricsInterval = defaultMetricsInterval;
+
+  setupOptions.additionalLabels = utils.validateInput({
+    input: additionalLabels,
+    isValidInputFn: utils.isArray,
+    defaultValue: [],
+    errorMessage: 'additionalLabels should be an array',
+  });
+
+  setupOptions.extractAdditionalLabelValuesFn = utils.validateInput({
+    input: extractAdditionalLabelValuesFn,
+    isValidInputFn: utils.isFunction,
+    defaultValue: () => ({}),
+    errorMessage: 'extractAdditionalLabelValuesFn should be a function',
+  });
+
+  const metricNames = utils.getMetricNames(
+    {
+      http_request_duration_seconds: 'http_request_duration_seconds',
+      app_version: 'app_version',
+      http_request_size_bytes: 'http_request_size_bytes',
+      http_response_size_bytes: 'http_response_size_bytes',
+      defaultMetricsPrefix: '',
+    },
+    useUniqueHistogramName ?? false,
+    metricsPrefix ?? '',
+    projectName
+  );
+
+  Prometheus.collectDefaultMetrics({
+    eventLoopMonitoringPrecision: defaultMetricsInterval,
+    prefix: `${metricNames.defaultMetricsPrefix}`,
+  });
+
+  PrometheusRegisterAppVersion(appVersion, metricNames.app_version);
+
+  const metricLabels = ['method', 'route', 'code', ...additionalLabels].filter(
+    Boolean
+  );
+
+  // Buckets for response time from 1ms to 500ms
+  const defaultDurationSecondsBuckets = [
+    0.001, 0.005, 0.015, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5,
+  ];
+  // Buckets for request size from 5 bytes to 10000 bytes
+  const defaultSizeBytesBuckets = [
+    5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000,
+  ];
+
+  setupOptions.responseTimeHistogram =
+    Prometheus.register.getSingleMetric(
+      metricNames.http_request_duration_seconds
+    ) ||
+    new Prometheus.Histogram({
+      name: metricNames.http_request_duration_seconds,
+      help: 'Duration of HTTP requests in seconds',
+      labelNames: metricLabels,
+      buckets: durationBuckets || defaultDurationSecondsBuckets,
+    });
+
+  setupOptions.requestSizeHistogram =
+    Prometheus.register.getSingleMetric(metricNames.http_request_size_bytes) ||
+    new Prometheus.Histogram({
+      name: metricNames.http_request_size_bytes,
+      help: 'Size of HTTP requests in bytes',
+      labelNames: metricLabels,
+      buckets: requestSizeBuckets || defaultSizeBytesBuckets,
+    });
+
+  setupOptions.responseSizeHistogram =
+    Prometheus.register.getSingleMetric(metricNames.http_response_size_bytes) ||
+    new Prometheus.Histogram({
+      name: metricNames.http_response_size_bytes,
+      help: 'Size of HTTP response in bytes',
+      labelNames: metricLabels,
+      buckets: responseSizeBuckets || defaultSizeBytesBuckets,
+    });
+
+  const middleware = new ExpressMiddleware(setupOptions);
+  return middleware.middleware.bind(middleware);
+}
+
+function PrometheusRegisterAppVersion(appVersion: string, metricName: string) {
+  const version = new Prometheus.Gauge({
+    name: metricName,
+    help: 'The service version by package.json',
+    labelNames: ['version', 'major', 'minor', 'patch'],
+  });
+
+  const [major, minor, patch] = appVersion.split('.');
+  version.labels(appVersion, major, minor, patch).set(1);
+}

src/server/middleware/prometheus/middleware.ts

@@ -0,0 +1,135 @@
+import Prometheus from 'prom-client';
+import { SetupOptions } from './types.js';
+import { Request, Response, NextFunction } from 'express';
+import * as utils from './utils.js';
+
+export class ExpressMiddleware {
+  constructor(public setupOptions: SetupOptions) {}
+
+  _collectDefaultServerMetrics(timeout: number) {
+    const NUMBER_OF_CONNECTIONS_METRICS_NAME =
+      'expressjs_number_of_open_connections';
+    this.setupOptions.numberOfConnectionsGauge =
+      Prometheus.register.getSingleMetric(NUMBER_OF_CONNECTIONS_METRICS_NAME) ||
+      new Prometheus.Gauge({
+        name: NUMBER_OF_CONNECTIONS_METRICS_NAME,
+        help: 'Number of open connections to the Express.js server',
+      });
+    if (this.setupOptions.server) {
+      setInterval(this._getConnections.bind(this), timeout).unref();
+    }
+  }
+
+  _getConnections() {
+    if (this.setupOptions && this.setupOptions.server) {
+      this.setupOptions.server.getConnections((error: any, count: any) => {
+        if (error) {
+          // debug('Error while collection number of open connections', error);
+        } else {
+          this.setupOptions.numberOfConnectionsGauge.set(count);
+        }
+      });
+    }
+  }
+
+  _handleResponse(req: Request, res: Response) {
+    const responseLength = parseInt(res.get('Content-Length')!) || 0;
+
+    const route = this._getRoute(req);
+
+    if (
+      route &&
+      utils.shouldLogMetrics(this.setupOptions.excludeRoutes!, route)
+    ) {
+      const labels = {
+        method: req.method,
+        route,
+        code: res.statusCode,
+        ...this.setupOptions.extractAdditionalLabelValuesFn!(req, res),
+      };
+      this.setupOptions.requestSizeHistogram.observe(
+        labels,
+        (req as any).metrics.contentLength
+      );
+      (req as any).metrics.timer(labels);
+      this.setupOptions.responseSizeHistogram.observe(labels, responseLength);
+    }
+  }
+
+  _getRoute(req: Request) {
+    let route = req.baseUrl;
+    if (req.route) {
+      if (req.route.path !== '/') {
+        route = route ? route + req.route.path : req.route.path;
+      }
+
+      if (!route || route === '' || typeof route !== 'string') {
+        route = req.originalUrl.split('?')[0];
+      } else {
+        const splittedRoute = route.split('/');
+        const splittedUrl = req.originalUrl.split('?')[0].split('/');
+        const routeIndex = splittedUrl.length - splittedRoute.length + 1;
+
+        const baseUrl = splittedUrl.slice(0, routeIndex).join('/');
+        route = baseUrl + route;
+      }
+
+      if (
+        this.setupOptions.includeQueryParams === true &&
+        Object.keys(req.query).length > 0
+      ) {
+        route = `${route}?${Object.keys(req.query)
+          .sort()
+          .map((queryParam) => `${queryParam}=<?>`)
+          .join('&')}`;
+      }
+    }
+
+    // nest.js - build request url pattern if exists
+    if (typeof req.params === 'object') {
+      Object.keys(req.params).forEach((paramName) => {
+        route = route.replace(req.params[paramName], ':' + paramName);
+      });
+    }
+
+    // this condition will evaluate to true only in
+    // express framework and no route was found for the request. if we log this metrics
+    // we'll risk in a memory leak since the route is not a pattern but a hardcoded string.
+    if (!route || route === '') {
+      // if (!req.route && res && res.statusCode === 404) {
+      route = 'N/A';
+    }
+
+    return route;
+  }
+
+  async middleware(req: Request, res: Response, next: NextFunction) {
+    if (!this.setupOptions.server && req.socket) {
+      this.setupOptions.server = (req.socket as any).server;
+      this._collectDefaultServerMetrics(
+        this.setupOptions.defaultMetricsInterval as any
+      );
+    }
+
+    const routeUrl = req.originalUrl || req.url;
+
+    if (routeUrl === this.setupOptions.metricsRoute) {
+      res.set('Content-Type', Prometheus.register.contentType);
+      return res.end(await Prometheus.register.metrics());
+    }
+    if (routeUrl === `${this.setupOptions.metricsRoute}.json`) {
+      return res.json(await Prometheus.register.getMetricsAsJSON());
+    }
+
+    (req as any).metrics = {
+      timer: (this.setupOptions as any).responseTimeHistogram.startTimer(),
+      contentLength: parseInt(req.get('content-length')!) || 0,
+    } as any;
+
+    res.once('finish', () => {
+      this._handleResponse(req, res);
+    });
+
+    return next();
+  }
+}

src/server/middleware/prometheus/types.ts

@@ -0,0 +1,46 @@
+import { Request, RequestHandler, Response } from 'express';
+import Prometheus from 'prom-client';
+
+export class HttpMetricsCollector {
+  constructor(options?: CollectorOpts);
+  static init(options?: CollectorOpts): void;
+  static collect(res: Response | any): void;
+}
+
+export interface ApiMetricsOpts {
+  metricsPath?: string;
+  defaultMetricsInterval?: number;
+  durationBuckets?: number[];
+  requestSizeBuckets?: number[];
+  responseSizeBuckets?: number[];
+  useUniqueHistogramName?: boolean;
+  metricsPrefix?: string;
+  excludeRoutes?: string[];
+  includeQueryParams?: boolean;
+  additionalLabels?: string[];
+  extractAdditionalLabelValuesFn?: (
+    req: Request,
+    res: Response
+  ) => Record<string, unknown>;
+}
+
+export interface CollectorOpts {
+  durationBuckets?: number[];
+  countClientErrors?: boolean;
+  useUniqueHistogramName?: boolean;
+  prefix?: string;
+}
+
+export interface SetupOptions {
+  metricsRoute?: string;
+  excludeRoutes?: string[];
+  includeQueryParams?: boolean;
+  defaultMetricsInterval?: number;
+  additionalLabels?: string[];
+  extractAdditionalLabelValuesFn?: (
+    req: Request,
+    res: Response
+  ) => Record<string, unknown>;
+  responseTimeHistogram?: Prometheus.Metric<string> | undefined;
+  [other: string]: any;
+}

src/server/middleware/prometheus/utils.ts

@@ -0,0 +1,63 @@
+'use strict';
+
+type MetricNames = { [key: string]: string };
+
+const getMetricNames = (
+  metricNames: MetricNames,
+  useUniqueHistogramName: boolean,
+  metricsPrefix: string,
+  projectName: string
+): MetricNames => {
+  const prefix = useUniqueHistogramName === true ? projectName : metricsPrefix;
+
+  if (prefix) {
+    Object.keys(metricNames).forEach((key) => {
+      metricNames[key] = `${prefix}_${metricNames[key]}`;
+    });
+  }
+
+  return metricNames;
+};
+
+const isArray = (input: unknown): input is any[] => Array.isArray(input);
+
+const isFunction = (input: unknown): input is Function =>
+  typeof input === 'function';
+
+const isString = (input: unknown): input is string => typeof input === 'string';
+
+const shouldLogMetrics = (excludeRoutes: string[], route: string): boolean =>
+  excludeRoutes.every((path) => !route.includes(path));
+
+interface ValidateInputParams<T> {
+  input: T | undefined;
+  isValidInputFn: (input: T) => boolean;
+  defaultValue: T;
+  errorMessage: string;
+}
+
+const validateInput = <T>({
+  input,
+  isValidInputFn,
+  defaultValue,
+  errorMessage,
+}: ValidateInputParams<T>): T => {
+  if (typeof input !== 'undefined') {
+    if (isValidInputFn(input)) {
+      return input;
+    } else {
+      throw new Error(errorMessage);
+    }
+  }
+
+  return defaultValue;
+};
+
+export {
+  getMetricNames,
+  isArray,
+  isFunction,
+  isString,
+  shouldLogMetrics,
+  validateInput,
+};

src/server/model/monitor/manager.ts

@@ -3,6 +3,7 @@ import { prisma } from '../_client.js';
 import { MonitorRunner } from './runner.js';
 import { logger } from '../../utils/logger.js';
 import { MonitorWithNotification } from './types.js';
+import { promMonitorRunnerCounter } from '../../utils/prometheus/client.js';
 
 export type MonitorUpsertData = Pick<
   Monitor,
@@ -154,6 +155,8 @@ export class MonitorManager {
       monitor
     ));
 
+    promMonitorRunnerCounter.set(Object.keys(this.monitorRunner).length);
+
     return runner;
   }
 }

src/server/model/serverStatus.ts

@@ -1,4 +1,5 @@
 import { ServerStatusInfo } from '../../types/index.js';
+import { promServerCounter } from '../utils/prometheus/client.js';
 import { createSubscribeInitializer, subscribeEventBus } from '../ws/shared.js';
 import { isServerOnline } from '@tianji/shared';
 
@@ -42,6 +43,13 @@ export function recordServerStatus(info: ServerStatusInfo) {
     payload,
   };
 
+  promServerCounter.set(
+    {
+      workspaceId,
+    },
+    Object.keys(serverMap[workspaceId]).length
+  );
+
   subscribeEventBus.emit(
     'onServerStatusUpdate',
     workspaceId,

src/server/model/user.ts

@@ -7,6 +7,7 @@ import { Prisma } from '@prisma/client';
 import { AdapterUser } from '@auth/core/adapters';
 import { md5 } from '../utils/common.js';
 import { logger } from '../utils/logger.js';
+import { promUserCounter } from '../utils/prometheus/client.js';
 
 async function hashPassword(password: string) {
   return await bcryptjs.hash(password, 10);
@@ -88,6 +89,8 @@ export async function createAdminUser(username: string, password: string) {
     return user;
   });
 
+  promUserCounter.inc();
+
   return user;
 }
 
@@ -130,6 +133,8 @@ export async function createUser(username: string, password: string) {
     return user;
   });
 
+  promUserCounter.inc();
+
   return user;
 }
 

src/server/package.json

@@ -60,6 +60,7 @@
     "passport": "^0.7.0",
     "passport-jwt": "^4.0.1",
     "ping": "^0.4.4",
+    "prom-client": "^15.1.3",
     "puppeteer": "23.4.1",
     "request-ip": "^3.3.0",
     "socket.io": "^4.7.4",

src/server/trpc/routers/workspace.ts

@@ -27,6 +27,7 @@ import {
 import { WorkspacesOnUsersModelSchema } from '../../prisma/zod/workspacesonusers.js';
 import { monitorManager } from '../../model/monitor/index.js';
 import { get, merge } from 'lodash-es';
+import { promWorkspaceCounter } from '../../utils/prometheus/client.js';
 
 export const workspaceRouter = router({
   create: protectProedure
@@ -63,6 +64,8 @@ export const workspaceRouter = router({
           },
         });
 
+        promWorkspaceCounter.inc();
+
         return await p.user.update({
           data: {
             currentWorkspaceId: newWorkspace.id,

src/server/trpc/trpc.ts

@@ -8,6 +8,7 @@ import { OpenApiMeta } from 'trpc-openapi';
 import { getSession } from '@auth/express';
 import { authConfig } from '../model/auth.js';
 import { get } from 'lodash-es';
+import { promTrpcRequest } from '../utils/prometheus/client.js';
 
 export async function createContext({ req }: { req: Request }) {
   const authorization = req.headers['authorization'] ?? '';
@@ -23,7 +24,33 @@ export type OpenApiMetaInfo = NonNullable<OpenApiMeta['openapi']>;
 
 export const middleware = t.middleware;
 export const router = t.router;
-export const publicProcedure = t.procedure;
+
+const prom = middleware(async (opts) => {
+  const path = opts.path;
+  const type = opts.type;
+
+  const endRequest = promTrpcRequest.startTimer({
+    route: path,
+    type,
+  });
+
+  try {
+    const res = await opts.next();
+
+    endRequest({
+      status: 'success',
+    });
+    return res;
+  } catch (err) {
+    endRequest({
+      status: 'error',
+    });
+
+    throw err;
+  }
+});
+
+export const publicProcedure = t.procedure.use(prom);
 
 const isUser = middleware(async (opts) => {
   // auth with token
@@ -62,7 +89,7 @@ const isUser = middleware(async (opts) => {
   throw new TRPCError({ code: 'UNAUTHORIZED', message: 'No Token or Session' });
 });
 
-export const protectProedure = t.procedure.use(isUser);
+export const protectProedure = t.procedure.use(prom).use(isUser);
 
 const isSystemAdmin = isUser.unstable_pipe(async (opts) => {
   const { ctx, input } = opts;
@@ -74,7 +101,7 @@ const isSystemAdmin = isUser.unstable_pipe(async (opts) => {
   return opts.next();
 });
 
-export const systemAdminProcedure = t.procedure.use(isSystemAdmin);
+export const systemAdminProcedure = t.procedure.use(prom).use(isSystemAdmin);
 export const workspaceProcedure = protectProedure
   .input(
     z.object({

src/server/utils/prometheus/client.ts

@@ -0,0 +1,28 @@
+import Prometheus from 'prom-client';
+
+export const promTrpcRequest = new Prometheus.Histogram({
+  name: 'trpc_request_duration_seconds',
+  help: 'Duration of TRPC requests in seconds',
+  labelNames: ['route', 'type', 'status'],
+});
+
+export const promUserCounter = new Prometheus.Gauge({
+  name: 'tianji_user_counter',
+  help: 'user counter',
+});
+
+export const promWorkspaceCounter = new Prometheus.Gauge({
+  name: 'tianji_workspace_counter',
+  help: 'workspace counter',
+});
+
+export const promMonitorRunnerCounter = new Prometheus.Gauge({
+  name: 'tianji_monitor_runner_counter',
+  help: 'monitor runner counter',
+});
+
+export const promServerCounter = new Prometheus.Gauge({
+  name: 'tianji_server_count',
+  help: 'server count',
+  labelNames: ['workspaceId'],
+});

src/server/utils/prometheus/index.ts

@@ -0,0 +1,12 @@
+import { prisma } from '../../model/_client.js';
+import { promUserCounter, promWorkspaceCounter } from './client.js';
+
+export function initCounter() {
+  prisma.workspace.count().then((count) => {
+    promWorkspaceCounter.set(count);
+  });
+
+  prisma.user.count().then((count) => {
+    promUserCounter.set(count);
+  });
+}