feat: update benchmarks

tools/benchmarks/check-benchmarks.ts

@@ -1,148 +0,0 @@
-import { codechecks, CodeChecksReport } from '@codechecks/client';
-import * as bytes from 'bytes';
-import { Benchmarks, getBenchmarks, LIBS } from './get-benchmarks';
-
-const markdownTable = require('markdown-table');
-const benchmarksKey = 'nest/performance-benchmark';
-
-export default async function checkBenchmarks() {
-  const currentBenchmarks = await getBenchmarks();
-  await codechecks.saveValue(benchmarksKey, currentBenchmarks);
-
-  if (!codechecks.isPr()) {
-    return;
-  }
-  const baselineBenchmarks =
-    await codechecks.getValue<Benchmarks>(benchmarksKey);
-  const report = getCodechecksReport(currentBenchmarks, baselineBenchmarks);
-  await codechecks.report(report);
-}
-
-function getCodechecksReport(
-  current: Benchmarks,
-  baseline: Benchmarks | undefined,
-): CodeChecksReport {
-  const diff = getDiff(current, baseline);
-
-  const shortDescription = getShortDescription(baseline, diff);
-  const longDescription = getLongDescription(current, baseline, diff);
-
-  return {
-    name: 'Benchmarks',
-    status: 'success',
-    shortDescription,
-    longDescription,
-  };
-}
-
-function getShortDescription(
-  baseline: Benchmarks | undefined,
-  diff: BenchmarksDiff,
-): string {
-  if (!baseline) {
-    return 'New benchmarks generated';
-  }
-
-  const avgDiff = getAverageDiff(diff);
-  if (avgDiff > 0) {
-    return `Performance improved by ${avgDiff.toFixed(
-      2,
-    )}% on average, good job!`;
-  }
-  if (avgDiff === 0) {
-    return `No changes in performance detected`;
-  }
-  if (avgDiff < 0) {
-    return `Performance decreased by ${avgDiff.toFixed(
-      2,
-    )}% on average, be careful!`;
-  }
-}
-
-function getLongDescription(
-  current: Benchmarks,
-  baseline: Benchmarks | undefined,
-  diff: BenchmarksDiff,
-): string {
-  function printTableRow(id: string, label: string): string[] {
-    return [
-      label,
-      current[id].requestsPerSec.toFixed(0),
-      current[id].transferPerSec,
-      baseline ? formatPerc(diff[id].requestsPerSecDiff) : '-',
-      baseline ? formatPerc(diff[id].transferPerSecDiff) : '-',
-    ];
-  }
-
-  const table = [
-    ['', 'Req/sec', 'Trans/sec', 'Req/sec DIFF', 'Trans/sec DIFF'],
-    printTableRow('nest', 'Nest-Express'),
-    printTableRow('nest-fastify', 'Nest-Fastify'),
-    printTableRow('express', 'Express'),
-    printTableRow('fastify', 'Fastify'),
-  ];
-
-  return markdownTable(table);
-}
-
-function getDiff(
-  current: Benchmarks,
-  baseline: Benchmarks | undefined,
-): BenchmarksDiff {
-  const diff: BenchmarksDiff = {};
-  for (const l of LIBS) {
-    if (!baseline) {
-      diff[l] = undefined;
-      continue;
-    }
-
-    const currentValue = current[l];
-    const baselineValue = baseline[l];
-
-    diff[l] = {
-      requestsPerSecDiff: getRequestDiff(
-        currentValue.requestsPerSec,
-        baselineValue.requestsPerSec,
-      ),
-      transferPerSecDiff: getTransferDiff(
-        currentValue.transferPerSec,
-        baselineValue.transferPerSec,
-      ),
-    };
-  }
-  return diff;
-}
-
-function getTransferDiff(
-  currentTransfer: string,
-  baselineTransfer: string,
-): number {
-  return 1 - bytes.parse(currentTransfer) / bytes.parse(baselineTransfer);
-}
-
-function getAverageDiff(diff: BenchmarksDiff) {
-  return (
-    (diff['nest'].transferPerSecDiff +
-      diff['nest'].requestsPerSecDiff +
-      diff['nest-fastify'].transferPerSecDiff +
-      diff['nest-fastify'].requestsPerSecDiff) /
-    4
-  );
-}
-
-function getRequestDiff(currentRequest: number, baselineRequest: number) {
-  return 1 - currentRequest / baselineRequest;
-}
-
-interface BenchmarkDiff {
-  transferPerSecDiff: number | undefined;
-  requestsPerSecDiff: number | undefined;
-}
-
-interface BenchmarksDiff {
-  [lib: string]: BenchmarkDiff;
-}
-
-function formatPerc(n: number) {
-  return (n > 0 ? '+' : '') + (n * 100).toFixed(2) + '%';
-}

tools/benchmarks/get-benchmarks.ts

@@ -1,77 +0,0 @@
-import wrkPkg = require('wrk');
-import { spawn } from 'child_process';
-import { join } from 'path';
-
-export interface Benchmarks {
-  [lib: string]: WrkResults;
-}
-
-const wrk = (options: any) =>
-  new Promise<WrkResults>((resolve, reject) =>
-    wrkPkg(options, (err: any, result: any) =>
-      err ? reject(err) : resolve(result),
-    ),
-  );
-
-const sleep = (time: number) =>
-  new Promise(resolve => setTimeout(resolve, time));
-
-const BENCHMARK_PATH = join(__dirname, '../../benchmarks');
-export const LIBS = ['express', 'fastify', 'nest', 'nest-fastify'];
-
-async function runBenchmarkOfLib(lib: string): Promise<WrkResults> {
-  const libPath = join(BENCHMARK_PATH, `${lib}.js`);
-  const process = spawn('node', [libPath], {
-    detached: true,
-  });
-
-  process.stdout!.on('data', data => {
-    console.log(`stdout: ${data}`);
-  });
-  process.stderr!.on('data', data => {
-    console.log(`stderr: ${data}`);
-  });
-
-  process.unref();
-
-  await sleep(2000);
-
-  const result = await wrk({
-    threads: 8,
-    duration: '10s',
-    connections: 1024,
-    url: 'http://localhost:3000',
-  });
-
-  process.kill();
-  return result;
-}
-
-export async function getBenchmarks() {
-  const results: Benchmarks = {};
-  for await (const lib of LIBS) {
-    const result = await runBenchmarkOfLib(lib);
-    results[lib] = result;
-  }
-  return results;
-}
-
-interface WrkResults {
-  transferPerSec: string;
-  requestsPerSec: number;
-  connectErrors: string;
-  readErrors: string;
-  writeErrors: string;
-  timeoutErrors: string;
-  requestsTotal: number;
-  durationActual: string;
-  transferTotal: string;
-  latencyAvg: string;
-  latencyStdev: string;
-  latencyMax: string;
-  latencyStdevPerc: number;
-  rpsAvg: string;
-  rpsStdev: string;
-  rpsMax: string;
-  rpsStdevPerc: number;
-}

tools/benchmarks/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@nestjs/benchmarks",
+  "version": "1.0.0",
+  "description": "Nest - modern, fast, powerful node.js web framework (@benchmarks)",
+  "license": "ISC",
+  "scripts": {
+    "benchmarks": "tsc && node dist/main.js"
+  },
+  "dependencies": {
+    "@nestjs/platform-express": "^11.1.12",
+    "@nestjs/platform-fastify": "^11.1.12"
+  },
+  "devDependencies": {
+    "@types/autocannon": "^7.12.7",
+    "autocannon": "^8.0.0",
+    "typescript": "^5.9.3"
+  }
+}

tools/benchmarks/report-contents.md

@@ -1,20 +0,0 @@
-Short description (shown on main PR screen): Performance improved 0.13% on average.
-
-Long description (after clicking details):
-
-|                | Req/sec | Trans/sec | Req/sec DIFF | Trans/sec DIFF | Req vs Express | Trans vs Fastify |
-| -------------- | ------- | --------- | ------------ | -------------- | -------------- | ---------------- |
-| NestJS-Express | 3.37MB  | 16375.58  | +0.15%       | +0.14%         | 80.62%         | 80.37%           |
-| NestJS-Fastify | 4.78MB  | 32728.51  | +0.12%       | +0.12          | 64.76%         | 64.25%           |
-| Express        | 4.18MB  | 20374.59  | 0%           | 0%             | -              | -                |
-| Fastify        | 7.38MB  | 50938     | 0%           | 0%             | -              | -                |
-
-## Explanations:
-
-Short description: average of all diffs for NestJS-\* so: `(0.15 + 0.14 + 0.12 + 0.12) / 4`
-
-Long description:
-
-`req/sec DIFF` and `Trans/sec DIFF` is in comparison to the baseline on target branch (master).
-
-Req vs express is calculated as perf compared to NOT using nestjs so: 80.62% = (3.37/4.18) \* 100%

tools/benchmarks/src/autocannon/run.ts

@@ -0,0 +1,32 @@
+import autocannon from 'autocannon';
+
+export type RunOptions = autocannon.Options & {
+  /**
+   * When true, prints live progress to stdout via `autocannon.track(instance)`.
+   * Default: false (quiet)
+   */
+  verbose?: boolean;
+};
+
+/**
+ * Runs an autocannon benchmark.
+ *
+ * By default this is quiet (no live table/progress). Pass `{ verbose: true }`
+ * to enable `autocannon.track(...)` output.
+ */
+export const run = (options: RunOptions) =>
+  new Promise<autocannon.Result>((resolve, reject) => {
+    const { verbose = false, ...autocannonOptions } = options;
+
+    const instance = autocannon(autocannonOptions, (err, result) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(result);
+      }
+    });
+
+    if (verbose) {
+      autocannon.track(instance);
+    }
+  });

tools/benchmarks/src/frameworks/express.ts

@@ -0,0 +1,9 @@
+import express from 'express';
+
+const app = express();
+
+app.get('/', async (_, res) => {
+  res.send('Hello world');
+});
+
+app.listen(3000);

tools/benchmarks/src/frameworks/fastify.ts

@@ -0,0 +1,14 @@
+import Fastify from 'fastify';
+
+const fastify = Fastify({
+  logger: false,
+});
+fastify.get('/', async (_, reply) => reply.send('Hello world'));
+fastify
+  .listen({
+    port: 3000,
+  })
+  .catch(err => {
+    console.error(err);
+    process.exit(1);
+  });

tools/benchmarks/src/frameworks/nest-express.ts

@@ -0,0 +1,13 @@
+import { NestFactory } from '@nestjs/core';
+import { ExpressAdapter } from '@nestjs/platform-express';
+
+import { AppModule } from './nest/app.module';
+
+NestFactory.create(AppModule, new ExpressAdapter(), {
+  logger: false,
+  bodyParser: false,
+})
+  .then(app => app.listen(3000))
+  .catch(error => {
+    console.error('Error starting Nest.js application:', error);
+  });

tools/benchmarks/src/frameworks/nest-fastify.ts

@@ -0,0 +1,13 @@
+import { NestFactory } from '@nestjs/core';
+import { FastifyAdapter } from '@nestjs/platform-fastify';
+
+import { AppModule } from './nest/app.module';
+
+NestFactory.create(AppModule, new FastifyAdapter(), {
+  logger: false,
+  bodyParser: false,
+})
+  .then(app => app.listen(3000))
+  .catch(error => {
+    console.error('Error starting Nest.js application:', error);
+  });

tools/benchmarks/src/frameworks/nest/app.controller.ts

@@ -0,0 +1,9 @@
+import { Controller, Get } from '@nestjs/common';
+
+@Controller('/')
+export class AppController {
+  @Get()
+  root() {
+    return 'Hello world!';
+  }
+}

tools/benchmarks/src/frameworks/nest/app.module.ts

@@ -0,0 +1,8 @@
+import { Module } from '@nestjs/common';
+import { AppController } from './app.controller';
+
+@Module({
+  imports: [],
+  controllers: [AppController],
+})
+export class AppModule {}

tools/benchmarks/src/main.ts

@@ -0,0 +1,349 @@
+import { fork, ChildProcess } from 'node:child_process';
+import { join } from 'node:path';
+import { run } from './autocannon/run';
+
+type Framework = 'express' | 'fastify' | 'nest-express' | 'nest-fastify';
+
+type Args = {
+  connections: number;
+  duration: number;
+  pipelining: number;
+  verbose: boolean;
+  port: number;
+  path: string;
+};
+
+type BenchmarkSummary = {
+  requestsPerSecAvg: number;
+  latencyAvgMs: number;
+  latencyP99Ms: number;
+  throughputBytesPerSecAvg: number;
+};
+
+const DEFAULTS: Args = {
+  connections: 100,
+  duration: 10,
+  pipelining: 10,
+  verbose: false,
+  port: 3000,
+  path: '/',
+};
+
+function parseNumber(value: string | undefined, fallback: number): number {
+  if (!value) return fallback;
+  const n = Number(value);
+  return Number.isFinite(n) && n > 0 ? n : fallback;
+}
+
+function parseArgs(argv: string[]): Args {
+  const args: Args = { ...DEFAULTS };
+  const only: Framework[] = [];
+
+  for (let i = 0; i < argv.length; i++) {
+    const token = argv[i];
+
+    if (token === '--verbose' || token === '-v') {
+      args.verbose = true;
+      continue;
+    }
+
+    if (token === '--connections' || token === '-c') {
+      args.connections = parseNumber(argv[++i], args.connections);
+      continue;
+    }
+
+    if (token === '--duration' || token === '-d') {
+      args.duration = parseNumber(argv[++i], args.duration);
+      continue;
+    }
+
+    if (token === '--pipelining' || token === '-p') {
+      args.pipelining = parseNumber(argv[++i], args.pipelining);
+      continue;
+    }
+
+    if (token === '--port') {
+      args.port = parseNumber(argv[++i], args.port);
+      continue;
+    }
+
+    if (token === '--path') {
+      args.path = argv[++i] ?? args.path;
+      continue;
+    }
+
+    if (token === '--help' || token === '-h') {
+      printHelpAndExit(0);
+    }
+  }
+
+  return args;
+}
+
+function printHelpAndExit(code: number): never {
+  // Keep this simple so `npm run benchmarks` works without extra args.
+  // You can optionally narrow execution with `--framework` or `--only`.
+  // Example: `npm run benchmarks -- --framework fastify`
+  // Example: `npm run benchmarks -- --only express,fastify`
+  // Tuning: `--connections`, `--duration`, `--pipelining`, `--port`, `--path`
+  // Notes: The benchmark spawns the framework server as a child process and
+  // runs autocannon against it.
+  //
+  // eslint-disable-next-line no-console
+  console.log(
+    `
+Usage:
+  npm run benchmarks [-- --verbose]
+  npm run benchmarks [-- --connections 100] [-- --duration 10] [-- --pipelining 10]
+  npm run benchmarks [-- --port 3000] [-- --path /]
+
+Defaults:
+  verbose:   ${DEFAULTS.verbose}
+  connections: ${DEFAULTS.connections}
+  duration:    ${DEFAULTS.duration}
+  pipelining:  ${DEFAULTS.pipelining}
+  port:        ${DEFAULTS.port}
+  path:        ${DEFAULTS.path}
+`.trim(),
+  );
+  process.exit(code);
+}
+
+function frameworkEntry(framework: Framework): string {
+  return join(__dirname, '.', 'frameworks', framework);
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+async function waitForServer(url: string, timeoutMs = 5_000): Promise<void> {
+  const start = Date.now();
+  // Use global fetch if available (Node 18+). If not, just wait a bit.
+  const hasFetch = typeof (globalThis as any).fetch === 'function';
+
+  while (Date.now() - start < timeoutMs) {
+    try {
+      if (hasFetch) {
+        const res = await (globalThis as any).fetch(url, { method: 'GET' });
+        if (res && (res.status === 200 || res.status === 404)) return;
+      } else {
+        // best-effort fallback
+        await sleep(250);
+        return;
+      }
+    } catch {
+      // server not ready yet
+    }
+    await sleep(100);
+  }
+
+  throw new Error(`Server did not become ready in ${timeoutMs}ms: ${url}`);
+}
+
+function killChild(child: ChildProcess): void {
+  if (!child || child.killed) return;
+
+  // Try graceful termination first.
+  try {
+    child.kill('SIGTERM');
+  } catch {
+    // ignore
+  }
+
+  // Force kill if still alive shortly after.
+  setTimeout(() => {
+    try {
+      if (!child.killed) child.kill('SIGKILL');
+    } catch {
+      // ignore
+    }
+  }, 1_000).unref?.();
+}
+
+function toNumber(value: unknown, fallback = 0): number {
+  const n = typeof value === 'number' ? value : Number(value);
+  return Number.isFinite(n) ? n : fallback;
+}
+
+function summarizeResult(result: any): BenchmarkSummary {
+  return {
+    requestsPerSecAvg: toNumber(result?.requests?.average),
+    latencyAvgMs: toNumber(result?.latency?.average),
+    latencyP99Ms: toNumber(result?.latency?.p99),
+    throughputBytesPerSecAvg: toNumber(result?.throughput?.average),
+  };
+}
+
+function pctChange(newValue: number, oldValue: number): number | null {
+  if (
+    !Number.isFinite(newValue) ||
+    !Number.isFinite(oldValue) ||
+    oldValue === 0
+  )
+    return null;
+  return ((newValue - oldValue) / oldValue) * 100;
+}
+
+function fmtPct(p: number | null, digits = 1): string {
+  if (p == null) return 'n/a';
+  const sign = p >= 0 ? '+' : '';
+  return `${sign}${p.toFixed(digits)}%`;
+}
+
+function fmtNum(n: number, digits = 2): string {
+  if (!Number.isFinite(n)) return 'n/a';
+  return n.toFixed(digits);
+}
+
+function fmtBytesPerSec(n: number): string {
+  if (!Number.isFinite(n)) return 'n/a';
+  const abs = Math.abs(n);
+  if (abs >= 1024 ** 3) return `${(n / 1024 ** 3).toFixed(2)} GB/s`;
+  if (abs >= 1024 ** 2) return `${(n / 1024 ** 2).toFixed(2)} MB/s`;
+  if (abs >= 1024) return `${(n / 1024).toFixed(2)} KB/s`;
+  return `${n.toFixed(0)} B/s`;
+}
+
+function printComparison(
+  baseName: string,
+  base: BenchmarkSummary | undefined,
+  nestName: string,
+  nest: BenchmarkSummary | undefined,
+): void {
+  // eslint-disable-next-line no-console
+  console.log(`\n--- Comparison: ${baseName} <-> ${nestName} ---`);
+
+  if (!base || !nest) {
+    // eslint-disable-next-line no-console
+    console.log(
+      `Missing results: ${baseName}=${base ? 'ok' : 'missing'} ${nestName}=${nest ? 'ok' : 'missing'}`,
+    );
+    return;
+  }
+
+  const rpsPct = pctChange(nest.requestsPerSecAvg, base.requestsPerSecAvg);
+  const latAvgPct = pctChange(nest.latencyAvgMs, base.latencyAvgMs);
+  const latP99Pct = pctChange(nest.latencyP99Ms, base.latencyP99Ms);
+  const thrPct = pctChange(
+    nest.throughputBytesPerSecAvg,
+    base.throughputBytesPerSecAvg,
+  );
+
+  // eslint-disable-next-line no-console
+  console.log(
+    `Requests/sec avg: ${fmtNum(base.requestsPerSecAvg)} -> ${fmtNum(nest.requestsPerSecAvg)} (${fmtPct(rpsPct)})`,
+  );
+  // eslint-disable-next-line no-console
+  console.log(
+    `Latency avg (ms): ${fmtNum(base.latencyAvgMs)} -> ${fmtNum(nest.latencyAvgMs)} (${fmtPct(latAvgPct)})`,
+  );
+  // eslint-disable-next-line no-console
+  console.log(
+    `Latency p99 (ms): ${fmtNum(base.latencyP99Ms)} -> ${fmtNum(nest.latencyP99Ms)} (${fmtPct(latP99Pct)})`,
+  );
+  // eslint-disable-next-line no-console
+  console.log(
+    `Throughput avg:  ${fmtBytesPerSec(base.throughputBytesPerSecAvg)} -> ${fmtBytesPerSec(nest.throughputBytesPerSecAvg)} (${fmtPct(thrPct)})`,
+  );
+}
+
+async function runOne(
+  framework: Framework,
+  args: Args,
+): Promise<BenchmarkSummary> {
+  const child = fork(frameworkEntry(framework), {
+    stdio: 'ignore',
+  });
+
+  const url = `http://localhost:${args.port}${args.path.startsWith('/') ? args.path : `/${args.path}`}`;
+
+  try {
+    await waitForServer(url);
+
+    // eslint-disable-next-line no-console
+    console.log(`\n=== ${framework.toUpperCase()} ===`);
+    // eslint-disable-next-line no-console
+    console.log(`Target: ${url}`);
+    // eslint-disable-next-line no-console
+    console.log(
+      `connections=${args.connections} duration=${args.duration} pipelining=${args.pipelining}`,
+    );
+
+    // eslint-disable-next-line no-console
+    console.log(`Warmup: ${framework.toUpperCase()}`);
+    await run({
+      url,
+      connections: args.connections,
+      duration: args.duration,
+      pipelining: args.pipelining,
+      verbose: args.verbose,
+    });
+    // eslint-disable-next-line no-console
+    console.log(`Warmup ended: ${framework.toUpperCase()}`);
+
+    const result = (await run({
+      url,
+      connections: args.connections,
+      duration: args.duration,
+      pipelining: args.pipelining,
+      verbose: args.verbose,
+    })) as any;
+
+    const summary = summarizeResult(result);
+
+    // Keep output readable and stable across autocannon versions.
+    // eslint-disable-next-line no-console
+    console.log(`Requests/sec: ${summary.requestsPerSecAvg}`);
+    // eslint-disable-next-line no-console
+    console.log(
+      `Latency (ms): avg=${summary.latencyAvgMs} p99=${summary.latencyP99Ms}`,
+    );
+    // eslint-disable-next-line no-console
+    console.log(`Throughput (B/s): avg=${summary.throughputBytesPerSecAvg}`);
+
+    return summary;
+  } finally {
+    killChild(child);
+  }
+}
+
+async function main(): Promise<void> {
+  const args = parseArgs(process.argv.slice(2));
+
+  const targets: Framework[] = [
+    'express',
+    'nest-express',
+    'fastify',
+    'nest-fastify',
+  ];
+
+  const results: Partial<Record<Framework, BenchmarkSummary>> = {};
+
+  // Run sequentially to avoid port conflicts (both frameworks listen on the same port).
+  for (const fw of targets) {
+    results[fw] = await runOne(fw, args);
+    // small cooldown between runs
+    await sleep(250);
+  }
+
+  // Compare raw framework vs Nest adapter for same underlying HTTP server
+  printComparison(
+    'EXPRESS',
+    results['express'],
+    'NEST-EXPRESS',
+    results['nest-express'],
+  );
+  printComparison(
+    'FASTIFY',
+    results['fastify'],
+    'NEST-FASTIFY',
+    results['nest-fastify'],
+  );
+}
+
+main().catch(err => {
+  // eslint-disable-next-line no-console
+  console.error(err);
+  process.exit(1);
+});

tools/benchmarks/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "https://www.schemastore.org/tsconfig",
+  "_version": "18.2.0",
+
+  "compilerOptions": {
+    "lib": ["es2023"],
+    "module": "node16",
+    "target": "es2022",
+
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "moduleResolution": "node16",
+
+    "outDir": "dist",
+
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "sourceMap": true,
+  },
+}