resolve conflicts

tools/benchmarks/check-benchmarks.ts

@@ -0,0 +1,160 @@
+import { codechecks, CodeChecksReport } from '@codechecks/client';
+import * as bytes from 'bytes';
+import { Benchmarks, getBenchmarks, LIBS } from './get-benchmarks';
+
+const markdownTable = require('markdown-table');
+const benchmarksKey = 'nest/performance-benchmark';
+
+export default async function checkBenchmarks() {
+  const currentBenchmarks = await getBenchmarks();
+  await codechecks.saveValue(benchmarksKey, currentBenchmarks);
+
+  if (!codechecks.isPr()) {
+    return;
+  }
+  const baselineBenchmarks = await codechecks.getValue<Benchmarks>(
+    benchmarksKey,
+  );
+  const report = getCodechecksReport(currentBenchmarks, baselineBenchmarks);
+  await codechecks.report(report);
+}
+
+function getCodechecksReport(
+  current: Benchmarks,
+  baseline: Benchmarks | undefined,
+): CodeChecksReport {
+  const diff = getDiff(current, baseline);
+
+  const shortDescription = getShortDescription(baseline, diff);
+  const longDescription = getLongDescription(current, baseline, diff);
+
+  return {
+    name: 'Benchmarks',
+    status: 'success',
+    shortDescription,
+    longDescription,
+  };
+}
+
+function getShortDescription(
+  baseline: Benchmarks | undefined,
+  diff: BenchmarksDiff,
+): string {
+  if (!baseline) {
+    return 'New benchmarks generated';
+  }
+
+  const avgDiff = getAverageDiff(diff);
+  if (avgDiff > 0) {
+    return `Performance improved by ${avgDiff.toFixed(
+      2,
+    )}% on average, good job!`;
+  }
+  if (avgDiff === 0) {
+    return `No changes in performance detected`;
+  }
+  if (avgDiff < 0) {
+    return `Performance decreased by ${avgDiff.toFixed(
+      2,
+    )}% on average, be careful!`;
+  }
+}
+
+function getLongDescription(
+  current: Benchmarks,
+  baseline: Benchmarks | undefined,
+  diff: BenchmarksDiff,
+): string {
+  const table = [
+    ['', 'Req/sec', 'Trans/sec', 'Req/sec DIFF', 'Trans/sec DIFF'],
+    [
+      'Nest-Express',
+      // tslint:disable:no-string-literal
+      current['nest'].requestsPerSec,
+      current['nest'].transferPerSec,
+      baseline ? diff['nest'].requestsPerSecDuff : '-',
+      baseline ? diff['nest'].transferPerSecDiff : '-',
+    ],
+    [
+      'Nest-Fastify',
+      current['nest-fastify'].requestsPerSec,
+      current['nest-fastify'].transferPerSec,
+      baseline ? diff['nest-fastify'].requestsPerSecDuff : '-',
+      baseline ? diff['nest-fastify'].transferPerSecDiff : '-',
+    ],
+    [
+      'Express',
+      current['express'].requestsPerSec,
+      current['express'].transferPerSec,
+      baseline ? diff.express.requestsPerSecDuff : '-',
+      baseline ? diff['express'].transferPerSecDiff : '-',
+    ],
+    [
+      'Fastify',
+      current['fastify'].requestsPerSec,
+      current['fastify'].transferPerSec,
+      baseline ? diff['fastify'].requestsPerSecDuff : '-',
+      baseline ? diff['fastify'].transferPerSecDiff : '-',
+    ],
+  ];
+
+  return markdownTable(table);
+}
+
+function getDiff(
+  current: Benchmarks,
+  baseline: Benchmarks | undefined,
+): BenchmarksDiff {
+  const diff = {};
+  for (const l of LIBS) {
+    if (!baseline) {
+      diff[l] = undefined;
+      continue;
+    }
+
+    const currentValue = current[l];
+    const baselineValue = baseline[l];
+
+    diff[l] = {
+      requestsPerSec: getRequestDiff(
+        currentValue.requestsPerSec,
+        baselineValue.requestsPerSec,
+      ),
+      transferPerSecDiff: getTransferDiff(
+        currentValue.transferPerSec,
+        baselineValue.transferPerSec,
+      ),
+    };
+  }
+  return diff;
+}
+
+function getTransferDiff(
+  currentTransfer: string,
+  baselineTransfer: string,
+): number {
+  return 1 - bytes.parse(currentTransfer) / bytes.parse(baselineTransfer);
+}
+
+function getAverageDiff(diff: BenchmarksDiff) {
+  return (
+    (diff['nest'].transferPerSecDiff +
+      diff['nest'].requestsPerSecDuff +
+      diff['nest-fastify'].transferPerSecDiff +
+      diff['nest-fastify'].requestsPerSecDuff) /
+    4
+  );
+}
+
+function getRequestDiff(currentRequest: number, baselineRequest: number) {
+  return 1 - currentRequest / baselineRequest;
+}
+
+interface BenchmarkDiff {
+  transferPerSecDiff: number | undefined;
+  requestsPerSecDuff: number | undefined;
+}
+
+interface BenchmarksDiff {
+  [lib: string]: BenchmarkDiff;
+}

tools/benchmarks/get-benchmarks.ts

@@ -0,0 +1,71 @@
+import wrkPkg = require('wrk');
+import { spawn } from 'child_process';
+import { join } from 'path';
+
+export interface Benchmarks {
+  [lib: string]: WrkResults;
+}
+
+const wrk = (options: any) =>
+  new Promise<WrkResults>((resolve, reject) =>
+    wrkPkg(options, (err: any, result: any) =>
+      err ? reject(err) : resolve(result),
+    ),
+  );
+
+const sleep = (time: number) =>
+  new Promise(resolve => setTimeout(resolve, time));
+
+const BENCHMARK_PATH = join(__dirname, '../../benchmarks');
+export const LIBS = ['express', 'fastify', 'nest', 'nest-fastify'];
+
+async function runBenchmarkOfLib(lib: string): Promise<WrkResults> {
+  const libPath = join(BENCHMARK_PATH, `${lib}.js`);
+  const process = spawn('node', [libPath], {
+    detached: true,
+    stdio: 'ignore',
+  });
+
+  process.unref();
+
+  await sleep(2000);
+
+  const result = await wrk({
+    threads: 8,
+    duraton: '10s',
+    connections: 1024,
+    url: 'http://localhost:3000',
+  });
+
+  process.kill();
+  return result;
+}
+
+export async function getBenchmarks() {
+  const results: Benchmarks = {};
+  for await (const lib of LIBS) {
+    const result = await runBenchmarkOfLib(lib);
+    results[lib] = result;
+  }
+  return results;
+}
+
+interface WrkResults {
+  transferPerSec: string;
+  requestsPerSec: number;
+  connectErrors: string;
+  readErrors: string;
+  writeErrors: string;
+  timeoutErrors: string;
+  requestsTotal: number;
+  durationActual: string;
+  transferTotal: string;
+  latencyAvg: string;
+  latencyStdev: string;
+  latencyMax: string;
+  latencyStdevPerc: number;
+  rpsAvg: string;
+  rpsStdev: string;
+  rpsMax: string;
+  rpsStdevPerc: number;
+}

tools/benchmarks/report-contents.md

@@ -0,0 +1,20 @@
+Short description (shown on main PR screen): Performance improved 0.13% on average.
+
+Long description (after clicking details):
+
+|                | Req/sec | Trans/sec | Req/sec DIFF | Trans/sec DIFF | Req vs Express | Trans vs Fastify |
+| -------------- | ------- | --------- | ------------ | -------------- | -------------- | ---------------- |
+| NestJS-Express | 3.37MB  | 16375.58  | +0.15%       | +0.14%         | 80.62%         | 80.37%           |
+| NestJS-Fastify | 4.78MB  | 32728.51  | +0.12%       | +0.12          | 64.76%         | 64.25%           |
+| Express        | 4.18MB  | 20374.59  | 0%           | 0%             | -              | -                |
+| Fastify        | 7.38MB  | 50938     | 0%           | 0%             | -              | -                |
+
+## Explanations:
+
+Short description: average of all diffs for NestJS-\* so: `(0.15 + 0.14 + 0.12 + 0.12) / 4`
+
+Long description:
+
+`req/sec DIFF` and `Trans/sec DIFF` is in comparison to the baseline on target branch (master).
+
+Req vs express is calculated as perf compared to NOT using nestjs so: 80.62% = (3.37/4.18) \* 100%