initial version of benchmark codecheck

tools/benchmarks/check-benchmarks.ts

@@ -1 +1,163 @@
-// TODO: implement @krzkaczor
+import { getBenchmarks, Benchmarks, LIBS } from './get-benchmarks';
+import { codechecks, CodeChecksReport } from '@codechecks/client';
+import * as bytes from 'bytes';
+
+const markdownTable = require('markdown-table');
+
+const benchmarksKey = 'nest/performance-benchmark';
+
+export default async function checkBenchmarks() {
+  const currentBenchmarks = await getBenchmarks();
+
+  await codechecks.saveValue(benchmarksKey, currentBenchmarks);
+
+  if (!codechecks.isPr()) {
+    return;
+  }
+
+  const baselineBenchmarks = await codechecks.getValue<Benchmarks>(
+    benchmarksKey,
+  );
+
+  const report = getCodechecksReport(currentBenchmarks, baselineBenchmarks);
+
+  await codechecks.report(report);
+}
+
+function getCodechecksReport(
+  current: Benchmarks,
+  baseline: Benchmarks | undefined,
+): CodeChecksReport {
+  const diff = getDiff(current, baseline);
+
+  const shortDescription = getShortDescription(baseline, diff);
+  const longDescription = getLongDescription(current, baseline, diff);
+
+  return {
+    name: 'Benchmarks',
+    status: 'success',
+    shortDescription,
+    longDescription,
+  };
+}
+
+function getShortDescription(
+  baseline: Benchmarks | undefined,
+  diff: BenchmarksDiff,
+): string {
+  if (!baseline) {
+    return 'New benchmarks generated';
+  }
+
+  const avgDiff = getAverageDiff(diff);
+
+  if (avgDiff > 0) {
+    return `Performance improved by ${avgDiff.toFixed(
+      2,
+    )}% on average, good job!`;
+  }
+  if (avgDiff === 0) {
+    return `No changes in performance detected`;
+  }
+  if (avgDiff < 0) {
+    return `Performance decreased by ${avgDiff.toFixed(
+      2,
+    )}% on average, be careful!`;
+  }
+}
+
+function getLongDescription(
+  current: Benchmarks,
+  baseline: Benchmarks | undefined,
+  diff: BenchmarksDiff,
+): string {
+  const table = [
+    ['', 'Req/sec', 'Trans/sec', 'Req/sec DIFF', 'Trans/sec DIFF'],
+    [
+      'Nest-Express',
+      current['nest'].requestsPerSec,
+      current['nest'].transferPerSec,
+      baseline ? diff['nest'].requestsPerSecDuff : '-',
+      baseline ? diff['nest'].transferPerSecDiff : '-',
+    ],
+    [
+      'Nest-Fastify',
+      current['nest-fastify'].requestsPerSec,
+      current['nest-fastify'].transferPerSec,
+      baseline ? diff['nest-fastify'].requestsPerSecDuff : '-',
+      baseline ? diff['nest-fastify'].transferPerSecDiff : '-',
+    ],
+    [
+      'Express',
+      current['express'].requestsPerSec,
+      current['express'].transferPerSec,
+      baseline ? diff['express'].requestsPerSecDuff : '-',
+      baseline ? diff['express'].transferPerSecDiff : '-',
+    ],
+    [
+      'Fastify',
+      current['fastify'].requestsPerSec,
+      current['fastify'].transferPerSec,
+      baseline ? diff['fastify'].requestsPerSecDuff : '-',
+      baseline ? diff['fastify'].transferPerSecDiff : '-',
+    ],
+  ];
+
+  return markdownTable(table);
+}
+
+function getDiff(
+  current: Benchmarks,
+  baseline: Benchmarks | undefined,
+): BenchmarksDiff {
+  const diff = {};
+  for (const l of LIBS) {
+    if (!baseline) {
+      diff[l] = undefined;
+      continue;
+    }
+
+    const currentValue = current[l];
+    const baselineValue = baseline[l];
+
+    diff[l] = {
+      requestsPerSec: getRequestDiff(
+        currentValue.requestsPerSec,
+        baselineValue.requestsPerSec,
+      ),
+      transferPerSecDiff: getTransferDiff(
+        currentValue.transferPerSec,
+        baselineValue.transferPerSec,
+      ),
+    };
+  }
+  return diff;
+}
+
+function getTransferDiff(
+  currentTransfer: string,
+  baselineTransfer: string,
+): number {
+  return 1 - bytes.parse(currentTransfer) / bytes.parse(baselineTransfer);
+}
+
+function getAverageDiff(diff: BenchmarksDiff) {
+  return (
+    (diff['nest'].transferPerSecDiff +
+      diff['nest'].requestsPerSecDuff +
+      diff['nest-fastify'].transferPerSecDiff +
+      diff['nest-fastify'].requestsPerSecDuff) /
+    4
+  );
+}
+
+function getRequestDiff(currentRequest: number, baselineRequest: number) {
+  return 1 - currentRequest / baselineRequest;
+}
+
+interface BenchmarkDiff {
+  transferPerSecDiff: number | undefined;
+  requestsPerSecDuff: number | undefined;
+}
+
+type BenchmarksDiff = { [lib: string]: BenchmarkDiff };

tools/benchmarks/get-benchmarks.ts

@@ -2,6 +2,8 @@ import wrkPkg = require('wrk');
 import { spawn } from 'child_process';
 import { join } from 'path';
 
+export type Benchmarks = { [lib: string]: WrkResults };
+
 const wrk = (options: any) =>
   new Promise<WrkResults>((resolve, reject) =>
     wrkPkg(options, (err: any, result: any) =>
@@ -13,13 +15,13 @@ const sleep = (time: number) =>
   new Promise(resolve => setTimeout(resolve, time));
 
 const BENCHMARK_PATH = join(__dirname, '../../benchmarks');
-const LIBS = ['express', 'fastify', 'nest', 'nest-fastify'];
+export const LIBS = ['express', 'fastify', 'nest', 'nest-fastify'];
 
 async function runBenchmarkOfLib(lib: string): Promise<WrkResults> {
   const libPath = join(BENCHMARK_PATH, `${lib}.js`);
   const process = spawn('node', [libPath], {
     detached: true,
-    stdio: 'ignore'
+    stdio: 'ignore',
   });
 
   process.unref();
@@ -39,7 +41,7 @@ async function runBenchmarkOfLib(lib: string): Promise<WrkResults> {
 }
 
 export async function getBenchmarks() {
-  const results: { [lib: string]: WrkResults } = {};
+  const results: Benchmarks = {};
   for await (const lib of LIBS) {
     const result = await runBenchmarkOfLib(lib);
     results[lib] = result;