From 7113bfeea4db2fe5f9884a8dea2cf2743c3249ee Mon Sep 17 00:00:00 2001 From: Erik Soehnel Date: Tue, 17 May 2022 15:09:52 +0200 Subject: [PATCH 1/5] benchmark each module in their own node process --- README.md | 7 +++ benchmarks/helpers/main.ts | 89 +++++++++++++++++++++++++++++++++----- benchmarks/index.ts | 6 ++- cases/index.ts | 64 +++++++++++++++------------ index.ts | 71 ++++++++++++++++++++++++++++-- package.json | 2 +- start.sh | 3 +- test/benchmarks.test.ts | 45 ++++++++++++++++++- 8 files changed, 240 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 4423011db..84b0072ae 100644 --- a/README.md +++ b/README.md @@ -74,3 +74,10 @@ function isMyDataValid(data: any) { // `res` is now type casted to the right type const res = isMyDataValid(data) ``` + +## Local Development + +* `npm run start` - run benchmarks for all modules +* `npm run start run zod myzod valita` - run benchmarks only for a few selected modules +* `npm run docs:serve` - result viewer +* `npm run test` - run tests on all modules diff --git a/benchmarks/helpers/main.ts b/benchmarks/helpers/main.ts index cdfd9354a..61098ef78 100644 --- a/benchmarks/helpers/main.ts +++ b/benchmarks/helpers/main.ts @@ -1,5 +1,5 @@ import { add, complete, cycle, suite } from 'benny'; -import { readFileSync, writeFileSync } from 'fs'; +import { readFileSync, writeFileSync, existsSync, unlinkSync } from 'fs'; import { join } from 'path'; import { writePreviewGraph } from './graph'; import { getRegisteredBenchmarks } from './register'; @@ -10,15 +10,18 @@ const NODE_VERSION = process.env.NODE_VERSION || process.version; const NODE_VERSION_FOR_PREVIEW = 17; const TEST_PREVIEW_GENERATION = false; -export async function main() { +/** + * Run all registered benchmarks and append the results to a file. + */ +export async function runAllBenchmarks() { if (TEST_PREVIEW_GENERATION) { - // just generate the preview without using benchmark data from a previous run + // during development: generate the preview using benchmark data from a previous run const allResults: BenchmarkResult[] = JSON.parse( readFileSync(join(DOCS_DIR, 'results', 'node-17.json')).toString() ).results; await writePreviewGraph({ - filename: join(DOCS_DIR, 'results', 'preview.svg'), + filename: previewSvgFilename(), values: allResults, }); @@ -42,24 +45,40 @@ export async function main() { }); } - writeFileSync( - join(DOCS_DIR, 'results', `node-${majorVersion}.json`), + // collect results of isolated benchmark runs into a single file + appendResults(allResults); +} - JSON.stringify({ - results: allResults, - }), +/** + * Remove the results json file. + */ +export function deleteResults() { + const fileName = resultsJsonFilename(); - { encoding: 'utf8' } - ); + if (existsSync(fileName)) { + unlinkSync(fileName); + } +} + +/** + * Generate the preview svg shown in the readme. + */ +export async function createPreviewGraph() { + const majorVersion = getNodeMajorVersion(); if (majorVersion === NODE_VERSION_FOR_PREVIEW) { + const allResults: BenchmarkResult[] = JSON.parse( + readFileSync(resultsJsonFilename()).toString() + ).results; + await writePreviewGraph({ - filename: join(DOCS_DIR, 'results', 'preview.svg'), + filename: previewSvgFilename(), values: allResults, }); } } +// run a benchmark fn with benny async function runBenchmarks(name: string, cases: BenchmarkCase[]) { const fns = cases.map(c => add(c.moduleName, () => c.run())); @@ -74,6 +93,52 @@ async function runBenchmarks(name: string, cases: BenchmarkCase[]) { ); } +// append results to an existing file or create a new one +function appendResults(results: BenchmarkResult[]) { + const fileName = resultsJsonFilename(); + const existingResults: BenchmarkResult[] = existsSync(fileName) + ? JSON.parse(readFileSync(fileName).toString()).results + : []; + + // check that we're appending unique data + const getKey = ({ + benchmark, + name, + nodeVersion, + }: BenchmarkResult): string => { + return JSON.stringify({ benchmark, name, nodeVersion }); + }; + const existingResultsIndex = new Set(existingResults.map(r => getKey(r))); + + results.forEach(r => { + if (existingResultsIndex.has(getKey(r))) { + console.error('Result %s already exists in', getKey(r), fileName); + + throw new Error('Duplicate result in result json file'); + } + }); + + writeFileSync( + fileName, + + JSON.stringify({ + results: [...existingResults, ...results], + }), + + { encoding: 'utf8' } + ); +} + +function resultsJsonFilename() { + const majorVersion = getNodeMajorVersion(); + + return join(DOCS_DIR, 'results', `node-${majorVersion}.json`); +} + +function previewSvgFilename() { + return join(DOCS_DIR, 'results', 'preview.svg'); +} + function getNodeMajorVersion() { let majorVersion = 0; diff --git a/benchmarks/index.ts b/benchmarks/index.ts index 48486a4a2..deb41eb40 100644 --- a/benchmarks/index.ts +++ b/benchmarks/index.ts @@ -1,4 +1,8 @@ -export { main } from './helpers/main'; +export { + runAllBenchmarks, + createPreviewGraph, + deleteResults, +} from './helpers/main'; export { addCase, AvailableBenchmarksIds, diff --git a/cases/index.ts b/cases/index.ts index 7d75246e7..7a93a98de 100644 --- a/cases/index.ts +++ b/cases/index.ts @@ -1,28 +1,36 @@ -import './ajv'; -import './bueno'; -import './class-validator'; -import './computed-types'; -import './decoders'; -import './io-ts'; -import './jointz'; -import './json-decoder'; -import './marshal'; -import './mojotech-json-type-validation'; -import './myzod'; -import './ok-computer'; -import './purify-ts'; -import './rulr'; -import './runtypes'; -import './simple-runtypes'; -import './spectypes'; -import './superstruct'; -import './suretype'; -import './toi'; -import './tson'; -import './ts-interface-checker'; -import './ts-json-validator'; -import './ts-utils'; -import './typeofweb-schema'; -import './valita'; -import './yup'; -import './zod'; +export const cases = [ + 'ajv', + 'bueno', + 'class-validator', + 'computed-types', + 'decoders', + 'io-ts', + 'jointz', + 'json-decoder', + 'marshal', + 'mojotech-json-type-validation', + 'myzod', + 'ok-computer', + 'purify-ts', + 'rulr', + 'runtypes', + 'simple-runtypes', + 'spectypes', + 'superstruct', + 'suretype', + 'toi', + 'ts-interface-checker', + 'ts-json-validator', + 'ts-utils', + 'tson', + 'typeofweb-schema', + 'valita', + 'yup', + 'zod', +] as const; + +export type CaseName = typeof cases[number]; + +export async function importCase(caseName: CaseName) { + await import('./' + caseName); +} diff --git a/index.ts b/index.ts index 2b4f5fa1e..52ddf4775 100644 --- a/index.ts +++ b/index.ts @@ -1,4 +1,69 @@ -import { main } from './benchmarks'; -import './cases'; +import * as childProcess from 'child_process'; +import * as benchmarks from './benchmarks'; +import * as cases from './cases'; -main(); +async function main() { + // a runtype lib would be handy here to check the passed command names ;) + const [command, ...args] = process.argv.slice(2); + + switch (command) { + case undefined: + case 'run': + // run the given or all benchmarks, each in its own node process, see + // https://github.com/moltar/typescript-runtime-type-benchmarks/issues/864 + { + console.log('Removing previous results'); + benchmarks.deleteResults(); + + const caseNames = args.length ? args : cases.cases; + + for (let c of caseNames) { + if (c === 'spectypes') { + // hack: manually run the spectypes compilation step - avoids + // having to run it before any other benchmark, esp when working + // locally and checking against a few selected ones. + childProcess.execSync('npm run compile:spectypes', { + stdio: 'inherit', + }); + } + + const cmd = [...process.argv.slice(0, 2), 'run-internal', c]; + + console.log('Executing "%s"', c); + childProcess.execSync(cmd.join(' '), { + stdio: 'inherit', + }); + } + } + break; + + case 'create-preview-svg': + // separate command, because preview generation needs the accumulated + // results from the benchmark runs + await benchmarks.createPreviewGraph(); + break; + + case 'run-internal': + // run the given benchmark(s) & append the results + { + const caseNames = args as cases.CaseName[]; + + for (let c of caseNames) { + console.log('Loading "%s"', c); + + await cases.importCase(c); + } + + await benchmarks.runAllBenchmarks(); + } + break; + + default: + console.error('unknown command:', command); + process.exit(1); + } +} + +main().catch(e => { + throw e; +}); diff --git a/package.json b/package.json index 0ac6f8e6a..55df906ea 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "scripts": { "lint": "gts check", "lint:fix": "gts fix", - "start": "npm run compile:spectypes && ts-node index.ts", + "start": "ts-node index.ts", "test:build": "npm run compile:spectypes && tsc --noEmit", "test": "npm run compile:spectypes && jest", "docs:serve": "serve docs", diff --git a/start.sh b/start.sh index 8818b89fa..a51ea128f 100755 --- a/start.sh +++ b/start.sh @@ -4,4 +4,5 @@ set -ex export NODE_VERSION="${NODE_VERSION:-$(node -v)}" -npm start +npm run start +npm run start create-preview-svg diff --git a/test/benchmarks.test.ts b/test/benchmarks.test.ts index 4ca6dfbe4..ce5c5a977 100644 --- a/test/benchmarks.test.ts +++ b/test/benchmarks.test.ts @@ -1,5 +1,48 @@ import { getRegisteredBenchmarks } from '../benchmarks'; -import '../cases'; +import { cases } from '../cases'; + +// all cases need to be imported here because jest cannot pic up dynamically +// imported `test` and `describe` +import '../cases/ajv'; +import '../cases/bueno'; +import '../cases/class-validator'; +import '../cases/computed-types'; +import '../cases/decoders'; +import '../cases/io-ts'; +import '../cases/jointz'; +import '../cases/json-decoder'; +import '../cases/marshal'; +import '../cases/mojotech-json-type-validation'; +import '../cases/myzod'; +import '../cases/ok-computer'; +import '../cases/purify-ts'; +import '../cases/rulr'; +import '../cases/runtypes'; +import '../cases/simple-runtypes'; +import '../cases/spectypes'; +import '../cases/superstruct'; +import '../cases/suretype'; +import '../cases/toi'; +import '../cases/ts-interface-checker'; +import '../cases/ts-json-validator'; +import '../cases/ts-utils'; +import '../cases/tson'; +import '../cases/typeofweb-schema'; +import '../cases/valita'; +import '../cases/yup'; +import '../cases/zod'; + +test('all cases must have been imported in tests', () => { + const registeredCases = new Set(); + + getRegisteredBenchmarks().forEach(([benchmarkId, benchmarkCases]) => { + benchmarkCases.forEach(b => { + registeredCases.add(b.moduleName); + }); + }); + + expect(registeredCases.size).toEqual(cases.length); +}); getRegisteredBenchmarks().forEach(([benchmarkId, benchmarkCases]) => { describe(benchmarkId, () => { From f63bfbd645632402a2741f203a1cf19da27fa058 Mon Sep 17 00:00:00 2001 From: Erik Soehnel Date: Sun, 22 May 2022 18:28:49 +0200 Subject: [PATCH 2/5] fix spectypes .eslintignore The node/no-published-import did not fail before because the .gitignore in the spectypes directory. Using the global .gitignore triggers it through - looks like a bug in eslint? Nevertheless for this project the rule is of no use. --- .eslintignore | 2 +- .eslintrc.json | 3 +++ .gitignore | 5 ++++- cases/spectypes/.gitignore | 1 - 4 files changed, 8 insertions(+), 3 deletions(-) delete mode 100644 cases/spectypes/.gitignore diff --git a/.eslintignore b/.eslintignore index a2a44f452..764ef7576 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,3 +1,3 @@ **/node_modules docs/dist -compiled/spectypes/build +cases/spectypes/build diff --git a/.eslintrc.json b/.eslintrc.json index 9a13a60a1..1f7f4824d 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -1,5 +1,8 @@ { "extends": "./node_modules/gts/", + "rules": { + "node/no-unpublished-import": "off" + }, "env": { "jest": true } diff --git a/.gitignore b/.gitignore index 5ac5a0a47..d6522733b 100644 --- a/.gitignore +++ b/.gitignore @@ -80,7 +80,7 @@ typings/ # nuxt.js build output .nuxt -# react / gatsby +# react / gatsby public/ # vuepress build output @@ -99,3 +99,6 @@ public/ # sourcemaps docs/dist/app.js.map + +# spectype build artifacts +cases/spectypes/build diff --git a/cases/spectypes/.gitignore b/cases/spectypes/.gitignore deleted file mode 100644 index 378eac25d..000000000 --- a/cases/spectypes/.gitignore +++ /dev/null @@ -1 +0,0 @@ -build From 80753e45302ed64c7b8cb2f9d7aefff5fc4eb235 Mon Sep 17 00:00:00 2001 From: Erik Soehnel Date: Sun, 22 May 2022 19:19:07 +0200 Subject: [PATCH 3/5] fix eslint complaints --- benchmarks/helpers/main.ts | 1 - index.ts | 6 ++++-- test/benchmarks.test.ts | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/benchmarks/helpers/main.ts b/benchmarks/helpers/main.ts index 61098ef78..5e1838daf 100644 --- a/benchmarks/helpers/main.ts +++ b/benchmarks/helpers/main.ts @@ -28,7 +28,6 @@ export async function runAllBenchmarks() { return; } - const majorVersion = getNodeMajorVersion(); const allResults: BenchmarkResult[] = []; for (const [benchmark, benchmarks] of getRegisteredBenchmarks()) { diff --git a/index.ts b/index.ts index 52ddf4775..a992df613 100644 --- a/index.ts +++ b/index.ts @@ -17,7 +17,7 @@ async function main() { const caseNames = args.length ? args : cases.cases; - for (let c of caseNames) { + for (const c of caseNames) { if (c === 'spectypes') { // hack: manually run the spectypes compilation step - avoids // having to run it before any other benchmark, esp when working @@ -48,7 +48,7 @@ async function main() { { const caseNames = args as cases.CaseName[]; - for (let c of caseNames) { + for (const c of caseNames) { console.log('Loading "%s"', c); await cases.importCase(c); @@ -60,6 +60,8 @@ async function main() { default: console.error('unknown command:', command); + + // eslint-disable-next-line no-process-exit process.exit(1); } } diff --git a/test/benchmarks.test.ts b/test/benchmarks.test.ts index ce5c5a977..29e7274a5 100644 --- a/test/benchmarks.test.ts +++ b/test/benchmarks.test.ts @@ -35,8 +35,8 @@ import '../cases/zod'; test('all cases must have been imported in tests', () => { const registeredCases = new Set(); - getRegisteredBenchmarks().forEach(([benchmarkId, benchmarkCases]) => { - benchmarkCases.forEach(b => { + getRegisteredBenchmarks().forEach(nameBenchmarkPair => { + nameBenchmarkPair[1].forEach(b => { registeredCases.add(b.moduleName); }); }); From d58be44012248d5a856b143a0505e1c394ce75c7 Mon Sep 17 00:00:00 2001 From: Erik Soehnel Date: Sun, 22 May 2022 19:36:15 +0200 Subject: [PATCH 4/5] add info to readme about the change --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 84b0072ae..80b5e144d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # 📊 Benchmark Comparison of Packages with Runtime Validation and TypeScript Support +** ⚡⚠ Benchmark results have changed after switching to isolated node processes for each benchmarked package, see [#864](https://github.com/moltar/typescript-runtime-type-benchmarks/issues/864) ⚠⚡ ** + ## Benchmark Results [![Fastest Packages - click to view details](docs/results/preview.svg)](https://moltar.github.io/typescript-runtime-type-benchmarks) From ea914ab58672c8d778a591b8bf58a57bf11d1fa6 Mon Sep 17 00:00:00 2001 From: Erik Soehnel Date: Sun, 22 May 2022 19:59:23 +0200 Subject: [PATCH 5/5] more emphasis to readme info message --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 80b5e144d..2b9a787be 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # 📊 Benchmark Comparison of Packages with Runtime Validation and TypeScript Support -** ⚡⚠ Benchmark results have changed after switching to isolated node processes for each benchmarked package, see [#864](https://github.com/moltar/typescript-runtime-type-benchmarks/issues/864) ⚠⚡ ** +- - - - +**⚡⚠ Benchmark results have changed after switching to isolated node processes for each benchmarked package, see [#864](https://github.com/moltar/typescript-runtime-type-benchmarks/issues/864) ⚠⚡** +- - - - ## Benchmark Results