benchmark: introduce benchmark combination filtering

mscdex · juanarbol · commit 6468f30d0dd7 · 2023-03-04T23:03:29.000-05:00
PR-URL: #45735 Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: Daeyeon Jeong <daeyeon.dev@gmail.com> Reviewed-By: Luigi Pinca <luigipinca@gmail.com> Reviewed-By: Ruben Bridgewater <ruben@bridgewater.de> Reviewed-By: Rafael Gonzaga <rafael.nunu@hotmail.com>
diff --git a/benchmark/common.js b/benchmark/common.js
@@ -3,6 +3,10 @@
 const child_process = require('child_process');
 const http_benchmarkers = require('./_http-benchmarkers.js');
 
+function allow() {
+  return true;
+}
+
 class Benchmark {
   constructor(fn, configs, options = {}) {
     // Used to make sure a benchmark only start a timer once
@@ -31,9 +35,17 @@ class Benchmark {
       this.flags = this.flags.concat(options.flags);
     }
 
+    if (typeof options.combinationFilter === 'function')
+      this.combinationFilter = options.combinationFilter;
+    else
+      this.combinationFilter = allow;
+
     // The configuration list as a queue of jobs
     this.queue = this._queue(this.options);
 
+    if (this.queue.length === 0)
+      return;
+
     // The configuration of the current job, head of the queue
     this.config = this.queue[0];
 
@@ -108,6 +120,7 @@ class Benchmark {
   _queue(options) {
     const queue = [];
     const keys = Object.keys(options);
+    const { combinationFilter } = this;
 
     // Perform a depth-first walk through all options to generate a
     // configuration list that contains all combinations.
@@ -131,7 +144,15 @@ class Benchmark {
         if (keyIndex + 1 < keys.length) {
           recursive(keyIndex + 1, currConfig);
         } else {
-          queue.push(currConfig);
+          // Check if we should allow the current combination
+          const allowed = combinationFilter({ ...currConfig });
+          if (typeof allowed !== 'boolean') {
+            throw new TypeError(
+              'Combination filter must always return a boolean'
+            );
+          }
+          if (allowed)
+            queue.push(currConfig);
         }
       }
     }
diff --git a/doc/contributing/writing-and-running-benchmarks.md b/doc/contributing/writing-and-running-benchmarks.md
@@ -450,8 +450,12 @@ The arguments of `createBenchmark` are:
   possible combinations of these parameters, unless specified otherwise.
   Each configuration is a property with an array of possible values.
   The configuration values can only be strings or numbers.
-* `options` {Object} The benchmark options. At the moment only the `flags`
-  option for specifying command line flags is supported.
+* `options` {Object} The benchmark options. Supported options:
+  * `flags` {Array} Contains node-specific command line flags to pass to
+    the child process.
+  * `combinationFilter` {Function} Has a single parameter which is an object
+    containing a combination of benchmark parameters. It should return `true`
+    or `false` to indicate whether the combination should be included or not.
 
 `createBenchmark` returns a `bench` object, which is used for timing
 the runtime of the benchmark. Run `bench.start()` after the initialization
