Reduce test startup time when running a subset of tests

Change the way that the `FILTER` option works in `make test FILTER=<pattern>`
(or equivalently the `--grep` option if using `gulp test --grep
<pattern>`) to control which test files are executed rather than
matching test descriptions.

This enables the test configuration function in `src/karma.config.js` to
limit the test bundle to only the matching files and their dependencies.
This can make the initial bundling much faster and bundle updates (when
using `make servetests`) somewhat faster.

On my system, `make test FILTER=menu-item` takes ~40s on master but only
~7s on this branch.

As before, the filename filter can be combined with the `.only` modifier
on `description` or `it` calls to do more fine-grained filtering of
tests within matching files.

 - Change `--grep` option to gulp to filter test files as part of the
   test bundling process instead of setting the `grep` mocha option

 - Make `make servetests` accept a `FILTER` argument for consistency
   with `make test`

 - Simplify code in gulpfile.js for launching Karma

 - Update documentation for `make test`

Makefile

@@ -29,7 +29,11 @@ endif
 
 .PHONY: servetests
 servetests: node_modules/.uptodate
+ifdef FILTER
+	node_modules/.bin/gulp test-watch --grep $(FILTER)
+else
 	node_modules/.bin/gulp test-watch
+endif
 
 .PHONY: lint
 lint: node_modules/.uptodate

docs/developers/developing.rst

@@ -110,8 +110,7 @@ Hypothesis uses Karma and mocha for testing. To run all the tests once, run:
    make test
 
 You can filter the tests which are run by running ``make test FILTER=<pattern>``.
-See the documentation for Mocha's
-`grep <https://mochajs.org/#g---grep-pattern>`_ option.
+Only test files matching ``<pattern>`` will be executed.
 
 To run tests and automatically re-run them whenever any source files change, run:
 

gulpfile.js

@@ -36,15 +36,12 @@ let liveReloadChangedFiles = [];
 
 function parseCommandLine() {
   commander
-    // Test configuration.
-    // See https://github.com/karma-runner/karma-mocha#configuration
-    .option('--grep [pattern]', 'Run only tests matching a given pattern')
+    .option(
+      '--grep [pattern]',
+      'Run only tests where filename matches a pattern'
+    )
     .parse(process.argv);
 
-  if (commander.grep) {
-    gulpUtil.log(`Running tests matching pattern /${commander.grep}/`);
-  }
-
   return {
     grep: commander.grep,
   };
@@ -384,37 +381,20 @@ gulp.task(
   )
 );
 
-function runKarma(baseConfig, opts, done) {
-  // See https://github.com/karma-runner/karma-mocha#configuration
-  const cliOpts = {
-    client: {
-      mocha: {
-        grep: taskArgs.grep,
-      },
-    },
-  };
-
+function runKarma({ singleRun }, done) {
   const karma = require('karma');
   new karma.Server(
-    Object.assign(
-      {},
     {
-        configFile: path.resolve(__dirname, baseConfig),
+      configFile: path.resolve(__dirname, './src/karma.config.js'),
+      grep: taskArgs.grep,
+      singleRun,
     },
-      cliOpts,
-      opts
-    ),
     done
   ).start();
 }
 
-gulp.task('test', function(callback) {
-  runKarma('./src/karma.config.js', { singleRun: true }, callback);
-});
-
-gulp.task('test-watch', function(callback) {
-  runKarma('./src/karma.config.js', {}, callback);
-});
+gulp.task('test', done => runKarma({ singleRun: true }, done));
+gulp.task('test-watch', done => runKarma({ singleRun: false }, done));
 
 gulp.task(
   'upload-sourcemaps',

src/karma.config.js

@@ -6,6 +6,7 @@
 
 const path = require('path');
 const envify = require('loose-envify/custom');
+const glob = require('glob');
 
 let chromeFlags = [];
 process.env.CHROME_BIN = require('puppeteer').executablePath();
@@ -50,6 +51,22 @@ if (process.env.RUNNING_IN_DOCKER) {
 }
 
 module.exports = function(config) {
+  let testFiles = [
+    'annotator/**/*-test.coffee',
+    '**/test/*-test.js',
+    '**/integration/*-test.js',
+  ];
+
+  if (config.grep) {
+    const allFiles = testFiles
+      .map(pattern => glob.sync(pattern, { cwd: __dirname }))
+      .flat();
+    testFiles = allFiles.filter(path => path.match(config.grep));
+
+    // eslint-disable-next-line no-console
+    console.log(`Running tests matching pattern "${config.grep}": `, testFiles);
+  }
+
   config.set({
     // base path that will be used to resolve all patterns (eg. files, exclude)
     basePath: './',
@@ -66,31 +83,13 @@ module.exports = function(config) {
       // Empty HTML file to assist with some tests
       { pattern: './annotator/test/empty.html', watched: false },
 
-      // Karma watching is disabled for these files because they are
-      // bundled with karma-browserify which handles watching itself via
-      // watchify
-
-      // Unit tests
-      {
-        pattern: 'annotator/**/*-test.coffee',
-        watched: false,
-        included: true,
-        served: true,
-      },
-      {
-        pattern: '**/test/*-test.js',
-        watched: false,
-        included: true,
-        served: true,
-      },
+      // Test modules.
+      ...testFiles.map(pattern => ({
+        pattern,
 
-      // Integration tests
-      {
-        pattern: '**/integration/*-test.js',
+        // Disable watching because karma-browserify handles this.
         watched: false,
-        included: true,
-        served: true,
-      },
+      })),
     ],
 
     // list of files to exclude