Merge pull request #3028 from hypothesis/chrome-ext-sourcemaps

Implement tools to enable Chrome extension sourcemaps

.jshintrc

@@ -9,6 +9,8 @@
   "strict": "global",
   "undef": true,
   "unused": true,
+  "esversion": 6,
+  "esnext": true,
   "globals": {
     "chrome": false,
     "h": false,

gulpfile.js

@@ -17,6 +17,7 @@ var sourcemaps = require('gulp-sourcemaps');
 var manifest = require('./scripts/gulp/manifest');
 var createBundle = require('./scripts/gulp/create-bundle');
 var vendorBundles = require('./scripts/gulp/vendor-bundles');
+var uploadToSentry = require('./scripts/gulp/upload-to-sentry');
 
 var IS_PRODUCTION_BUILD = process.env.NODE_ENV === 'production';
 var SCRIPT_DIR = 'build/scripts';
@@ -28,6 +29,13 @@ function isSASSFile(file) {
   return file.path.match(/\.scss$/);
 }
 
+function getEnv(key) {
+  if (!process.env.hasOwnProperty(key)) {
+    throw new Error(`Environment variable ${key} is not set`);
+  }
+  return process.env[key];
+}
+
 /** A list of all modules included in vendor bundles. */
 var vendorModules = Object.keys(vendorBundles.bundles)
   .reduce(function (deps, key) {
@@ -246,3 +254,13 @@ gulp.task('test-watch-extension', function (callback) {
     configFile: __dirname + '/h/browser/chrome/karma.config.js',
   }, callback).start();
 });
+
+gulp.task('upload-sourcemaps', function () {
+  gulp.src(['build/scripts/*.js', 'build/scripts/*.map'])
+    .pipe(uploadToSentry({
+      apiKey: getEnv('SENTRY_API_KEY'),
+      release: getEnv('SENTRY_RELEASE_VERSION'),
+      organization: getEnv('SENTRY_ORGANIZATION'),
+      project: getEnv('SENTRY_PROJECT'),
+    }));
+});

h/static/scripts/app.coffee

 require('./polyfills')
 
-# initialize Raven. This is required at the top of this file
+# Initialize Raven. This is required at the top of this file
 # so that it happens early in the app's startup flow
 settings = require('./settings')(document)
 if settings.raven
-  require('./raven').init(settings.raven)
-
+  raven = require('./raven')
+  raven.init(settings.raven)
 
 angular = require('angular')
 
@@ -13,6 +13,12 @@ angular = require('angular')
 # it must be require'd after angular is first require'd
 require('autofill-event')
 
+# Setup Angular integration for Raven
+if settings.raven
+  raven.angularModule(angular)
+else
+  angular.module('ngRaven', [])
+
 streamer = require('./streamer')
 
 resolve =
@@ -89,7 +95,7 @@ module.exports = angular.module('h', [
   ['ui.bootstrap', require('./vendor/ui-bootstrap-custom-tpls-0.13.4')][0]
 
   # Local addons
-  require('./raven').angularModule().name
+  'ngRaven'
 ])
 
 .controller('AppController', require('./app-controller'))

h/static/scripts/raven.js

+'use strict';
+
 /**
  * This module configures Raven for reporting crashes
  * to Sentry.
@@ -10,17 +12,73 @@
  * as a dependency.
  */
 
-require('core-js/fn/object/assign')
+require('core-js/fn/object/assign');
+
+var Raven = require('raven-js');
+
+// This is only used in apps where Angular is used,
+// but is required globally due to
+// https://github.com/thlorenz/proxyquireify/issues/40
+//
+// Fortunately it does not pull in Angular as a dependency but returns
+// a function that takes it as an input argument.
+var angularPlugin = require('raven-js/plugins/angular');
+
+/**
+ * Returns the input URL if it is an HTTP URL or the filename part of the URL
+ * otherwise.
+ *
+ * @param {string} url - The script URL associated with an exception stack
+ *                       frame.
+ */
+function convertLocalURLsToFilenames(url) {
+  if (!url) {
+    return url;
+  }
+
+  if (url.match(/https?:/)) {
+    return url;
+  }
 
-var Raven = require('raven-js')
+  // Strip the query string (which is used as a cache buster)
+  // and extract the filename from the URL
+  return url.replace(/\?.*/,'').split('/').slice(-1)[0];
+}
 
-var enabled = false;
+/**
+ * Return a transformed version of @p data with local URLs replaced
+ * with filenames.
+ *
+ * In environments where the client is served from a local URL,
+ * eg. chrome-extension://<ID>/scripts/bundle.js, the script URL
+ * and the sourcemap it references will not be accessible to Sentry.
+ *
+ * Therefore on the client we replace references to such URLs with just
+ * the filename part and then as part of the release process, upload both
+ * the source file and the source map to Sentry.
+ *
+ * Using just the filename allows us to upload a single set of source files
+ * and sourcemaps for a release though a given release of H might be served
+ * from multiple actual URLs (eg. different browser extensions).
+ */
+function translateSourceURLs(data) {
+  try {
+    var frames = data.exception.values[0].stacktrace.frames;
+    frames.forEach(function (frame) {
+      frame.filename = convertLocalURLsToFilenames(frame.filename);
+    });
+    data.culprit = frames[0].filename;
+  } catch (err) {
+    console.warn('Failed to normalize error stack trace', err, data);
+  }
+  return data;
+}
 
 function init(config) {
   Raven.config(config.dsn, {
     release: config.release,
+    dataCallback: translateSourceURLs,
   }).install();
-  enabled = true;
   installUnhandledPromiseErrorHandler();
 }
 
@@ -35,18 +93,27 @@ function setUserInfo(info) {
 /**
  * Initializes and returns the Angular module which provides
  * a custom wrapper around Angular's $exceptionHandler service,
- * logging any exceptions passed to it using Sentry
+ * logging any exceptions passed to it using Sentry.
+ *
+ * This must be invoked _after_ Raven is configured using init().
  */
-function angularModule() {
-  if (enabled) {
-    if (window.angular) {
-      var angularPlugin = require('raven-js/plugins/angular');
-      angularPlugin(Raven, angular);
-    }
-  } else {
-    // define a stub module in environments where Raven is not enabled
-    angular.module('ngRaven', []);
-  }
+function angularModule(angular) {
+  var prevCallback = Raven._globalOptions.dataCallback;
+  angularPlugin(Raven, angular);
+
+  // Hack: Ensure that both our data callback and the one provided by
+  // the Angular plugin are run when submitting errors.
+  //
+  // The Angular plugin replaces any previously installed
+  // data callback with its own which does not in turn call the
+  // previously registered callback that we registered when calling
+  // Raven.config().
+  //
+  // See https://github.com/getsentry/raven-js/issues/522
+  var angularCallback = Raven._globalOptions.dataCallback;
+  Raven.setDataCallback(function (data) {
+    return angularCallback(prevCallback(data));
+  });
   return angular.module('ngRaven');
 }
 

h/static/scripts/test/raven-test.js

+'use strict';
+
 var proxyquire = require('proxyquire');
 
+function noCallThru(stub) {
+  return Object.assign(stub, {'@noCallThru':true});
+}
+
+function fakeExceptionData(scriptURL) {
+  return {
+    exception: {
+      values: [{
+        stacktrace: {
+          frames: [{
+            filename: scriptURL,
+          }]
+        }
+      }]
+    },
+    culprit: scriptURL,
+  };
+}
+
 describe('raven', function () {
+  // A stub for the callback that the Angular plugin installs with
+  // Raven.setDataCallback()
+  var fakeAngularTransformer;
+  var fakeAngularPlugin;
   var fakeRavenJS;
   var raven;
 
@@ -9,10 +34,26 @@ describe('raven', function () {
       config: sinon.stub().returns({
         install: sinon.stub(),
       }),
+
       captureException: sinon.stub(),
+
+      setDataCallback: function (callback) {
+        this._globalOptions.dataCallback = callback;
+      },
+
+      _globalOptions: {
+        dataCallback: undefined,
+      },
     };
+
+    fakeAngularTransformer = sinon.stub();
+    fakeAngularPlugin = sinon.spy(function (Raven) {
+      Raven.setDataCallback(fakeAngularTransformer);
+    });
+
     raven = proxyquire('../raven', {
-      'raven-js': fakeRavenJS,
+      'raven-js': noCallThru(fakeRavenJS),
+      'raven-js/plugins/angular': noCallThru(fakeAngularPlugin),
     });
   });
 
@@ -32,6 +73,36 @@ describe('raven', function () {
     });
   });
 
+  describe('pre-submission data transformation', function () {
+    var dataCallback;
+
+    beforeEach(function () {
+      raven.init({dsn: 'dsn', release: 'release'});
+      var configOpts = fakeRavenJS.config.args[0][1];
+      dataCallback = configOpts && configOpts.dataCallback;
+    });
+
+    it('installs a transformer', function () {
+      assert.ok(dataCallback);
+    });
+
+    it('replaces non-HTTP URLs with filenames', function () {
+      var scriptURL = 'chrome-extension://1234/public/bundle.js';
+      var transformed = dataCallback(fakeExceptionData(scriptURL));
+      assert.equal(transformed.culprit, 'bundle.js');
+      var transformedStack = transformed.exception.values[0].stacktrace;
+      assert.equal(transformedStack.frames[0].filename, 'bundle.js');
+    });
+
+    it('does not modify HTTP URLs', function () {
+      var scriptURL = 'https://hypothes.is/assets/scripts/bundle.js';
+      var transformed = dataCallback(fakeExceptionData(scriptURL));
+      assert.equal(transformed.culprit, scriptURL);
+      var transformedStack = transformed.exception.values[0].stacktrace;
+      assert.equal(transformedStack.frames[0].filename, scriptURL);
+    });
+  });
+
   describe('.report()', function () {
     it('extracts the message property from Error-like objects', function () {
       raven.report({message: 'An error'}, 'context');
@@ -53,4 +124,35 @@ describe('raven', function () {
       });
     });
   });
+
+  describe('.angularModule()', function () {
+    var angularStub;
+
+    beforeEach(function () {
+      angularStub = {
+        module: sinon.stub(),
+      };
+    });
+
+    it('installs the Angular plugin', function () {
+      raven.init('dsn');
+      raven.angularModule(angularStub);
+      assert.calledWith(fakeAngularPlugin, fakeRavenJS, angularStub);
+    });
+
+    it('installs the data transformers', function () {
+      raven.init('dsn');
+      var originalTransformer = sinon.stub();
+      fakeRavenJS._globalOptions.dataCallback = originalTransformer;
+      raven.angularModule(angularStub);
+      fakeRavenJS._globalOptions.dataCallback(
+        fakeExceptionData('app.bundle.js')
+      );
+
+      // Check that both our data transformer and the one provided by
+      // the Angular plugin were invoked
+      assert.called(originalTransformer);
+      assert.called(fakeAngularTransformer);
+    });
+  });
 });

package.json

@@ -80,6 +80,7 @@
     "proxyquire": "^1.6.0",
     "proxyquire-universal": "^1.0.8",
     "proxyquireify": "^3.0.0",
+    "request": "^2.69.0",
     "sinon": "1.16.1"
   },
   "engines": {

scripts/gulp/upload-to-sentry.js

+'use strict';
+
+var fs = require('fs');
+var gulpUtil = require('gulp-util');
+var path = require('path');
+var request = require('request');
+var through = require('through2');
+
+/**
+ * interface SentryOptions {
+ *   /// The Sentry API key
+ *   apiKey: string;
+ *   /// The release string for the release to create
+ *   release: string;
+ *   /// The organization slug to use when constructing the API URL
+ *   organization: string;
+ *   /// The project name slug to use when constructing the API URL
+ *   project: string;
+ * }
+ */
+
+/** Wrapper around request() that returns a Promise. */
+function httpRequest(opts) {
+  return new Promise(function (resolve, reject) {
+    request(opts, function (err, response, body) {
+      if (err) {
+        reject(err);
+      } else {
+        resolve({
+          status: response.statusCode,
+          body: body,
+        });
+      }
+    });
+  });
+}
+
+/**
+ * Upload a stream of Vinyl files as a Sentry release.
+ *
+ * This creates a new release in Sentry using the organization, project
+ * and release settings in @p opts and uploads the input stream of Vinyl
+ * files as artefacts for the release.
+ *
+ * @param {SentryOptions} opts
+ * @return {NodeJS.ReadWriteStream} - A stream into which Vinyl files from
+ *                                    gulp.src() etc. can be piped.
+ */
+module.exports = function uploadToSentry(opts) {
+
+  // A map of already-created release versions.
+  // Once the release has been successfully created, this is used
+  // to avoid creating it again.
+  var createdReleases = {};
+
+  return through.obj(function (file, enc, callback) {
+    enc = enc;
+
+    gulpUtil.log(`Uploading ${file.path} to Sentry`);
+
+    var sentryURL =
+      `https://app.getsentry.com/api/0/projects/${opts.organization}/${opts.project}/releases`;
+
+    var releaseCreated;
+    if (createdReleases[opts.release]) {
+      releaseCreated = Promise.resolve();
+    } else {
+      releaseCreated = httpRequest({
+          uri: `${sentryURL}/`,
+          method: 'POST',
+          auth: {
+            user: opts.apiKey,
+            password: '',
+          },
+          body: {
+            version: opts.release,
+          },
+          json: true,
+        }).then(function (result) {
+          if (result.status === 201 ||
+              (result.status === 400 &&
+               result.body.detail.match(/already exists/))) {
+            createdReleases[opts.release] = true;
+            return;
+          }
+        });
+    }
+
+    releaseCreated.then(function () {
+      return httpRequest({
+        uri: `${sentryURL}/${opts.release}/files/`,
+        method: 'POST',
+        auth: {
+          user: opts.apiKey,
+          password: '',
+        },
+        formData: {
+          file: fs.createReadStream(file.path),
+          name: path.basename(file.path),
+        },
+      });
+    }).then(function (result) {
+      if (result.status === 201) {
+        callback();
+        return;
+      }
+      var message =
+        `Uploading file failed: ${result.status}: ${result.body}`;
+      throw new Error(message);
+    }).catch(function (err) {
+      gulpUtil.log('Sentry upload failed: ', err);
+      throw err;
+    });
+  });
+};