Merge pull request #798 from hypothesis/qa-release

Deploy QA release of client as part of master branch builds

Jenkinsfile

@@ -12,6 +12,9 @@ node {
     // Git branch which releases are deployed from.
     releaseFromBranch = "master"
 
+    // S3 bucket where the embedded client is served from.
+    s3Bucket = "cdn.hypothes.is"
+
     // Pre-release suffix added to new package version number when deploying,
     // eg. "testing".
     //
@@ -31,6 +34,11 @@ node {
       returnStdout: true
     ).trim()
 
+    lastCommitHash = sh (
+      script: 'git show HEAD --no-patch --format="%h"',
+      returnStdout: true
+    ).trim()
+
     if (lastCommitAuthor == "jenkins-hypothesis") {
         echo "Skipping build of automated commit created by Jenkins"
         return
@@ -65,6 +73,46 @@ node {
         return
     }
 
+    milestone()
+    stage('Publish to QA') {
+        qaVersion = pkgVersion + "-${lastCommitHash}"
+        nodeEnv.inside("-e HOME=${workspace}") {
+            withCredentials([
+                string(credentialsId: 'npm-token', variable: 'NPM_TOKEN'),
+                usernamePassword(credentialsId: 'github-jenkins-user',
+                                 passwordVariable: 'GITHUB_TOKEN_NOT_USED',
+                                 usernameVariable: 'GITHUB_USERNAME'),
+                [$class: 'AmazonWebServicesCredentialsBinding', credentialsId: 's3-cdn']
+                ]) {
+                sh """
+                git config --replace-all user.email ${env.GITHUB_USERNAME}@hypothes.is
+                git config --replace-all user.name ${env.GITHUB_USERNAME}
+                """
+
+                // Build a prerelease version of the client, configured to load
+                // the sidebar from the qa h deployment.
+                sh """
+                export SIDEBAR_APP_URL=https://qa.hypothes.is/app.html
+                yarn version --no-git-tag-version --new-version ${qaVersion}
+                """
+
+                // Deploy to S3, so the package can be served by
+                // https://qa.hypothes.is/embed.js.
+                //
+                // If we decide to build a QA browser extension using the QA
+                // client in future then we will need to deploy to npm as well.
+                sh """
+                export AWS_ACCESS_KEY_ID=${env.AWS_ACCESS_KEY_ID}
+                export AWS_SECRET_ACCESS_KEY=${env.AWS_SECRET_ACCESS_KEY}
+                scripts/deploy-to-s3.js --bucket ${s3Bucket} --tag qa --no-cache-entry
+                """
+            }
+        }
+
+        // Revert back to the pre-QA commit.
+        sh "git checkout ${lastCommitHash}"
+    }
+
     milestone()
     stage('Publish') {
         input(message: "Publish new client release?")
@@ -82,15 +130,17 @@ node {
                 string(credentialsId: 'npm-token', variable: 'NPM_TOKEN'),
                 usernamePassword(credentialsId: 'github-jenkins-user',
                                   passwordVariable: 'GITHUB_TOKEN',
-                                  usernameVariable: 'GITHUB_USERNAME')]) {
+                                  usernameVariable: 'GITHUB_USERNAME'),
+                [$class: 'AmazonWebServicesCredentialsBinding', credentialsId: 's3-cdn']
+                ]) {
 
                 // Configure commit author for version bump commit and auth credentials
                 // for pushing tag to GitHub.
                 //
                 // See https://git-scm.com/docs/git-credential-store
                 sh """
-                git config user.email ${env.GITHUB_USERNAME}@hypothes.is
-                git config user.name ${env.GITHUB_USERNAME}
+                git config --replace-all user.email ${env.GITHUB_USERNAME}@hypothes.is
+                git config --replace-all user.name ${env.GITHUB_USERNAME}
                 git config credential.helper store
                 echo https://${env.GITHUB_USERNAME}:${env.GITHUB_TOKEN}@github.com >> \$HOME/.git-credentials
                 """
@@ -113,16 +163,15 @@ node {
                 sh "echo '//registry.npmjs.org/:_authToken=${env.NPM_TOKEN}' >> \$HOME/.npmrc"
                 sh "npm publish --tag ${npmTag}"
                 sh "scripts/wait-for-npm-release.sh"
-            }
-        }
-
-        echo "Uploading package ${pkgName} v${newPkgVersion} to CDN"
 
-        // Upload the contents of the package to an S3 bucket, which it
-        // will then be served from.
-        docker.image('nickstenning/s3-npm-publish')
-              .withRun('', "${pkgName}@${newPkgVersion} s3://cdn.hypothes.is") { c ->
-                sh "docker logs --follow ${c.id}"
+                // Deploy the client to cdn.hypothes.is, where the embedded
+                // client is served from by https://hypothes.is/embed.js.
+                sh """
+                export AWS_ACCESS_KEY_ID=${env.AWS_ACCESS_KEY_ID}
+                export AWS_SECRET_ACCESS_KEY=${env.AWS_SECRET_ACCESS_KEY}
+                scripts/deploy-to-s3.js --bucket ${s3Bucket}
+                """
+            }
         }
     }
 }

package.json

@@ -16,6 +16,7 @@
     "angulartics": "0.17.2",
     "autofill-event": "0.0.1",
     "autoprefixer": "^6.0.3",
+    "aws-sdk": "^2.345.0",
     "babel-preset-es2015": "^6.24.0",
     "babelify": "^7.3.0",
     "browserify": "^13.0.0",
@@ -80,6 +81,7 @@
     "mocha": "^2.4.5",
     "ng-tags-input": "^3.1.1",
     "node-uuid": "^1.4.3",
+    "npm-packlist": "^1.1.12",
     "postcss": "^5.0.6",
     "postcss-url": "^5.1.1",
     "proxyquire": "^1.7.10",
@@ -146,7 +148,6 @@
     "lint": "eslint .",
     "test": "gulp test",
     "report-coverage": "codecov -f coverage/coverage-final.json",
-    "preversion": "./scripts/preversion.sh",
     "version": "make clean all && ./scripts/update-changelog.js && git add CHANGELOG.md",
     "postversion": "./scripts/postversion.sh",
     "prepublish": "yarn run build"

scripts/deploy-to-s3.js

+#!/usr/bin/env node
+
+'use strict';
+
+const fs = require('fs');
+const { extname } = require('path');
+
+const commander = require('commander');
+const packlist = require('npm-packlist');
+const AWS = require('aws-sdk');
+
+/**
+ * File extension / mime type associations for file types we actually use.
+ */
+const MIME_TYPES = {
+  '.css': 'text/css',
+  '.md': 'text/markdown',
+  '.js': 'application/javascript',
+  '.json': 'application/json',
+  '.map': 'application/octet-stream',
+  '.svg': 'image/svg+xml',
+  '.woff': 'font/woff',
+  '.woff2': 'font/woff2',
+};
+
+/**
+ * Return the `Content-Type` header value for a given file.
+ */
+function contentTypeFromFilename(path) {
+  const extension = extname(path);
+  if (!extension) {
+    // Fallback if no extension.
+    return 'text/plain';
+  }
+
+  if (MIME_TYPES[extension]) {
+    return MIME_TYPES[extension];
+  }
+  throw new Error(`Unable to look up Content-Type for ${path}`);
+}
+
+class S3Uploader {
+  constructor(bucket) {
+    this.s3 = new AWS.S3();
+    this.bucket = bucket;
+    this.region = null;
+  }
+
+  async upload(destPath, srcFile, { cacheControl }) {
+    if (!this.region) {
+      // Find out where the S3 bucket is.
+      const regionResult = await this.s3.getBucketLocation({
+        Bucket: this.bucket,
+      }).promise();
+      this.region = regionResult.LocationConstraint;
+      this.s3 = new AWS.S3({ region: this.region });
+    }
+
+    const fileContent = fs.readFileSync(srcFile);
+    const params = {
+      ACL: 'public-read',
+      Bucket: this.bucket,
+      Key: destPath,
+      Body: fileContent,
+      CacheControl: cacheControl,
+      ContentType: contentTypeFromFilename(srcFile),
+    };
+    return this.s3.putObject(params).promise();
+  }
+}
+
+/**
+ * Uploads the content of the npm package in the current directory to S3.
+ *
+ * Creates the following keys in the S3 bucket:
+ *
+ * - `<package name>/<version>/<file>` for each `<file>` in the package.
+ * - `<package name>@<version>` is a copy of the entry point for the package.
+ * - `<package name>@<tag>` is a copy of the entry point for the package if
+ *   `tag` is non-empty.
+ * - `<package name>` is a copy of the entry point if `tag` is empty.
+ *
+ * Files are made publicly readable. Keys containing the full package version
+ * are set to be cached indefinitely by the browser. Keys that are pointers
+ * to the current version only have a short cache lifetime.
+ */
+async function uploadPackageToS3(bucket, options) {
+  // Get list of files that are distributed with the package, respecting
+  // the `files` field in `package.json`, `.npmignore` etc.
+  const files = await packlist({ path: '.' });
+
+  // Get name, version and main module of the package.
+  const packageJson = require(`${process.cwd()}/package.json`);
+  const packageName = packageJson.name;
+  const version = packageJson.version;
+  const entryPoint = packageJson.main;
+
+  // Configure uploads to S3.
+  const uploader = new S3Uploader(bucket);
+  const cacheForever = 'public, max-age=315360000, immutable';
+
+  // Upload all package files to `$PACKAGE_NAME/$VERSION`.
+  const uploads = files.map(file =>
+    uploader.upload(
+      `${packageName}/${version}/${file}`,
+      file,
+      { cacheControl: cacheForever },
+    )
+  );
+  await Promise.all(uploads);
+
+  // Upload a copy of the entry-point to `$PACKAGE_NAME@$VERSION`.
+  await uploader.upload(
+    `${packageName}@${version}`,
+    entryPoint,
+    { cacheControl: cacheForever },
+  );
+
+  // Upload a copy of the entry-point to `$PACKAGE_NAME` or `$PACKAGE_NAME@$TAG`.
+  // This enables creating URLs that always point to the current version of
+  // the package.
+  let aliasCacheControl;
+  if (options.cacheEntry) {
+    // nb. When deploying to cdn.hypothes.is, the max-age seen by the browser is
+    // the higher of the value here and CloudFlare's "Browser Cache TTL"
+    // setting.
+    aliasCacheControl = 'public, max-age=1800, must-revalidate';
+  } else {
+    aliasCacheControl = 'no-cache';
+  }
+
+  let aliasPath;
+  if (!options.tag) {
+    aliasPath = `${packageName}`;
+  } else {
+    aliasPath = `${packageName}@${options.tag}`;
+  }
+  await uploader.upload(aliasPath, entryPoint, {
+    cacheControl: aliasCacheControl,
+  });
+}
+
+commander
+  .option('--bucket [bucket]', 'S3 bucket name')
+  .option('--tag [tag]', 'Version tag')
+  .option('--no-cache-entry', 'Prevent CDN/browser caching of entry point')
+  .parse(process.argv);
+
+const options = {
+  tag: commander.tag,
+  cacheEntry: commander.cacheEntry,
+};
+
+uploadPackageToS3(commander.bucket, options)
+  .catch(err => {
+    console.error('Failed to upload S3 package', err);
+    process.exit(1);
+  });

scripts/postversion.sh

@@ -4,6 +4,13 @@ set -eu
 
 cd "$(dirname "$0")"
 
+# Skip GitHub release creation for QA releases.
+is_prerelease=$(node -p "require('./package.json').version.includes('-')")
+if [ $is_prerelease = "true" ]; then
+  echo "Skipping GitHub release creation for pre-release"
+  exit 0
+fi
+
 # nb. The remote refname is fully qualified because this script is run in a CI
 # environment where not all heads may have been fetched.
 git push https://github.com/hypothesis/client.git HEAD:refs/heads/$BRANCH_NAME --follow-tags

scripts/preversion.sh

-#!/bin/sh
-
-set -eu
-
-# Check that tag creation works.
-# The tag is not currently signed because Jenkins does not have this set up.
-git tag --message "Dummy Tag" dummy-tag
-git tag --delete dummy-tag > /dev/null
-
-# Check GitHub API access token
-CLIENT_INFO_URL=https://api.github.com/repos/hypothesis/client
-REPO_TMPFILE=/tmp/client-repo.json
-curl -s -H "Authorization: Bearer $GITHUB_TOKEN" $CLIENT_INFO_URL > $REPO_TMPFILE
-CAN_PUSH=$(node -p -e "perms = require('$REPO_TMPFILE').permissions, perms && perms.push")
-
-if [ "$CAN_PUSH" != "true" ]; then
-  echo "Cannot push to GitHub using the access token '$GITHUB_TOKEN'"
-  exit 1
-fi

scripts/update-changelog.js

@@ -48,4 +48,9 @@ ${changelist}
   fs.writeFileSync(changelogPath, updatedChangelog);
 }
 
+if (pkg.version.includes('-')) {
+  console.warn('Skipping changelog update for pre-release version');
+  return;
+}
+
 updateChangeLog();

scripts/wait-for-npm-release.sh

@@ -5,12 +5,22 @@
 #
 # This script is needed because a new release of a package is not always
 # immediately available after "npm publish" returns.
+#
+# Usage: wait-for-npm-release.sh [<dist-tag>]
+#
+# <dist-tag> defaults to "latest".
+
+if [ -z "$1" ]; then
+  dist_tag=latest
+else
+  dist_tag=$1
+fi
 
 expected_version=$(node -p "require('./package.json').version")
 while [ true ]
 do
-  released_version=$(npm show hypothesis dist-tags.latest)
-  if [ $released_version = $expected_version ]; then
+  released_version=$(npm show hypothesis dist-tags.$dist_tag)
+  if [ "$released_version" = "$expected_version" ]; then
     break
   fi
 

yarn.lock

@@ -495,6 +495,21 @@ autoprefixer@^6.0.3:
     postcss "^5.2.16"
     postcss-value-parser "^3.2.3"
 
+aws-sdk@^2.345.0:
+  version "2.345.0"
+  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.345.0.tgz#857cecf0ed1cc0a3969d3d2239da237993db4b87"
+  integrity sha512-t5/8nmu7m+jQUq//ssP/dWKFX0sDEYlxvTnFUJhtRmr7UNzssHgw4bYkK/poMpmpM8otgw80l2M3bAdNrOlMYw==
+  dependencies:
+    buffer "4.9.1"
+    events "1.1.1"
+    ieee754 "1.1.8"
+    jmespath "0.15.0"
+    querystring "0.2.0"
+    sax "1.2.1"
+    url "0.10.3"
+    uuid "3.1.0"
+    xml2js "0.4.19"
+
 aws-sign2@~0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.6.0.tgz#14342dd38dbcc94d0e5b87d763cd63612c0e794f"
@@ -1417,7 +1432,7 @@ buffer-xor@^1.0.3:
   resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9"
   integrity sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=
 
-buffer@^4.1.0:
+buffer@4.9.1, buffer@^4.1.0:
   version "4.9.1"
   resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298"
   integrity sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=
@@ -2855,16 +2870,16 @@ eventemitter3@^3.0.0:
   resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-3.1.0.tgz#090b4d6cdbd645ed10bf750d4b5407942d7ba163"
   integrity sha512-ivIvhpq/Y0uSjcHDcOIccjmYjGLcP09MFGE7ysAwkAvkXfpZlC985pH2/ui64DKazbTW/4kN3yqozUxlXzI6cA==
 
+events@1.1.1, events@~1.1.0:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
+  integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=
+
 events@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/events/-/events-2.0.0.tgz#cbbb56bf3ab1ac18d71c43bb32c86255062769f2"
   integrity sha512-r/M5YkNg9zwI8QbSf7tsDWWJvO3PGwZXyG7GpFAxtMASnHL2eblFd7iHiGPtyGKKFPZ59S63NeX10Ws6WqGDcg==
 
-events@~1.1.0:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
-  integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=
-
 evp_bytestokey@^1.0.0, evp_bytestokey@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz#7fcbdb198dc71959432efe13842684e0525acb02"
@@ -4183,6 +4198,11 @@ iconv-lite@^0.4.4:
   dependencies:
     safer-buffer "^2.1.0"
 
+ieee754@1.1.8:
+  version "1.1.8"
+  resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.8.tgz#be33d40ac10ef1926701f6f08a2d86fbfd1ad3e4"
+  integrity sha1-vjPUCsEO8ZJnAfbwii2G+/0a0+Q=
+
 ieee754@^1.1.4:
   version "1.1.11"
   resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.11.tgz#c16384ffe00f5b7835824e67b6f2bd44a5229455"
@@ -4710,6 +4730,11 @@ jade@0.26.3:
     commander "0.6.1"
     mkdirp "0.3.0"
 
+jmespath@0.15.0:
+  version "0.15.0"
+  resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217"
+  integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=
+
 jquery@^3.2.1:
   version "3.3.1"
   resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.3.1.tgz#958ce29e81c9790f31be7792df5d4d95fc57fbca"
@@ -5834,6 +5859,14 @@ npm-bundled@^1.0.1:
   resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.0.3.tgz#7e71703d973af3370a9591bafe3a63aca0be2308"
   integrity sha512-ByQ3oJ/5ETLyglU2+8dBObvhfWXX8dtPZDMePCahptliFX2iIuhyEszyFk401PZUNQH20vvdW5MLjJxkwU80Ow==
 
+npm-packlist@^1.1.12:
+  version "1.1.12"
+  resolved "https://registry.yarnpkg.com/npm-packlist/-/npm-packlist-1.1.12.tgz#22bde2ebc12e72ca482abd67afc51eb49377243a"
+  integrity sha512-WJKFOVMeAlsU/pjXuqVdzU0WfgtIBCupkEVwn+1Y0ERAbUfWw8R4GjgVbaKnUjRoD2FoQbHOCbOyT5Mbs9Lw4g==
+  dependencies:
+    ignore-walk "^3.0.1"
+    npm-bundled "^1.0.1"
+
 npm-packlist@^1.1.6:
   version "1.1.10"
   resolved "https://registry.yarnpkg.com/npm-packlist/-/npm-packlist-1.1.10.tgz#1039db9e985727e464df066f4cf0ab6ef85c398a"
@@ -7116,7 +7149,12 @@ sass-graph@^2.2.4:
     scss-tokenizer "^0.2.3"
     yargs "^7.0.0"
 
-sax@^1.2.4:
+sax@1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
+  integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o=
+
+sax@>=0.6.0, sax@^1.2.4:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
   integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
@@ -8214,6 +8252,14 @@ url-template@^2.0.8:
   resolved "https://registry.yarnpkg.com/url-template/-/url-template-2.0.8.tgz#fc565a3cccbff7730c775f5641f9555791439f21"
   integrity sha1-/FZaPMy/93MMd19WQflVV5FDnyE=
 
+url@0.10.3:
+  version "0.10.3"
+  resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64"
+  integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=
+  dependencies:
+    punycode "1.3.2"
+    querystring "0.2.0"
+
 url@~0.11.0:
   version "0.11.0"
   resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1"
@@ -8271,6 +8317,11 @@ utils-merge@1.0.1:
   resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
   integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=
 
+uuid@3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04"
+  integrity sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==
+
 uuid@^3.0.0, uuid@^3.1.0:
   version "3.2.1"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.2.1.tgz#12c528bb9d58d0b9265d9a2f6f0fe8be17ff1f14"
@@ -8486,6 +8537,19 @@ xml-char-classes@^1.0.0:
   resolved "https://registry.yarnpkg.com/xml-char-classes/-/xml-char-classes-1.0.0.tgz#64657848a20ffc5df583a42ad8a277b4512bbc4d"
   integrity sha1-ZGV4SKIP/F31g6Qq2KJ3tFErvE0=
 
+xml2js@0.4.19:
+  version "0.4.19"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
+  integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==
+  dependencies:
+    sax ">=0.6.0"
+    xmlbuilder "~9.0.1"
+
+xmlbuilder@~9.0.1:
+  version "9.0.7"
+  resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"
+  integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=
+
 xmldom@^0.1.19:
   version "0.1.27"
   resolved "https://registry.yarnpkg.com/xmldom/-/xmldom-0.1.27.tgz#d501f97b3bdb403af8ef9ecc20573187aadac0e9"