build: payload github status (#4549)

* build: payload github status

* Show payload deta on PRs using the Github Statuses API.

* Update config variables

Author: devversion

tools/dashboard/functions/github/github-status.ts

@@ -4,7 +4,7 @@ const request = require('request');
 const {version, name} = require('../package.json');
 
 /** API token for the Github repository. Required to set the github status on commits and PRs. */
-const repoToken = config().repoToken;
+const repoToken = config().secret.github;
 
 /** Data that must be specified to set a Github PR status. */
 export type GithubStatusData = {

tools/dashboard/functions/jwt/verify-token.ts

@@ -2,10 +2,10 @@ import {verify} from 'jsonwebtoken';
 import {config} from 'firebase-functions';
 
 /** The JWT secret. This is used to validate JWT. */
-const jwtSecret = config().jwtSecret;
+const jwtSecret = config().secret.jwt;
 
 /** The repo slug. This is used to validate the JWT is sent from correct repo. */
-const repoSlug = config().repoSlug;
+const repoSlug = config().repo.slug;
 
 export function verifyToken(token: string): boolean {
   try {

tools/gulp/tasks/payload.ts

@@ -1,18 +1,20 @@
 import {task} from 'gulp';
 import {join} from 'path';
 import {statSync} from 'fs';
-import {spawnSync} from 'child_process';
-import {isTravisMasterBuild} from '../util/travis-ci';
-import {openFirebaseDashboardApp} from '../util/firebase';
+import {isTravisBuild, isTravisMasterBuild} from '../util/travis-ci';
 import {buildConfig} from '../packaging/build-config';
+import {openFirebaseDashboardApp, openFirebaseDashboardAppAsGuest} from '../util/firebase';
+
+// These imports lack of type definitions.
+const request = require('request');
 
 /** Path to the directory where all bundles are living. */
 const bundlesDir = join(buildConfig.outputDir, 'bundles');
 
 /** Task which runs test against the size of material. */
-task('payload', ['material:clean-build'], () => {
+task('payload', ['material:clean-build'], async () => {
 
-  let results = {
+  const results = {
     timestamp: Date.now(),
     // Material bundles
     material_umd: getBundleSize('material.umd.js'),
@@ -29,9 +31,23 @@ task('payload', ['material:clean-build'], () => {
   // Print the results to the console, so we can read it from the CI.
   console.log('Payload Results:', JSON.stringify(results, null, 2));
 
-  // Publish the results to firebase when it runs on Travis and not as a PR.
-  if (isTravisMasterBuild()) {
-    return publishResults(results);
+  if (isTravisBuild()) {
+    // Open a connection to Firebase. For PRs the connection will be established as a guest.
+    const firebaseApp = isTravisMasterBuild() ?
+        openFirebaseDashboardApp() :
+        openFirebaseDashboardAppAsGuest();
+    const database = firebaseApp.database();
+    const currentSha = process.env['TRAVIS_PULL_REQUEST_SHA'] || process.env['TRAVIS_COMMIT'];
+
+    // Upload the payload results and calculate the payload diff in parallel. Otherwise the
+    // payload task will take much more time inside of Travis builds.
+    await Promise.all([
+      uploadPayloadResults(database, currentSha, results),
+      calculatePayloadDiff(database, currentSha, results)
+    ]);
+
+    // Disconnect database connection because Firebase otherwise prevents Gulp from exiting.
+    firebaseApp.delete();
   }
 
 });
@@ -46,14 +62,81 @@ function getFilesize(filePath: string) {
   return statSync(filePath).size / 1000;
 }
 
-/** Publishes the given results to the firebase database. */
-function publishResults(results: any) {
-  const latestSha = spawnSync('git', ['rev-parse', 'HEAD']).stdout.toString().trim();
-  const dashboardApp = openFirebaseDashboardApp();
-  const database = dashboardApp.database();
+/**
+ * Calculates the difference between the last and current library payload.
+ * The results will be published as a commit status on Github.
+ */
+async function calculatePayloadDiff(database: any, currentSha: string, currentPayload: any) {
+  const authToken = process.env['FIREBASE_ACCESS_TOKEN'];
+
+  if (!authToken) {
+    console.error('Cannot calculate Payload diff because there is no "FIREBASE_ACCESS_TOKEN" ' +
+      'environment variable set.');
+    return;
+  }
+
+  const previousPayload = await getLastPayloadResults(database);
+
+  if (!previousPayload) {
+    console.warn('There are no previous payload results uploaded. Cannot calculate payload ' +
+      'difference for this job');
+    return;
+  }
+
+  // Calculate library sizes by combining the CDK and Material FESM 2015 bundles.
+  const previousSize = previousPayload.cdk_fesm_2015 + previousPayload.material_fesm_2015;
+  const currentSize = currentPayload.cdk_fesm_2015 + currentPayload.material_fesm_2015;
+  const deltaSize = currentSize - previousSize;
+
+  // Update the Github status of the current commit by sending a request to the dashboard
+  // firebase http trigger function.
+  await updateGithubStatus(currentSha, deltaSize, authToken);
+}
+
+/**
+ * Updates the Github status of a given commit by sending a request to a Firebase function of
+ * the dashboard. The function has access to the Github repository and can set status for PRs too.
+ */
+async function updateGithubStatus(commitSha: string, payloadDiff: number, authToken: string) {
+  const options = {
+    url: 'https://us-central1-material2-board.cloudfunctions.net/payloadGithubStatus',
+    headers: {
+      'User-Agent': 'Material2/PayloadTask',
+      'auth-token': authToken,
+      'commit-sha': commitSha,
+      'commit-payload-diff': payloadDiff
+    }
+  };
+
+  return new Promise((resolve, reject) => {
+    request(options, (err: any, response: any, body: string) => {
+      if (err) {
+        reject(`Dashboard Error ${err.toString()}`);
+      } else {
+        console.info('Dashboard Response:', JSON.parse(body).message);
+        resolve(response.statusCode);
+      }
+    });
+  });
+}
+
+/** Uploads the current payload results to the Dashboard database. */
+async function uploadPayloadResults(database: any, currentSha: string, currentPayload: any) {
+  if (isTravisMasterBuild()) {
+    await database.ref('payloads').child(currentSha).set(currentPayload);
+  }
+}
+
+/** Gets the last payload uploaded from previous Travis builds. */
+async function getLastPayloadResults(database: admin.database.Database) {
+  const snapshot = await database.ref('payloads')
+    .orderByChild('timestamp')
+    .limitToLast(1)
+    .once('value');
+
+  // The value of the DataSnapshot is an object with the SHA as a key. Later only the
+  // first value of the object will be returned because the SHA is unnecessary.
+  const results = snapshot.val();
 
-  // Write the results to the payloads object with the latest Git SHA as key.
-  return database.ref('payloads').child(latestSha).set(results)
-    .catch((err: any) => console.error(err))
-    .then(() => dashboardApp.delete());
+  return snapshot.hasChildren() ? results[Object.keys(results)[0]] : null;
 }

tools/gulp/util/firebase.ts

@@ -5,22 +5,30 @@ const cloudStorage = require('@google-cloud/storage');
 // Firebase configuration for the Screenshot project. Use the config from the screenshot functions.
 const screenshotFirebaseConfig = require('../../screenshot-test/functions/config.json');
 
-/** Opens a connection to the Firebase dashboard app. */
-export function openFirebaseDashboardApp() {
+/** Database URL of the dashboard firebase project.*/
+const dashboardDatabaseUrl = 'https://material2-board.firebaseio.com';
+
+/** Opens a connection to the Firebase dashboard app using a service account. */
+export function openFirebaseDashboardApp(asGuest = false) {
   // Initialize the Firebase application with firebaseAdmin credentials.
   // Credentials need to be for a Service Account, which can be created in the Firebase console.
   return firebaseAdmin.initializeApp({
+    databaseURL: dashboardDatabaseUrl,
     credential: firebaseAdmin.credential.cert({
       project_id: 'material2-board',
       client_email: '[email protected]',
       // In Travis CI the private key will be incorrect because the line-breaks are escaped.
       // The line-breaks need to persist in the service account private key.
       private_key: decode(process.env['MATERIAL2_BOARD_FIREBASE_SERVICE_KEY'])
     }),
-    databaseURL: 'https://material2-board.firebaseio.com'
   });
 }
 
+/** Opens a connection to the Firebase dashboard app with no authentication. */
+export function openFirebaseDashboardAppAsGuest() {
+  return firebase.initializeApp({ databaseURL: dashboardDatabaseUrl });
+}
+
 /**
  * Open Google Cloud Storage for screenshots.
  * The files uploaded to google cloud are also available to firebase storage.

tools/gulp/util/travis-ci.ts

@@ -2,3 +2,7 @@
 export function isTravisMasterBuild() {
   return process.env['TRAVIS_PULL_REQUEST'] === 'false';
 }
+
+export function isTravisBuild() {
+  return process.env['TRAVIS'] === 'true';
+}