[Fixed] Run CI on a single machine by generating cloudbuild files

.gitignore

@@ -58,3 +58,4 @@ tfjs-backend-wasm/wasm-out/*.js
 tfjs-backend-wasm/wasm-out/*.wasm
 yalc.lock
 yarn-error.log
+cloudbuild_generated.yml

cloudbuild.yml

@@ -5,138 +5,25 @@ steps:
   id: 'yarn'
   args: ['install']
 
-# Run find-affected-packages to find affected files in each folder.
+# Generate cloudbuild_generated.yml,
+# which builds and tests all affected packages.
 - name: 'node:10'
   entrypoint: 'yarn'
-  id: 'find-affected-packages'
-  args: ['find-affected-packages']
+  id: 'generate-cloudbuild-for-packages'
+  args: ['generate-cloudbuild-for-packages']
   waitFor: ['yarn']
   env:
   - 'COMMIT_SHA=$COMMIT_SHA'
   - 'BRANCH_NAME=$BRANCH_NAME'
   - 'BASE_BRANCH=$_BASE_BRANCH'
+  - 'NIGHTLY=$_NIGHTLY'
 
-# Core.
+# Run the generated cloudbuild file
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: 'bash'
-  id: 'tfjs-core'
-  args: ['./scripts/run-build.sh', 'tfjs-core']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# CPU backend.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-backend-cpu'
-  args: ['./scripts/run-build.sh', 'tfjs-backend-cpu']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# Webgl backend.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-backend-webgl'
-  args: ['./scripts/run-build.sh', 'tfjs-backend-webgl']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# Converter.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-converter'
-  args: ['./scripts/run-build.sh', 'tfjs-converter']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# Data.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-data'
-  args: ['./scripts/run-build.sh', 'tfjs-data']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# Layers.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-layers'
-  args: ['./scripts/run-build.sh', 'tfjs-layers']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# Union.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs'
-  args: ['./scripts/run-build.sh', 'tfjs']
-  waitFor: ['find-affected-packages']
-
-# Vis.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-vis'
-  args: ['./scripts/run-build.sh', 'tfjs-vis']
-  waitFor: ['find-affected-packages']
-
-# WebGPU.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-backend-webgpu'
-  args: ['./scripts/run-build.sh', 'tfjs-backend-webgpu']
-  waitFor: ['find-affected-packages']
-
-# WASM.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-backend-wasm'
-  args: ['./scripts/run-build.sh', 'tfjs-backend-wasm']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# React Native.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-react-native'
-  args: ['./scripts/run-build.sh', 'tfjs-react-native']
-  waitFor: ['find-affected-packages']
-
-# Node CPU.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-node'
-  args: ['./scripts/run-build.sh', 'tfjs-node']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# Node GPU.
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-node-gpu'
-  args: ['./scripts/run-build.sh', 'tfjs-node-gpu']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# Release notes test
-- name: 'node:10'
-  id: 'test-monorepo'
-  entrypoint: 'yarn'
-  args: ['test-release-notes']
-  waitFor: ['yarn']
-
-# E2E test
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'e2e'
-  args: ['./scripts/run-build.sh', 'e2e']
-  waitFor: ['find-affected-packages']
-  env: ['NIGHTLY=$_NIGHTLY']
-
-# Inference API
-- name: 'gcr.io/cloud-builders/gcloud'
-  entrypoint: 'bash'
-  id: 'tfjs-inference'
-  args: ['scripts/run-build.sh', 'tfjs-inference']
-  waitFor: ['find-affected-packages']
+  id: 'run-cloudbuild'
+  args: ['./scripts/run-build.sh']
+  waitFor: ['generate-cloudbuild-for-packages']
   env: ['NIGHTLY=$_NIGHTLY']
 
 # General settings.

e2e/cloudbuild.yml

@@ -21,8 +21,8 @@ steps:
   id: 'test'
   args: ['test-ci']
   env: ['BROWSERSTACK_USERNAME=deeplearnjs1', 'NIGHTLY=$_NIGHTLY']
-
   secretEnv: ['BROWSERSTACK_KEY']
+  waitFor: ['yarn', 'build-deps']
 
 secrets:
 - kmsKeyName: projects/learnjs-174218/locations/global/keyRings/tfjs/cryptoKeys/enc

package.json

@@ -3,21 +3,25 @@
     "@octokit/rest": "15.17.0",
     "@types/argparse": "^1.0.38",
     "@types/jasmine": "2.8.7",
-    "@types/node": "^12.7.5",
     "@types/mkdirp": "^0.5.2",
+    "@types/node": "^12.7.5",
     "@types/shelljs": "^0.8.7",
     "argparse": "^1.0.10",
     "chalk": "~2.4.2",
     "clang-format": "~1.2.4",
+    "console-table-printer": "^2.4.32",
     "jasmine": "3.1.0",
-    "shelljs": "~0.8.3",
+    "js-yaml": "^3.14.0",
     "mkdirp": "~0.5.1",
+    "shelljs": "~0.8.3",
     "ts-node": "~8.8.2",
     "tslint": "~5.20.0",
     "typescript": "3.5.3"
   },
   "scripts": {
-    "find-affected-packages": "./scripts/find-affected-packages.js",
+    "test-packages-ci": "yarn generate-cloudbuild-for-packages && ./scripts/run-build.sh",
+    "generate-cloudbuild-for-packages": "./scripts/generate_cloudbuild_for_packages.js",
+    "test-generate-cloudbuild": "jasmine run scripts/generate_cloudbuild_test.js",
     "release": "ts-node -s ./scripts/release.ts",
     "release-tfjs": "ts-node -s ./scripts/release-tfjs.ts",
     "publish-npm": "ts-node -s ./scripts/publish-npm.ts",

scripts/cloudbuild.yml

@@ -0,0 +1,14 @@
+steps:
+# Install common dependencies.
+- name: 'node:10'
+  id: 'yarn-common'
+  entrypoint: 'yarn'
+  args: ['install']
+
+# Test generate_cloudbuild.js
+- name: 'node:10'
+  dir: 'scripts'
+  id: 'test-generate-cloudbuild'
+  entrypoint: 'yarn'
+  args: ['test-generate-cloudbuild']
+  waitFor: ['yarn-common']

scripts/cloudbuild_general_config.yml

@@ -0,0 +1,21 @@
+steps:
+# Install top-level deps.
+- name: 'node:10'
+  entrypoint: 'yarn'
+  id: 'yarn-common'
+  args: ['install']
+
+# General configuration
+secrets:
+- kmsKeyName: projects/learnjs-174218/locations/global/keyRings/tfjs/cryptoKeys/enc
+  secretEnv:
+    BROWSERSTACK_KEY: CiQAkwyoIW0LcnxymzotLwaH4udVTQFBEN4AEA5CA+a3+yflL2ASPQAD8BdZnGARf78MhH5T9rQqyz9HNODwVjVIj64CTkFlUCGrP1B2HX9LXHWHLmtKutEGTeFFX9XhuBzNExA=
+    FIREBASE_KEY: CiQAkwyoIXmET39jOD3ywloCIa6+WUpu3w49twpMmkMqy0vS+YsSUAAD8BdZQGOL8FKEBxr/1jl0G78OigwlNVHjD3usZobNtlOp8tV/9iacb8zPFqy0SwIO1gvz3HRr+VU7c7LS2qqaTCdacZF+dx3VJNewvdZu
+timeout: 3600s
+logsBucket: 'gs://tfjs-build-logs'
+substitutions:
+  _NIGHTLY: ''
+options:
+  logStreamingOption: 'STREAM_ON'
+  machineType: 'N1_HIGHCPU_32'
+  substitution_option: 'ALLOW_LOOSE'