test: run e2e tests on pre-compiled packages

The NPM packages being tested must be pre-compiled and the tar packages specified via --package. This way the real packages such as snapshots, release artifacts or cached packages can be tested. Previously the e2e tests compiled and packaged during test execution.

Author: jbedard

.circleci/dynamic_config.yml

@@ -264,7 +264,7 @@ jobs:
         # Waits for the Saucelabs tunnel to be ready. This ensures that we don't run tests
         # too early without Saucelabs not being ready.
       - run: ./scripts/saucelabs/wait-for-tunnel.sh
-      - run: node ./tests/legacy-cli/run_e2e ./tests/legacy-cli/e2e/tests/misc/browsers.ts
+      - run: node ./tests/legacy-cli/run_e2e --glob="tests/misc/browsers.ts"
       - run: ./scripts/saucelabs/stop-tunnel.sh
       - fail_fast
 
@@ -273,6 +273,10 @@ jobs:
     steps:
       - custom_attach_workspace
       - run: yarn build
+      - persist_to_workspace:
+          root: *workspace_location
+          paths:
+            - dist/_*.tgz
 
   build-bazel-e2e:
     executor: action-executor
@@ -376,6 +380,8 @@ jobs:
             # Path where Arsenal Image Mounter files are downloaded.
             # Must match path in .circleci/win-ram-disk.ps1
             - ./aim
+      # Build the npm packages for the e2e tests
+      - run: yarn build
       # Run partial e2e suite on PRs only. Release branches will run the full e2e suite.
       - run:
           name: Execute E2E Tests

docs/DEVELOPER.md

@@ -82,8 +82,10 @@ You can find more info about debugging [tests with Bazel in the docs.](https://g
 
 ### End to end tests
 
-- Run: `node tests/legacy-cli/run_e2e.js`
+- Compile the packages being tested: `yarn build`
+- Run all tests: `node tests/legacy-cli/run_e2e.js`
 - Run a subset of the tests: `node tests/legacy-cli/run_e2e.js tests/legacy-cli/e2e/tests/i18n/ivy-localize-*`
+- Run on a custom set of npm packages (tar files): `node tests/legacy-cli/run_e2e.js --package _angular_cli.tgz _angular_create.tgz dist/*.tgz ...`
 
 When running the debug commands, Node will stop and wait for a debugger to attach.
 You can attach your IDE to the debugger to stop on breakpoints and step through the code. Also, see [IDE Specific Usage](#ide-specific-usage) for a

lib/BUILD.bazel

@@ -118,6 +118,7 @@
     "@types/progress": "^2.0.3",
     "@types/resolve": "^1.17.1",
     "@types/semver": "^7.3.12",
+    "@types/tar": "^6.1.2",
     "@types/text-table": "^0.2.1",
     "@types/uuid": "^8.0.0",
     "@types/yargs": "^17.0.8",

package.json

@@ -1,26 +1,30 @@
 import { getGlobalVariable } from '../utils/env';
+import { PkgInfo } from '../utils/packages';
 import { globalNpm, extractNpmEnv } from '../utils/process';
 import { isPrereleaseCli } from '../utils/project';
 
 export default async function () {
   const testRegistry: string = getGlobalVariable('package-registry');
-  await globalNpm(
-    [
-      'run',
-      'admin',
-      '--',
-      'publish',
-      '--no-versionCheck',
-      '--no-branchCheck',
-      `--registry=${testRegistry}`,
-      '--tag',
-      isPrereleaseCli() ? 'next' : 'latest',
-    ],
-    {
-      ...extractNpmEnv(),
-      // Also set an auth token value for the local test registry which is required by npm 7+
-      // even though it is never actually used.
-      'NPM_CONFIG__AUTH': 'e2e-testing',
-    },
+  const packageTars: PkgInfo[] = Object.values(getGlobalVariable('package-tars'));
+
+  // Publish packages specified with --package
+  await Promise.all(
+    packageTars.map(({ path: p }) =>
+      globalNpm(
+        [
+          'publish',
+          `--registry=${testRegistry}`,
+          '--tag',
+          isPrereleaseCli() ? 'next' : 'latest',
+          p,
+        ],
+        {
+          ...extractNpmEnv(),
+          // Also set an auth token value for the local test registry which is required by npm 7+
+          // even though it is never actually used.
+          'NPM_CONFIG__AUTH': 'e2e-testing',
+        },
+      ),
+    ),
   );
 }

tests/legacy-cli/e2e/setup/010-local-publish.ts

@@ -4,7 +4,7 @@ import { createProjectFromAsset } from '../../utils/assets';
 import { expectFileMatchToExist, readFile } from '../../utils/fs';
 import { getActivePackageManager } from '../../utils/packages';
 import { ng, noSilentNg } from '../../utils/process';
-import { isPrereleaseCli, useCIChrome, useCIDefaults, NgCLIVersion } from '../../utils/project';
+import { isPrereleaseCli, useCIChrome, useCIDefaults, getNgCLIVersion } from '../../utils/project';
 
 export default async function () {
   let restoreRegistry: (() => Promise<void>) | undefined;
@@ -32,7 +32,7 @@ export default async function () {
     const cliMajorProjectVersion = new SemVer(cliVersion).major;
 
     // CLI current version.
-    const cliMajorVersion = NgCLIVersion.major;
+    const cliMajorVersion = getNgCLIVersion().major;
 
     for (let version = cliMajorProjectVersion + 1; version < cliMajorVersion; version++) {
       // Run all the migrations until the current build major version - 1.

tests/legacy-cli/e2e/tests/update/update.ts

@@ -6,11 +6,11 @@ ts_library(
     srcs = glob(["**/*.ts"]),
     visibility = ["//visibility:public"],
     deps = [
-        "//lib",
         "//tests/legacy-cli/e2e/ng-snapshot",
         "@npm//@types/glob",
         "@npm//@types/node-fetch",
         "@npm//@types/semver",
+        "@npm//@types/tar",
         "@npm//@types/yargs-parser",
         "@npm//ansi-colors",
         "@npm//glob",
@@ -19,6 +19,7 @@ ts_library(
         "@npm//puppeteer",
         "@npm//rxjs",
         "@npm//semver",
+        "@npm//tar",
         "@npm//tree-kill",
         "@npm//verdaccio",
         "@npm//verdaccio-auth-memory",

tests/legacy-cli/e2e/utils/BUILD.bazel

@@ -1,6 +1,12 @@
 import { getGlobalVariable } from './env';
 import { ProcessOutput, silentNpm, silentYarn } from './process';
 
+export interface PkgInfo {
+  readonly name: string;
+  readonly version: string;
+  readonly path: string;
+}
+
 export function getActivePackageManager(): 'npm' | 'yarn' {
   const value = getGlobalVariable('package-manager');
   if (value && value !== 'npm' && value !== 'yarn') {

tests/legacy-cli/e2e/utils/packages.ts

@@ -2,12 +2,11 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { prerelease, SemVer } from 'semver';
 import yargsParser from 'yargs-parser';
-import { packages } from '../../../../lib/packages';
 import { getGlobalVariable } from './env';
 import { prependToFile, readFile, replaceInFile, writeFile } from './fs';
 import { gitCommit } from './git';
 import { findFreePort } from './network';
-import { installWorkspacePackages } from './packages';
+import { installWorkspacePackages, PkgInfo } from './packages';
 import { exec, execAndWaitForOutputToMatch, git, ng } from './process';
 
 export function updateJsonFile(filePath: string, fn: (json: any) => any | void) {
@@ -96,6 +95,8 @@ export async function prepareProjectForE2e(name: string) {
 }
 
 export function useBuiltPackagesVersions(): Promise<void> {
+  const packages: { [name: string]: PkgInfo } = getGlobalVariable('package-tars');
+
   return updateJsonFile('package.json', (json) => {
     json['dependencies'] ??= {};
     json['devDependencies'] ??= {};
@@ -221,8 +222,12 @@ export async function useCIChrome(projectDir: string = ''): Promise<void> {
   }
 }
 
-export const NgCLIVersion = new SemVer(packages['@angular/cli'].version);
+export function getNgCLIVersion(): SemVer {
+  const packages: { [name: string]: PkgInfo } = getGlobalVariable('package-tars');
+
+  return new SemVer(packages['@angular/cli'].version);
+}
 
 export function isPrereleaseCli(): boolean {
-  return (prerelease(NgCLIVersion)?.length ?? 0) > 0;
+  return (prerelease(getNgCLIVersion())?.length ?? 0) > 0;
 }

tests/legacy-cli/e2e/utils/project.ts

@@ -0,0 +1,39 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import fs from 'fs';
+import { normalize } from 'path';
+import { Parse } from 'tar';
+
+/**
+ * Extract and return the contents of a single file out of a tar file.
+ *
+ * @param tarball the tar file to extract from
+ * @param filePath the path of the file to extract
+ * @returns the Buffer of file or an error on fs/tar error or file not found
+ */
+export async function extractFile(tarball: string, filePath: string): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    fs.createReadStream(tarball)
+      .pipe(
+        new Parse({
+          strict: true,
+          filter: (p) => normalize(p) === normalize(filePath),
+          // TODO: @types/tar 'entry' does not have ReadEntry.on
+          onentry: (entry: any) => {
+            const chunks: Buffer[] = [];
+
+            entry.on('data', (chunk: any) => chunks!.push(chunk));
+            entry.on('error', reject);
+            entry.on('finish', () => resolve(Buffer.concat(chunks!)));
+          },
+        }),
+      )
+      .on('close', () => reject(`${tarball} does not contain ${filePath}`));
+  });
+}