feat: support building in CJS and ESM formats (#2296)

* feat: support ESM and CJS formats

* update imports to use .js extension

* various fixes to get dual formats working

* update internal tooling

* add .js to imports, fix typing

* add gapic-tools to dev deps

* rename .jsdoc.js to .cjs

* add files under cjs to output

* fix merge issue

* add files to owlbot ignore

* add module tracking string to library version

* merge and fix conflicts

* revert parse function

* add pack-n-play tests

* fix merge conflicts

* import cleanups in transfer manager tests

* create separate packageJSON helper in CJS to avoid breaking esbuild

* fix merge conflicts

Author: ddelgrosso1

.jsdoc.js

@@ -0,0 +1,29 @@
+{
+  "opts": {
+    "readme": "./README.md",
+    "package": "./package.json",
+    "template": "./node_modules/jsdoc-fresh",
+    "recurse": true,
+    "verbose": true,
+    "destination": "./docs/"
+  },
+  "plugins": ["plugins/markdown", "jsdoc-region-tag"],
+  "source": {
+    "excludePattern": "(^|\\/|\\\\)[._]",
+    "include": ["build/cjs/src"],
+    "includePattern": "\\.js$"
+  },
+  "templates": {
+    "copyright": "Copyright 2019 Google, LLC.",
+    "includeDate": false,
+    "sourceFiles": false,
+    "systemName": "@google-cloud/storage",
+    "theme": "lumen",
+    "default": {
+      "outputSourceFiles": false
+    }
+  },
+  "markdown": {
+    "idInHeadings": true
+  }
+}

.jsdoc.json

@@ -21,6 +21,7 @@ import {
   deleteTestFile,
 } from './testBenchUtil';
 import * as uuid from 'uuid';
+import {getDirName} from '../src/util.js';
 
 const FILE_SIZE_BYTES = 9 * 1024 * 1024;
 const CHUNK_SIZE_BYTES = 2 * 1024 * 1024;
@@ -390,7 +391,10 @@ export async function bucketSetStorageClass(options: ConformanceTestOptions) {
 export async function bucketUploadResumableInstancePrecondition(
   options: ConformanceTestOptions
 ) {
-  const filePath = path.join(__dirname, `test-data/tmp-${uuid.v4()}.txt`);
+  const filePath = path.join(
+    getDirName(),
+    `../conformance-test/test-data/tmp-${uuid.v4()}.txt`
+  );
   createTestFileFromBuffer(FILE_SIZE_BYTES, filePath);
   if (options.bucket!.instancePreconditionOpts) {
     options.bucket!.instancePreconditionOpts.ifGenerationMatch = 0;
@@ -405,7 +409,10 @@ export async function bucketUploadResumableInstancePrecondition(
 }
 
 export async function bucketUploadResumable(options: ConformanceTestOptions) {
-  const filePath = path.join(__dirname, `test-data/tmp-${uuid.v4()}.txt`);
+  const filePath = path.join(
+    getDirName(),
+    `../conformance-test/test-data/tmp-${uuid.v4()}.txt`
+  );
   createTestFileFromBuffer(FILE_SIZE_BYTES, filePath);
   if (options.preconditionRequired) {
     await options.bucket!.upload(filePath, {
@@ -433,8 +440,8 @@ export async function bucketUploadMultipartInstancePrecondition(
   }
   await options.bucket!.upload(
     path.join(
-      __dirname,
-      '../../conformance-test/test-data/retryStrategyTestData.json'
+      getDirName(),
+      '../../../conformance-test/test-data/retryStrategyTestData.json'
     ),
     {resumable: false}
   );
@@ -448,16 +455,16 @@ export async function bucketUploadMultipart(options: ConformanceTestOptions) {
   if (options.preconditionRequired) {
     await options.bucket!.upload(
       path.join(
-        __dirname,
-        '../../conformance-test/test-data/retryStrategyTestData.json'
+        getDirName(),
+        '../../../conformance-test/test-data/retryStrategyTestData.json'
       ),
       {resumable: false, preconditionOpts: {ifGenerationMatch: 0}}
     );
   } else {
     await options.bucket!.upload(
       path.join(
-        __dirname,
-        '../../conformance-test/test-data/retryStrategyTestData.json'
+        getDirName(),
+        '../../../conformance-test/test-data/retryStrategyTestData.json'
       ),
       {resumable: false}
     );

.mocharc.js renamed to .mocharc.cjs

@@ -15,7 +15,7 @@
  */
 import * as testFile from '../test-data/retryStrategyTestData.json';
 import {executeScenario, RetryTestCase} from '../conformanceCommon';
-import * as assert from 'assert';
+import assert from 'assert';
 
 const SCENARIO_NUMBER_TO_TEST = 5;
 const retryTestCase: RetryTestCase | undefined = testFile.retryTests.find(

.prettierrc.js renamed to .prettierrc.cjs

@@ -15,7 +15,7 @@
  */
 import * as testFile from '../test-data/retryStrategyTestData.json';
 import {executeScenario, RetryTestCase} from '../conformanceCommon';
-import * as assert from 'assert';
+import assert from 'assert';
 
 const SCENARIO_NUMBER_TO_TEST = 4;
 const retryTestCase: RetryTestCase | undefined = testFile.retryTests.find(

conformance-test/libraryMethods.ts

@@ -15,7 +15,7 @@
  */
 import * as testFile from '../test-data/retryStrategyTestData.json';
 import {executeScenario, RetryTestCase} from '../conformanceCommon';
-import * as assert from 'assert';
+import assert from 'assert';
 
 const SCENARIO_NUMBER_TO_TEST = 1;
 const retryTestCase: RetryTestCase | undefined = testFile.retryTests.find(

conformance-test/scenarios/scenarioFive.ts

@@ -15,7 +15,7 @@
  */
 import * as testFile from '../test-data/retryStrategyTestData.json';
 import {executeScenario, RetryTestCase} from '../conformanceCommon';
-import * as assert from 'assert';
+import assert from 'assert';
 
 const SCENARIO_NUMBER_TO_TEST = 7;
 const retryTestCase: RetryTestCase | undefined = testFile.retryTests.find(

conformance-test/scenarios/scenarioFour.ts

@@ -15,7 +15,7 @@
  */
 import * as testFile from '../test-data/retryStrategyTestData.json';
 import {executeScenario, RetryTestCase} from '../conformanceCommon';
-import * as assert from 'assert';
+import assert from 'assert';
 
 const SCENARIO_NUMBER_TO_TEST = 6;
 const retryTestCase: RetryTestCase | undefined = testFile.retryTests.find(