feat: Transfer Manager Metrics (#2305)

* feat: Transfer Manager Metrics

* fix: Add missing `gccl-gcs-cmd/` prefix

* fix: remove unused method

* refactor: update logic for both headers

* refactor: use `Symbol.for` in case the file is evaluated multiple times

* fix: add missing URL param

* fix: if header doesn't exist, add it

* test: Add `GCCL_GCS_CMD_KEY` TM tests

* feat: Add base `x-goog-api-client` for XML API

* chore: dep type change

* fix: pass `GCCL_GCS_CMD_KEY`

* refactor: remove unused

* test: Add `GCCL_GCS_CMD` tests

* chore: cleanup

Author: d-goog

src/file.ts

@@ -61,6 +61,7 @@ import {
   ApiError,
   Duplexify,
   DuplexifyConstructor,
+  GCCL_GCS_CMD_KEY,
 } from './nodejs-common/util';
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const duplexify: DuplexifyConstructor = require('duplexify');
@@ -221,6 +222,7 @@ type PublicResumableUploadOptions =
 export interface CreateResumableUploadOptions
   extends Pick<resumableUpload.UploadConfig, PublicResumableUploadOptions> {
   preconditionOpts?: PreconditionOptions;
+  [GCCL_GCS_CMD_KEY]?: resumableUpload.UploadConfig[typeof GCCL_GCS_CMD_KEY];
 }
 
 export type CreateResumableUploadResponse = [string];
@@ -371,6 +373,7 @@ export interface CreateReadStreamOptions {
   start?: number;
   end?: number;
   decompress?: boolean;
+  [GCCL_GCS_CMD_KEY]?: string;
 }
 
 export interface SaveOptions extends CreateWriteStreamOptions {
@@ -1580,12 +1583,16 @@ class File extends ServiceObject<File, FileMetadata> {
         headers.Range = `bytes=${tailRequest ? end : `${start}-${end}`}`;
       }
 
-      const reqOpts = {
+      const reqOpts: DecorateRequestOptions = {
         uri: '',
         headers,
         qs: query,
       };
 
+      if (options[GCCL_GCS_CMD_KEY]) {
+        reqOpts[GCCL_GCS_CMD_KEY] = options[GCCL_GCS_CMD_KEY];
+      }
+
       this.requestStream(reqOpts)
         .on('error', err => {
           throughStream.destroy(err);
@@ -1738,6 +1745,7 @@ class File extends ServiceObject<File, FileMetadata> {
         userProject: options.userProject || this.userProject,
         retryOptions: retryOptions,
         params: options?.preconditionOpts || this.instancePreconditionOpts,
+        [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY],
       },
       callback!
     );
@@ -3907,6 +3915,7 @@ class File extends ServiceObject<File, FileMetadata> {
       params: options?.preconditionOpts || this.instancePreconditionOpts,
       chunkSize: options?.chunkSize,
       highWaterMark: options?.highWaterMark,
+      [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY],
     });
 
     uploadStream
@@ -3951,6 +3960,7 @@ class File extends ServiceObject<File, FileMetadata> {
         name: this.name,
       },
       uri: uri,
+      [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY],
     };
 
     if (this.generation !== undefined) {

src/nodejs-common/service.ts

@@ -21,6 +21,7 @@ import {Interceptor} from './service-object';
 import {
   BodyResponseCallback,
   DecorateRequestOptions,
+  GCCL_GCS_CMD_KEY,
   MakeAuthenticatedRequest,
   PackageJson,
   util,
@@ -253,6 +254,12 @@ export class Service {
       } gccl-invocation-id/${uuid.v4()}`,
     };
 
+    if (reqOpts[GCCL_GCS_CMD_KEY]) {
+      reqOpts.headers[
+        'x-goog-api-client'
+      ] += ` gccl-gcs-cmd/${reqOpts[GCCL_GCS_CMD_KEY]}`;
+    }
+
     if (reqOpts.shouldReturnStream) {
       return this.makeAuthenticatedRequest(reqOpts) as {} as r.Request;
     } else {

src/nodejs-common/util.ts

@@ -36,6 +36,14 @@ import {getRuntimeTrackingString} from '../util';
 
 const packageJson = require('../../../package.json');
 
+/**
+ * A unique symbol for providing a `gccl-gcs-cmd` value
+ * for the `X-Goog-API-Client` header.
+ *
+ * E.g. the `V` in `X-Goog-API-Client: gccl-gcs-cmd/V`
+ **/
+export const GCCL_GCS_CMD_KEY = Symbol.for('GCCL_GCS_CMD');
+
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const duplexify: DuplexifyConstructor = require('duplexify');
 
@@ -214,7 +222,9 @@ export interface MakeWritableStreamOptions {
   request?: r.Options;
 
   makeAuthenticatedRequest(
-    reqOpts: r.OptionsWithUri,
+    reqOpts: r.OptionsWithUri & {
+      [GCCL_GCS_CMD_KEY]?: string;
+    },
     fnobj: {
       onAuthenticated(
         err: Error | null,
@@ -233,6 +243,7 @@ export interface DecorateRequestOptions extends r.CoreOptions {
   interceptors_?: Interceptor[];
   shouldReturnStream?: boolean;
   projectId?: string;
+  [GCCL_GCS_CMD_KEY]?: string;
 }
 
 export interface ParsedHttpResponseBody {
@@ -530,7 +541,9 @@ export class Util {
           body: writeStream,
         },
       ],
-    } as {} as r.OptionsWithUri;
+    } as {} as r.OptionsWithUri & {
+      [GCCL_GCS_CMD_KEY]?: string;
+    };
 
     options.makeAuthenticatedRequest(reqOpts, {
       onAuthenticated(err, authenticatedReqOpts) {
@@ -539,7 +552,9 @@ export class Util {
           return;
         }
 
-        requestDefaults.headers = util._getDefaultHeaders();
+        requestDefaults.headers = util._getDefaultHeaders(
+          reqOpts[GCCL_GCS_CMD_KEY]
+        );
         const request = teenyRequest.defaults(requestDefaults);
         request(authenticatedReqOpts!, (err, resp, body) => {
           util.handleResp(err, resp, body, (err, data) => {
@@ -863,7 +878,9 @@ export class Util {
       maxRetryValue = config.retryOptions.maxRetries;
     }
 
-    requestDefaults.headers = this._getDefaultHeaders();
+    requestDefaults.headers = this._getDefaultHeaders(
+      reqOpts[GCCL_GCS_CMD_KEY]
+    );
     const options = {
       request: teenyRequest.defaults(requestDefaults),
       retries: autoRetryValue !== false ? maxRetryValue : 0,
@@ -1014,13 +1031,19 @@ export class Util {
       : [optionsOrCallback as T, cb as C];
   }
 
-  _getDefaultHeaders() {
-    return {
+  _getDefaultHeaders(gcclGcsCmd?: string) {
+    const headers = {
       'User-Agent': util.getUserAgentFromPackageJson(packageJson),
       'x-goog-api-client': `${getRuntimeTrackingString()} gccl/${
         packageJson.version
       } gccl-invocation-id/${uuid.v4()}`,
     };
+
+    if (gcclGcsCmd) {
+      headers['x-goog-api-client'] += ` gccl-gcs-cmd/${gcclGcsCmd}`;
+    }
+
+    return headers;
   }
 }
 

src/resumable-upload.ts

@@ -27,6 +27,7 @@ import retry = require('async-retry');
 import {RetryOptions, PreconditionOptions} from './storage';
 import * as uuid from 'uuid';
 import {getRuntimeTrackingString} from './util';
+import {GCCL_GCS_CMD_KEY} from './nodejs-common/util';
 
 const NOT_FOUND_STATUS_CODE = 404;
 const RESUMABLE_INCOMPLETE_STATUS_CODE = 308;
@@ -193,6 +194,8 @@ export interface UploadConfig extends Pick<WritableOptions, 'highWaterMark'> {
    * Configuration options for retrying retryable errors.
    */
   retryOptions: RetryOptions;
+
+  [GCCL_GCS_CMD_KEY]?: string;
 }
 
 export interface ConfigMetadata {
@@ -274,6 +277,7 @@ export class Upload extends Writable {
   private localWriteCache: Buffer[] = [];
   private localWriteCacheByteLength = 0;
   private upstreamEnded = false;
+  #gcclGcsCmd?: string;
 
   constructor(cfg: UploadConfig) {
     super(cfg);
@@ -347,6 +351,8 @@ export class Upload extends Writable {
       : NaN;
     this.contentLength = isNaN(contentLength) ? '*' : contentLength;
 
+    this.#gcclGcsCmd = cfg[GCCL_GCS_CMD_KEY];
+
     this.once('writing', () => {
       if (this.uri) {
         this.continueUploading();
@@ -585,6 +591,14 @@ export class Upload extends Writable {
       delete metadata.contentType;
     }
 
+    let googAPIClient = `${getRuntimeTrackingString()} gccl/${
+      packageJson.version
+    } gccl-invocation-id/${this.currentInvocationId.uri}`;
+
+    if (this.#gcclGcsCmd) {
+      googAPIClient += ` gccl-gcs-cmd/${this.#gcclGcsCmd}`;
+    }
+
     // Check if headers already exist before creating new ones
     const reqOpts: GaxiosOptions = {
       method: 'POST',
@@ -598,9 +612,7 @@ export class Upload extends Writable {
       ),
       data: metadata,
       headers: {
-        'x-goog-api-client': `${getRuntimeTrackingString()} gccl/${
-          packageJson.version
-        } gccl-invocation-id/${this.currentInvocationId.uri}`,
+        'x-goog-api-client': googAPIClient,
         ...headers,
       },
     };
@@ -766,10 +778,16 @@ export class Upload extends Writable {
       },
     });
 
+    let googAPIClient = `${getRuntimeTrackingString()} gccl/${
+      packageJson.version
+    } gccl-invocation-id/${this.currentInvocationId.chunk}`;
+
+    if (this.#gcclGcsCmd) {
+      googAPIClient += ` gccl-gcs-cmd/${this.#gcclGcsCmd}`;
+    }
+
     const headers: GaxiosOptions['headers'] = {
-      'x-goog-api-client': `${getRuntimeTrackingString()} gccl/${
-        packageJson.version
-      } gccl-invocation-id/${this.currentInvocationId.chunk}`,
+      'x-goog-api-client': googAPIClient,
     };
 
     // If using multiple chunk upload, set appropriate header
@@ -904,15 +922,21 @@ export class Upload extends Writable {
   }
 
   private async getAndSetOffset() {
+    let googAPIClient = `${getRuntimeTrackingString()} gccl/${
+      packageJson.version
+    } gccl-invocation-id/${this.currentInvocationId.offset}`;
+
+    if (this.#gcclGcsCmd) {
+      googAPIClient += ` gccl-gcs-cmd/${this.#gcclGcsCmd}`;
+    }
+
     const opts: GaxiosOptions = {
       method: 'PUT',
       url: this.uri!,
       headers: {
         'Content-Length': 0,
         'Content-Range': 'bytes */*',
-        'x-goog-api-client': `${getRuntimeTrackingString()} gccl/${
-          packageJson.version
-        } gccl-invocation-id/${this.currentInvocationId.offset}`,
+        'x-goog-api-client': googAPIClient,
       },
     };
     try {