feat: customization options for retries (#441)

* feat: customization options for retries

* fix linter problems

* handle case where number values can be user set to 0

* simplify return statement into one liner

* update ApiError interface definition

* ensure totalTimeout is respected

* linter fixes

README.md

@@ -234,6 +234,66 @@ If you already have a resumable URI from a previously-created resumable upload,
 
 If the bucket being accessed has `requesterPays` functionality enabled, this can be set to control which project is billed for the access of this file.
 
+##### config.retryOptions
+
+- Type: `object`
+- *Optional*
+
+Parameters used to control retrying operations.
+
+```js
+interface RetryOptions {
+  retryDelayMultiplier?: number;
+  totalTimeout?: number;
+  maxRetryDelay?: number;
+  autoRetry?: boolean;
+  maxRetries?: number;
+  retryableErrorFn?: (err: ApiError) => boolean;
+}
+```
+
+##### config.retryOptions.retryDelayMultiplier
+
+- Type: `number`
+- *Optional*
+
+Base number used for exponential backoff. Default 2.
+
+##### config.retryOptions.totalTimeout
+
+- Type: `number`
+- *Optional*
+
+Upper bound on the total amount of time to attempt retrying, in seconds. Default: 600.
+
+##### config.retryOptions.maxRetryDelay
+
+- Type: `number`
+- *Optional*
+
+The maximum time to delay between retries, in seconds. Default: 64.
+
+##### config.retryOptions.autoRetry
+
+- Type: `boolean`
+- *Optional*
+
+Whether or not errors should be retried. Default: true.
+
+##### config.retryOptions.maxRetries
+
+- Type: `number`
+- *Optional*
+
+The maximum number of retries to attempt. Default: 5.
+
+##### config.retryOptions.retryableErrorFn
+
+- Type: `function`
+- *Optional*
+
+Custom function returning a boolean inidicating whether or not to retry an error.
+
 ---
 <a name="events"></a>
 ### Events

src/index.ts

@@ -21,6 +21,10 @@ const TERMINATED_UPLOAD_STATUS_CODE = 410;
 const RESUMABLE_INCOMPLETE_STATUS_CODE = 308;
 const RETRY_LIMIT = 5;
 const DEFAULT_API_ENDPOINT_REGEX = /.*\.googleapis\.com/;
+const MAX_RETRY_DELAY = 64;
+const RETRY_DELAY_MULTIPLIER = 2;
+const MAX_TOTAL_RETRY_TIMEOUT = 600;
+const AUTO_RETRY_VALUE = true;
 
 export const PROTOCOL_REGEX = /^(\w*):\/\//;
 
@@ -176,6 +180,11 @@ export interface UploadConfig {
    * can be set to control which project is billed for the access of this file.
    */
   userProject?: string;
+
+  /**
+   * Configuration options for retrying retriable errors.
+   */
+  retryOptions?: RetryOptions;
 }
 
 export interface ConfigMetadata {
@@ -193,6 +202,19 @@ export interface ConfigMetadata {
   contentType?: string;
 }
 
+export interface RetryOptions {
+  retryDelayMultiplier?: number;
+  totalTimeout?: number;
+  maxRetryDelay?: number;
+  autoRetry?: boolean;
+  maxRetries?: number;
+  retryableErrorFn?: (err: ApiError) => boolean;
+}
+
+export interface ApiError extends Error {
+  code?: number;
+}
+
 export class Upload extends Pumpify {
   bucket: string;
   file: string;
@@ -228,6 +250,12 @@ export class Upload extends Pumpify {
   numBytesWritten = 0;
   numRetries = 0;
   contentLength: number | '*';
+  retryLimit: number = RETRY_LIMIT;
+  maxRetryDelay: number = MAX_RETRY_DELAY;
+  retryDelayMultiplier: number = RETRY_DELAY_MULTIPLIER;
+  maxRetryTotalTimeout: number = MAX_TOTAL_RETRY_TIMEOUT;
+  timeOfFirstRequest: number;
+  retryableErrorFn?: (err: ApiError) => boolean;
   private bufferStream?: PassThrough;
   private offsetStream?: PassThrough;
 
@@ -296,11 +324,33 @@ export class Upload extends Pumpify {
       configPath,
     });
 
+    const autoRetry = cfg?.retryOptions?.autoRetry || AUTO_RETRY_VALUE;
     this.uriProvidedManually = !!cfg.uri;
     this.uri = cfg.uri || this.get('uri');
     this.numBytesWritten = 0;
     this.numRetries = 0;
 
+    if (autoRetry && cfg?.retryOptions?.maxRetries !== undefined) {
+      this.retryLimit = cfg.retryOptions.maxRetries;
+    } else if (!autoRetry) {
+      this.retryLimit = 0;
+    }
+
+    if (cfg?.retryOptions?.maxRetryDelay !== undefined) {
+      this.maxRetryDelay = cfg.retryOptions.maxRetryDelay;
+    }
+
+    if (cfg?.retryOptions?.retryDelayMultiplier !== undefined) {
+      this.retryDelayMultiplier = cfg.retryOptions.retryDelayMultiplier;
+    }
+
+    if (cfg?.retryOptions?.totalTimeout !== undefined) {
+      this.maxRetryTotalTimeout = cfg.retryOptions.totalTimeout;
+    }
+
+    this.timeOfFirstRequest = Date.now();
+    this.retryableErrorFn = cfg?.retryOptions?.retryableErrorFn;
+
     const contentLength = cfg.metadata
       ? Number(cfg.metadata.contentLength)
       : NaN;
@@ -645,29 +695,59 @@ export class Upload extends Pumpify {
    * @return {bool} is the request good?
    */
   private onResponse(resp: GaxiosResponse) {
-    if (resp.status === 404) {
-      if (this.numRetries < RETRY_LIMIT) {
-        this.numRetries++;
-        this.startUploading();
-      } else {
-        this.destroy(new Error('Retry limit exceeded - ' + resp.data));
-      }
+    if (
+      (this.retryableErrorFn &&
+        this.retryableErrorFn({
+          code: resp.status,
+          message: resp.statusText,
+          name: resp.statusText,
+        })) ||
+      resp.status === 404 ||
+      (resp.status > 499 && resp.status < 600)
+    ) {
+      this.attemptDelayedRetry(resp);
       return false;
     }
-    if (resp.status > 499 && resp.status < 600) {
-      if (this.numRetries < RETRY_LIMIT) {
-        const randomMs = Math.round(Math.random() * 1000);
-        const waitTime = Math.pow(2, this.numRetries) * 1000 + randomMs;
-        this.numRetries++;
-        setTimeout(this.continueUploading.bind(this), waitTime);
+
+    this.emit('response', resp);
+    return true;
+  }
+
+  /**
+   * @param resp GaxiosResponse object from previous attempt
+   */
+  private attemptDelayedRetry(resp: GaxiosResponse) {
+    if (this.numRetries < this.retryLimit) {
+      if (resp.status === 404) {
+        this.startUploading();
       } else {
-        this.destroy(new Error('Retry limit exceeded - ' + resp.data));
+        const retryDelay = this.getRetryDelay();
+        if (retryDelay <= 0) {
+          this.destroy(
+            new Error(`Retry total time limit exceeded - ${resp.data}`)
+          );
+          return;
+        }
+        setTimeout(this.continueUploading.bind(this), retryDelay);
       }
-      return false;
+      this.numRetries++;
+    } else {
+      this.destroy(new Error('Retry limit exceeded - ' + resp.data));
     }
+  }
 
-    this.emit('response', resp);
-    return true;
+  /**
+   * @returns {number} the amount of time to wait before retrying the request
+   */
+  private getRetryDelay(): number {
+    const randomMs = Math.round(Math.random() * 1000);
+    const waitTime =
+      Math.pow(this.retryDelayMultiplier, this.numRetries) * 1000 + randomMs;
+    const maxAllowableDelayMs =
+      this.maxRetryTotalTimeout * 1000 - (Date.now() - this.timeOfFirstRequest);
+    const maxRetryDelayMs = this.maxRetryDelay * 1000;
+
+    return Math.min(waitTime, maxRetryDelayMs, maxAllowableDelayMs);
   }
 
   /*

test/test.ts

@@ -16,7 +16,7 @@ import * as path from 'path';
 import * as sinon from 'sinon';
 import {PassThrough, Stream} from 'stream';
 
-import {CreateUriCallback, PROTOCOL_REGEX} from '../src';
+import {ApiError, CreateUriCallback, PROTOCOL_REGEX} from '../src';
 import {GaxiosOptions, GaxiosError, GaxiosResponse} from 'gaxios';
 
 nock.disableNetConnect();
@@ -1361,6 +1361,16 @@ describe('gcs-resumable-upload', () => {
       it('should return true', () => {
         assert.strictEqual(up.onResponse(RESP), true);
       });
+
+      it('should handle a custom status code when passed a retry function', () => {
+        const RESP = {status: 1000};
+        const customHandlerFunction = (err: ApiError) => {
+          return err.code === 1000;
+        };
+        up.retryableErrorFn = customHandlerFunction;
+
+        assert.strictEqual(up.onResponse(RESP), false);
+      });
     });
   });
 
@@ -1411,4 +1421,50 @@ describe('gcs-resumable-upload', () => {
       }
     });
   });
+
+  describe('#getRetryDelay', () => {
+    beforeEach(() => {
+      up.timeOfFirstRequest = Date.now();
+    });
+
+    it('should return exponential retry delay', () => {
+      const min = Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000;
+      const max =
+        Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000 + 1000;
+      const delayValue = up.getRetryDelay();
+
+      assert(delayValue >= min && delayValue <= max);
+    });
+
+    it('allows overriding the delay multiplier', () => {
+      [1, 2, 3].forEach(delayMultiplier => {
+        up.retryDelayMultiplier = delayMultiplier;
+        const min = Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000;
+        const max =
+          Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000 + 1000;
+        const delayValue = up.getRetryDelay();
+
+        assert(delayValue >= min && delayValue <= max);
+      });
+    });
+
+    it('allows overriding the number of retries', () => {
+      [1, 2, 3].forEach(numRetry => {
+        up.numRetries = numRetry;
+        const min = Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000;
+        const max =
+          Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000 + 1000;
+        const delayValue = up.getRetryDelay();
+
+        assert(delayValue >= min && delayValue <= max);
+      });
+    });
+
+    it('returns the value of maxRetryDelay when calculated values are larger', () => {
+      up.maxRetryDelay = 1;
+      const delayValue = up.getRetryDelay();
+
+      assert.strictEqual(delayValue, 1000);
+    });
+  });
 });