From 200723407881c24f77e5b4dc3ed4527799945133 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso <85250797+ddelgrosso1@users.noreply.github.com> Date: Mon, 19 Jul 2021 15:21:15 -0400 Subject: [PATCH] feat: customization options for retries (#441) * feat: customization options for retries * fix linter problems * handle case where number values can be user set to 0 * simplify return statement into one liner * update ApiError interface definition * ensure totalTimeout is respected * linter fixes --- README.md | 60 +++++++++++++++++++++++++++ src/index.ts | 114 +++++++++++++++++++++++++++++++++++++++++++-------- test/test.ts | 58 +++++++++++++++++++++++++- 3 files changed, 214 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 3ed1498..579a850 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,66 @@ If you already have a resumable URI from a previously-created resumable upload, If the bucket being accessed has `requesterPays` functionality enabled, this can be set to control which project is billed for the access of this file. +##### config.retryOptions + +- Type: `object` +- *Optional* + +Parameters used to control retrying operations. + +```js +interface RetryOptions { + retryDelayMultiplier?: number; + totalTimeout?: number; + maxRetryDelay?: number; + autoRetry?: boolean; + maxRetries?: number; + retryableErrorFn?: (err: ApiError) => boolean; +} +``` + +##### config.retryOptions.retryDelayMultiplier + +- Type: `number` +- *Optional* + +Base number used for exponential backoff. Default 2. + +##### config.retryOptions.totalTimeout + +- Type: `number` +- *Optional* + +Upper bound on the total amount of time to attempt retrying, in seconds. Default: 600. + +##### config.retryOptions.maxRetryDelay + +- Type: `number` +- *Optional* + +The maximum time to delay between retries, in seconds. Default: 64. + +##### config.retryOptions.autoRetry + +- Type: `boolean` +- *Optional* + +Whether or not errors should be retried. Default: true. + +##### config.retryOptions.maxRetries + +- Type: `number` +- *Optional* + +The maximum number of retries to attempt. Default: 5. + +##### config.retryOptions.retryableErrorFn + +- Type: `function` +- *Optional* + +Custom function returning a boolean inidicating whether or not to retry an error. + --- ### Events diff --git a/src/index.ts b/src/index.ts index 4bffae4..c5bf5be 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,6 +21,10 @@ const TERMINATED_UPLOAD_STATUS_CODE = 410; const RESUMABLE_INCOMPLETE_STATUS_CODE = 308; const RETRY_LIMIT = 5; const DEFAULT_API_ENDPOINT_REGEX = /.*\.googleapis\.com/; +const MAX_RETRY_DELAY = 64; +const RETRY_DELAY_MULTIPLIER = 2; +const MAX_TOTAL_RETRY_TIMEOUT = 600; +const AUTO_RETRY_VALUE = true; export const PROTOCOL_REGEX = /^(\w*):\/\//; @@ -176,6 +180,11 @@ export interface UploadConfig { * can be set to control which project is billed for the access of this file. */ userProject?: string; + + /** + * Configuration options for retrying retriable errors. + */ + retryOptions?: RetryOptions; } export interface ConfigMetadata { @@ -193,6 +202,19 @@ export interface ConfigMetadata { contentType?: string; } +export interface RetryOptions { + retryDelayMultiplier?: number; + totalTimeout?: number; + maxRetryDelay?: number; + autoRetry?: boolean; + maxRetries?: number; + retryableErrorFn?: (err: ApiError) => boolean; +} + +export interface ApiError extends Error { + code?: number; +} + export class Upload extends Pumpify { bucket: string; file: string; @@ -228,6 +250,12 @@ export class Upload extends Pumpify { numBytesWritten = 0; numRetries = 0; contentLength: number | '*'; + retryLimit: number = RETRY_LIMIT; + maxRetryDelay: number = MAX_RETRY_DELAY; + retryDelayMultiplier: number = RETRY_DELAY_MULTIPLIER; + maxRetryTotalTimeout: number = MAX_TOTAL_RETRY_TIMEOUT; + timeOfFirstRequest: number; + retryableErrorFn?: (err: ApiError) => boolean; private bufferStream?: PassThrough; private offsetStream?: PassThrough; @@ -296,11 +324,33 @@ export class Upload extends Pumpify { configPath, }); + const autoRetry = cfg?.retryOptions?.autoRetry || AUTO_RETRY_VALUE; this.uriProvidedManually = !!cfg.uri; this.uri = cfg.uri || this.get('uri'); this.numBytesWritten = 0; this.numRetries = 0; + if (autoRetry && cfg?.retryOptions?.maxRetries !== undefined) { + this.retryLimit = cfg.retryOptions.maxRetries; + } else if (!autoRetry) { + this.retryLimit = 0; + } + + if (cfg?.retryOptions?.maxRetryDelay !== undefined) { + this.maxRetryDelay = cfg.retryOptions.maxRetryDelay; + } + + if (cfg?.retryOptions?.retryDelayMultiplier !== undefined) { + this.retryDelayMultiplier = cfg.retryOptions.retryDelayMultiplier; + } + + if (cfg?.retryOptions?.totalTimeout !== undefined) { + this.maxRetryTotalTimeout = cfg.retryOptions.totalTimeout; + } + + this.timeOfFirstRequest = Date.now(); + this.retryableErrorFn = cfg?.retryOptions?.retryableErrorFn; + const contentLength = cfg.metadata ? Number(cfg.metadata.contentLength) : NaN; @@ -645,29 +695,59 @@ export class Upload extends Pumpify { * @return {bool} is the request good? */ private onResponse(resp: GaxiosResponse) { - if (resp.status === 404) { - if (this.numRetries < RETRY_LIMIT) { - this.numRetries++; - this.startUploading(); - } else { - this.destroy(new Error('Retry limit exceeded - ' + resp.data)); - } + if ( + (this.retryableErrorFn && + this.retryableErrorFn({ + code: resp.status, + message: resp.statusText, + name: resp.statusText, + })) || + resp.status === 404 || + (resp.status > 499 && resp.status < 600) + ) { + this.attemptDelayedRetry(resp); return false; } - if (resp.status > 499 && resp.status < 600) { - if (this.numRetries < RETRY_LIMIT) { - const randomMs = Math.round(Math.random() * 1000); - const waitTime = Math.pow(2, this.numRetries) * 1000 + randomMs; - this.numRetries++; - setTimeout(this.continueUploading.bind(this), waitTime); + + this.emit('response', resp); + return true; + } + + /** + * @param resp GaxiosResponse object from previous attempt + */ + private attemptDelayedRetry(resp: GaxiosResponse) { + if (this.numRetries < this.retryLimit) { + if (resp.status === 404) { + this.startUploading(); } else { - this.destroy(new Error('Retry limit exceeded - ' + resp.data)); + const retryDelay = this.getRetryDelay(); + if (retryDelay <= 0) { + this.destroy( + new Error(`Retry total time limit exceeded - ${resp.data}`) + ); + return; + } + setTimeout(this.continueUploading.bind(this), retryDelay); } - return false; + this.numRetries++; + } else { + this.destroy(new Error('Retry limit exceeded - ' + resp.data)); } + } - this.emit('response', resp); - return true; + /** + * @returns {number} the amount of time to wait before retrying the request + */ + private getRetryDelay(): number { + const randomMs = Math.round(Math.random() * 1000); + const waitTime = + Math.pow(this.retryDelayMultiplier, this.numRetries) * 1000 + randomMs; + const maxAllowableDelayMs = + this.maxRetryTotalTimeout * 1000 - (Date.now() - this.timeOfFirstRequest); + const maxRetryDelayMs = this.maxRetryDelay * 1000; + + return Math.min(waitTime, maxRetryDelayMs, maxAllowableDelayMs); } /* diff --git a/test/test.ts b/test/test.ts index a95b3fd..411f213 100644 --- a/test/test.ts +++ b/test/test.ts @@ -16,7 +16,7 @@ import * as path from 'path'; import * as sinon from 'sinon'; import {PassThrough, Stream} from 'stream'; -import {CreateUriCallback, PROTOCOL_REGEX} from '../src'; +import {ApiError, CreateUriCallback, PROTOCOL_REGEX} from '../src'; import {GaxiosOptions, GaxiosError, GaxiosResponse} from 'gaxios'; nock.disableNetConnect(); @@ -1361,6 +1361,16 @@ describe('gcs-resumable-upload', () => { it('should return true', () => { assert.strictEqual(up.onResponse(RESP), true); }); + + it('should handle a custom status code when passed a retry function', () => { + const RESP = {status: 1000}; + const customHandlerFunction = (err: ApiError) => { + return err.code === 1000; + }; + up.retryableErrorFn = customHandlerFunction; + + assert.strictEqual(up.onResponse(RESP), false); + }); }); }); @@ -1411,4 +1421,50 @@ describe('gcs-resumable-upload', () => { } }); }); + + describe('#getRetryDelay', () => { + beforeEach(() => { + up.timeOfFirstRequest = Date.now(); + }); + + it('should return exponential retry delay', () => { + const min = Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000; + const max = + Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000 + 1000; + const delayValue = up.getRetryDelay(); + + assert(delayValue >= min && delayValue <= max); + }); + + it('allows overriding the delay multiplier', () => { + [1, 2, 3].forEach(delayMultiplier => { + up.retryDelayMultiplier = delayMultiplier; + const min = Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000; + const max = + Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000 + 1000; + const delayValue = up.getRetryDelay(); + + assert(delayValue >= min && delayValue <= max); + }); + }); + + it('allows overriding the number of retries', () => { + [1, 2, 3].forEach(numRetry => { + up.numRetries = numRetry; + const min = Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000; + const max = + Math.pow(up.retryDelayMultiplier, up.numRetries) * 1000 + 1000; + const delayValue = up.getRetryDelay(); + + assert(delayValue >= min && delayValue <= max); + }); + }); + + it('returns the value of maxRetryDelay when calculated values are larger', () => { + up.maxRetryDelay = 1; + const delayValue = up.getRetryDelay(); + + assert.strictEqual(delayValue, 1000); + }); + }); });