Merge pull request #109 from applandinc/feat/upload-retry

feat: Retry uploads

bin/upload.rb

@@ -0,0 +1,10 @@
+Command = "node ./built/cli.js upload -v -d ../../land-of-apps/sample_app_6th_ed --report-file ../../land-of-apps/sample_app_6th_ed/appland-findings.json --app scanner-demo/sample_app_6th_ed"
+Times = (ARGV[0] || 3).to_i
+
+threads = ([Command] * Times).map do |cmd|
+  Thread.new {
+    system cmd
+  }
+end
+
+threads.map(&:join)

package.json

@@ -53,7 +53,7 @@
     "typescript": "^4.4.2"
   },
   "dependencies": {
-    "@appland/client": "^1.1.3",
+    "@appland/client": "^1.3.0",
     "@appland/models": "^1.14.1",
     "@appland/sql-parser": "^1.4.0",
     "ajv": "^8.8.2",

src/cli/ci/command.ts

@@ -9,14 +9,14 @@ import { parseConfigFile } from '../../configuration/configurationProvider';
 import { AbortError, ValidationError } from '../../errors';
 import { ScanResults } from '../../report/scanResults';
 import { verbose } from '../../rules/lib/util';
-import { create as uploadScannerJob } from '../../integration/appland/scannerJob/create';
 import { newFindings } from '../../findings';
 import findingsReport from '../../report/findingsReport';
 import summaryReport from '../../report/summaryReport';
 
 import { ExitCode } from '../exitCode';
 import resolveAppId from '../resolveAppId';
 import validateFile from '../validateFile';
+import upload from '../upload';
 import { default as buildScanner } from '../scan/scanner';
 
 import CommandOptions from './options';
@@ -98,7 +98,9 @@ export default {
       summaryReport(scanResults, true);
 
       if (doUpload) {
-        const uploadResponse = await uploadScannerJob(rawScanResults, appId, mergeKey);
+        const uploadResponse = await upload(rawScanResults, appId, mergeKey, {
+          maxRetries: 3,
+        });
         reportUploadURL(uploadResponse.summary.numFindings, uploadResponse.url);
       }
 

src/cli/upload.ts

@@ -0,0 +1,99 @@
+import { queue } from 'async';
+import { readFile } from 'fs/promises';
+import { join } from 'path';
+
+import { AppMap as AppMapStruct } from '@appland/models';
+
+import { verbose } from '../rules/lib/util';
+import { ScanResults } from '../report/scanResults';
+import {
+  create as createAppMap,
+  CreateOptions as CreateAppMapOptions,
+  UploadAppMapResponse,
+} from '../integration/appland/appMap/create';
+import { create as createMapset } from '../integration/appland/mapset/create';
+import {
+  create as createScannerJob,
+  UploadResponse,
+} from '../integration/appland/scannerJob/create';
+import { RetryOptions } from '../integration/appland/retryOptions';
+
+export default async function create(
+  scanResults: ScanResults,
+  appId: string,
+  mergeKey?: string,
+  options: RetryOptions = {}
+): Promise<UploadResponse> {
+  if (verbose()) console.log(`Uploading AppMaps and findings to application '${appId}'`);
+
+  const { findings } = scanResults;
+
+  const relevantFilePaths = [
+    ...new Set(findings.filter((f) => f.appMapFile).map((f) => f.appMapFile)),
+  ] as string[];
+
+  const appMapUUIDByFileName: Record<string, string> = {};
+  const branchCount: Record<string, number> = {};
+  const commitCount: Record<string, number> = {};
+
+  const createAppMapOptions = {
+    app: appId,
+  } as CreateAppMapOptions;
+
+  const q = queue((filePath: string, callback) => {
+    if (verbose()) console.log(`Uploading AppMap ${filePath}`);
+
+    readFile(filePath)
+      .then((buffer: Buffer) => {
+        const appMapStruct = JSON.parse(buffer.toString()) as AppMapStruct;
+        const metadata = appMapStruct.metadata;
+        const branch = appMapStruct.metadata.git?.branch;
+        const commit = appMapStruct.metadata.git?.commit;
+        if (branch) {
+          branchCount[branch] ||= 1;
+          branchCount[branch] += 1;
+        }
+        if (commit) {
+          commitCount[commit] ||= 1;
+          commitCount[commit] += 1;
+        }
+
+        return createAppMap(buffer, Object.assign(options, { ...createAppMapOptions, metadata }));
+      })
+      .then((appMap: UploadAppMapResponse) => {
+        if (appMap) {
+          appMapUUIDByFileName[filePath] = appMap.uuid;
+        }
+      })
+      .then(() => callback())
+      .catch(callback);
+  }, 3);
+  q.error((err, filePath: string) => {
+    console.error(`An error occurred uploading ${filePath}: ${err}`);
+  });
+  if (verbose()) console.log(`Uploading ${relevantFilePaths.length} AppMaps`);
+  q.push(relevantFilePaths);
+  await q.drain();
+
+  const mostFrequent = (counts: Record<string, number>): string | undefined => {
+    if (Object.keys(counts).length === 0) return;
+
+    const maxCount = Object.values(counts).reduce((max, count) => Math.max(max, count), 0);
+    return Object.entries(counts).find((e) => e[1] === maxCount)![0];
+  };
+
+  const branch = mostFrequent(branchCount);
+  const commit = mostFrequent(commitCount);
+  const mapset = await createMapset(
+    appId,
+    Object.values(appMapUUIDByFileName),
+    Object.assign(options, {
+      branch,
+      commit,
+    })
+  );
+
+  console.warn('Uploading findings');
+
+  return createScannerJob(scanResults, mapset.id, appMapUUIDByFileName, { mergeKey }, options);
+}

src/cli/upload/command.ts

@@ -1,16 +1,16 @@
 import { Arguments, Argv } from 'yargs';
 import { readFile } from 'fs/promises';
 
-import { create as createScannerJob } from '../../integration/appland/scannerJob/create';
 import { ScanResults } from '../../report/scanResults';
 import { verbose } from '../../rules/lib/util';
 
 import validateFile from '../validateFile';
-
-import CommandOptions from './options';
 import resolveAppId from '../resolveAppId';
 import reportUploadURL from '../reportUploadURL';
 
+import CommandOptions from './options';
+import upload from '../upload';
+
 export default {
   command: 'upload',
   describe: 'Upload Findings to the AppMap Server',
@@ -49,7 +49,9 @@ export default {
     const appId = await resolveAppId(appIdArg, appmapDir);
 
     const scanResults = JSON.parse((await readFile(reportFile)).toString()) as ScanResults;
-    const uploadResponse = await createScannerJob(scanResults, appId, mergeKey);
+    const uploadResponse = await upload(scanResults, appId, mergeKey, {
+      maxRetries: 3,
+    });
 
     reportUploadURL(uploadResponse.summary.numFindings, uploadResponse.url);
   },

src/integration/appland/appMap/create.ts

@@ -1,8 +1,10 @@
 import { IncomingMessage } from 'http';
 
-import { buildRequest, handleError } from '@appland/client/dist/src';
+import { buildRequest, handleError, retryOn503, retryOnError } from '@appland/client/dist/src';
 import FormData from 'form-data';
 import { Metadata } from '@appland/models';
+import { RetryOptions } from '../retryOptions';
+import retry from '../retry';
 
 export type UploadAppMapResponse = {
   uuid: string;
@@ -15,33 +17,39 @@ export type CreateOptions = {
 
 export async function create(
   data: Buffer,
-  options: CreateOptions = {}
+  options: CreateOptions,
+  retryOptions: RetryOptions = {}
 ): Promise<UploadAppMapResponse> {
-  const form = new FormData();
-  form.append('data', data.toString());
-  if (options.metadata) {
-    form.append('metadata', JSON.stringify(options.metadata));
-  }
-  if (options.app) {
-    form.append('app', options.app);
-  }
+  const retrier = retry(`Upload AppMap`, retryOptions, makeRequest);
 
-  const request = await buildRequest('api/appmaps');
-  return new Promise<IncomingMessage>((resolve, reject) => {
-    const req = request.requestFunction(
-      request.url,
-      {
-        method: 'POST',
-        headers: {
-          ...request.headers,
-          ...form.getHeaders(),
+  async function makeRequest(): Promise<IncomingMessage> {
+    const form = new FormData();
+    form.append('data', data.toString());
+    if (options.metadata) {
+      form.append('metadata', JSON.stringify(options.metadata));
+    }
+    if (options.app) {
+      form.append('app', options.app);
+    }
+    const request = await buildRequest('api/appmaps');
+    return new Promise<IncomingMessage>((resolve, reject) => {
+      const req = request.requestFunction(
+        request.url,
+        {
+          method: 'POST',
+          headers: {
+            ...request.headers,
+            ...form.getHeaders(),
+          },
         },
-      },
-      resolve
-    );
-    req.on('error', reject);
-    form.pipe(req);
-  })
+        resolve
+      );
+      req.on('error', retryOnError(retrier, resolve, reject));
+      form.pipe(req);
+    }).then(retryOn503(retrier));
+  }
+
+  return makeRequest()
     .then(handleError)
     .then((response: IncomingMessage) => {
       return new Promise<UploadAppMapResponse>((resolve, reject) => {

src/integration/appland/mapset/create.ts

@@ -1,6 +1,9 @@
 import { IncomingMessage } from 'http';
 
-import { buildRequest, handleError } from '@appland/client/dist/src';
+import { buildRequest, handleError, retryOn503, retryOnError } from '@appland/client/dist/src';
+import { verbose } from '../../../rules/lib/util';
+import { RetryOptions } from '../retryOptions';
+import retry from '../retry';
 
 export type CreateResponse = {
   id: number;
@@ -25,33 +28,40 @@ export type CreateOptions = {
 export async function create(
   appId: string,
   appMapIds: string[],
-  options: CreateOptions = {}
+  options: CreateOptions,
+  retryOptions: RetryOptions = {}
 ): Promise<CreateResponse> {
-  console.log(`Creating mapset in app ${appId} with ${appMapIds.length} AppMaps`);
+  if (verbose()) console.log(`Creating mapset in app ${appId} with ${appMapIds.length} AppMaps`);
 
-  const payload = JSON.stringify({
-    app: appId,
-    appmaps: appMapIds,
-    ...options,
-  });
-  const request = await buildRequest('api/mapsets');
-  return new Promise<IncomingMessage>((resolve, reject) => {
-    const req = request.requestFunction(
-      request.url,
-      {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-          'Content-Length': Buffer.byteLength(payload),
-          ...request.headers,
+  const retrier = retry(`Create Mapset`, retryOptions, makeRequest);
+
+  async function makeRequest(): Promise<IncomingMessage> {
+    const payload = JSON.stringify({
+      app: appId,
+      appmaps: appMapIds,
+      ...options,
+    });
+    const request = await buildRequest('api/mapsets');
+    return new Promise<IncomingMessage>((resolve, reject) => {
+      const req = request.requestFunction(
+        request.url,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'Content-Length': Buffer.byteLength(payload),
+            ...request.headers,
+          },
         },
-      },
-      resolve
-    );
-    req.on('error', reject);
-    req.write(payload);
-    req.end();
-  })
+        resolve
+      );
+      req.on('error', retryOnError(retrier, resolve, reject));
+      req.write(payload);
+      req.end();
+    }).then(retryOn503(retrier));
+  }
+
+  return makeRequest()
     .then(handleError)
     .then((response: IncomingMessage) => {
       return new Promise<CreateResponse>((resolve, reject) => {

src/integration/appland/retry.ts

@@ -0,0 +1,37 @@
+import { IncomingMessage } from 'http';
+import { RetryOptions } from './retryOptions';
+import {
+  RejectFunction,
+  RetryHandler,
+  ResolveFunction,
+} from '@appland/client/dist/src/retryHandler';
+import { verbose } from '../../rules/lib/util';
+
+const RetryDelay = 500;
+const MaxRetries = 3;
+
+export default function retry(
+  description: string,
+  retryOptions: RetryOptions,
+  retryFn: () => Promise<IncomingMessage>
+): RetryHandler {
+  const maxRetries = retryOptions.maxRetries ?? MaxRetries;
+  const retryDelay = retryOptions.retryDelay ?? RetryDelay;
+
+  let retryCount = 0;
+
+  function computeDelay(): number {
+    return retryDelay * Math.pow(2, retryCount - 1);
+  }
+
+  return (resolve: ResolveFunction, reject: RejectFunction): void => {
+    retryCount += 1;
+    if (retryCount > maxRetries) {
+      reject(new Error(`${description} failed: Max retries exceeded.`));
+    }
+    if (verbose()) {
+      console.log(`Retrying ${description} in ${computeDelay()}ms`);
+    }
+    setTimeout(() => retryFn().then(resolve).catch(reject), computeDelay());
+  };
+}

src/integration/appland/retryOptions.ts

@@ -0,0 +1,4 @@
+export type RetryOptions = {
+  maxRetries?: number;
+  retryDelay?: number;
+};