Count 429 errors

.github/generateReport.js

@@ -10,8 +10,8 @@ function generateReport(title, dirName, mutantsDirName){
 
 function generateLLMorpheusReport(title, dirName, mutantsDirName){
   let report = `# ${title}\n`
-  report += '| Project | #Prompts | #Mutants | #Killed | #Survived | #Timeout | MutationScore | LLMorpheus Time | Stryker Time | #Prompt Tokens | #Completion Tokens | #Total Tokens  |\n';
-  report += '|:--------|:---------|:---------|:--------|:----------|----------|---------------|-----------------|--------------|----------------|--------------------|----------------|\n';
+  report += '| Project | #Prompts | #Mutants | #Killed | #Survived | #Timeout | MutationScore | LLMorpheus Time | Stryker Time | #Prompt Tokens | #Completion Tokens | #Total Tokens  | #Retries | # Failures |\n';
+  report += '|:--------|:---------|:---------|:--------|:----------|----------|---------------|-----------------|--------------|----------------|--------------------|----------------|----------|------------|\n';
   const files = fs.readdirSync(dirName);
   let totalMutants = 0;
   let totalKilled = 0;
@@ -23,6 +23,8 @@ function generateLLMorpheusReport(title, dirName, mutantsDirName){
   let totalPromptTokens = 0;
   let totalCompletionTokens = 0;
   let totalTotalTokens = 0;
+  let totalNrRetries = 0;
+  let totalNrFailures = 0;
   for (const benchmark of files) {  
     const data = fs.readFileSync(`${dirName}/${benchmark}/StrykerInfo.json`, 'utf8');
     const jsonObj = JSON.parse(data);
@@ -54,9 +56,13 @@ function generateLLMorpheusReport(title, dirName, mutantsDirName){
     totalCompletionTokens += nrCompletionTokens;
     const nrTotalTokens = llmJsonObj.totalTokens;
     totalTotalTokens += nrTotalTokens;
-    report += `| ${benchmark} | ${nrPrompts} | ${nrTotal} | ${nrKilled} | ${nrSurvived} | ${nrTimedOut} | ${mutationScore} | ${llmorpheusTime} | ${strykerTime} | ${nrPromptTokens} | ${nrCompletionTokens} | ${nrTotalTokens} |\n`;
+    nrRetries = llmJsonObj.nrRetries;
+    totalNrRetries += nrRetries;
+    nrFailures = llmJsonObj.nrFailures;
+    totalNrFailures += nrFailures;
+    report += `| ${benchmark} | ${nrPrompts} | ${nrTotal} | ${nrKilled} | ${nrSurvived} | ${nrTimedOut} | ${mutationScore} | ${llmorpheusTime} | ${strykerTime} | ${nrPromptTokens} | ${nrCompletionTokens} | ${nrTotalTokens} | ${nrRetries} | ${nrFailures} |\n`;
   } 
-  report += `| Total | ${totalPrompts} | ${totalMutants} | ${totalKilled} | ${totalSurvived} | ${totalTimedOut} | - | ${totalLLMorpheusTime.toFixed(2)} | ${totalStrykerTime.toFixed(2)} | ${totalPromptTokens} | ${totalCompletionTokens} | ${totalTotalTokens} |\n`;
+  report += `| Total | ${totalPrompts} | ${totalMutants} | ${totalKilled} | ${totalSurvived} | ${totalTimedOut} | - | ${totalLLMorpheusTime.toFixed(2)} | ${totalStrykerTime.toFixed(2)} | ${totalPromptTokens} | ${totalCompletionTokens} | ${totalTotalTokens} | ${totalNrRetries} | ${totalNrFailures} |\n`;
 
   const metaData = retrieveMetaData(mutantsDirName);
 

src/generator/MutantGenerator.ts

@@ -413,6 +413,8 @@ export class MutantGenerator {
     const nrSyntacticallyInvalid = this.mutationStats.nrSyntacticallyInvalid;
     const nrIdentical = this.mutationStats.nrIdentical;
     const nrDuplicate = this.mutationStats.nrDuplicate;
+    const nrRetries = this.model.getFailureCounter().nrRetries;
+    const nrFailures = this.model.getFailureCounter().nrFailures;
     fs.writeFileSync(
       resultsFileName,
       JSON.stringify(
@@ -428,11 +430,14 @@ export class MutantGenerator {
           totalCompletionTokens: this.mutationStats.totalCompletionTokens,
           totalTokens: this.mutationStats.totalTokens,
           metaInfo: this.metaInfo,
+          nrRetries,
+          nrFailures,
         },
         null,
         2
       )
     );
+    console.log(`nrRetries = ${nrRetries}, nrFailures = ${nrFailures}`);
     console.log(`summary written to  ${resultsFileName}\n`);
 
     this.printAndLog(

src/model/CachingModel.ts

@@ -1,7 +1,7 @@
 import fs from "fs";
 import path from "path";
 import crypto from "crypto";
-import { IModel } from "./IModel";
+import { IModel, IModelFailureCounter } from "./IModel";
 import { PostOptions, defaultPostOptions } from "./IModel";
 import { IQueryResult } from "./IQueryResult";
 
@@ -19,6 +19,11 @@ export class CachingModel implements IModel {
     this.modelName = `${model.getModelName()}`;
     console.log(`Using cache dir: ${cacheDir}`);
   }
+
+  getFailureCounter(): IModelFailureCounter {
+    return this.model.getFailureCounter();
+  }
+
   getModelName(): string {
     return `${this.modelName}`;
   }

src/model/IModel.ts

@@ -11,6 +11,7 @@ export interface IModel {
   getModelName(): string;
   getTemperature(): number;
   getMaxTokens(): number;
+  getFailureCounter(): IModelFailureCounter;
 }
 
 export const defaultPostOptions = {
@@ -27,3 +28,8 @@ export const defaultOpenAIPostoptions = {
 
 export type PostOptions = Partial<typeof defaultPostOptions>;
 export type OpenAIPostOptions = Partial<typeof defaultOpenAIPostoptions>;
+
+export interface IModelFailureCounter {
+  nrRetries: number;
+  nrFailures: number;
+}

src/model/MockModel.ts

@@ -1,7 +1,7 @@
 import fs from "fs";
 import path from "path";
 import crypto from "crypto";
-import { IModel } from "./IModel";
+import { IModel, IModelFailureCounter } from "./IModel";
 import { defaultPostOptions } from "./IModel";
 import { IQueryResult } from "./IQueryResult";
 
@@ -16,6 +16,10 @@ export class MockModel implements IModel {
     this.modelName = `${modelName}`;
   }
 
+  getFailureCounter(): IModelFailureCounter {
+    return { nrRetries: 0, nrFailures: 0 };
+  }
+
   getModelName(): string {
     return this.modelName;
   }

src/model/Model.ts

@@ -7,7 +7,7 @@ import {
   RateLimiter,
 } from "../util/promise-utils";
 import { retry } from "../util/promise-utils";
-import { IModel } from "./IModel";
+import { IModel, IModelFailureCounter } from "./IModel";
 import { PostOptions, defaultPostOptions } from "./IModel";
 import { getEnv } from "../util/code-utils";
 import { IQueryResult } from "./IQueryResult";
@@ -26,6 +26,7 @@ export class Model implements IModel {
 
   protected instanceOptions: PostOptions;
   protected rateLimiter: RateLimiter;
+  protected counter: IModelFailureCounter = { nrRetries: 0, nrFailures: 0 };
 
   constructor(
     private modelName: string,
@@ -114,11 +115,15 @@ export class Model implements IModel {
               headers: Model.LLMORPHEUS_LLM_AUTH_HEADERS,
             })
           ),
-        this.metaInfo.nrAttempts
+        this.metaInfo.nrAttempts,
+        () => {
+          this.counter.nrRetries++;
+        }
       );
     } catch (e) {
       if (res?.status === 429) {
         console.error(`*** 429 error: ${e}`);
+        this.counter.nrFailures++;
       }
       throw e;
     }
@@ -155,4 +160,8 @@ export class Model implements IModel {
       total_tokens,
     };
   }
+
+  public getFailureCounter(): IModelFailureCounter {
+    return this.counter;
+  }
 }

src/model/ReplayModel.ts

@@ -1,6 +1,6 @@
 import fs from "fs";
 import path from "path";
-import { IModel } from "./IModel";
+import { IModel, IModelFailureCounter } from "./IModel";
 import { IQueryResult } from "./IQueryResult";
 import { MetaInfo } from "../generator/MetaInfo";
 
@@ -21,6 +21,10 @@ export class ReplayModel implements IModel {
     this.initializeMap();
   }
 
+  getFailureCounter(): IModelFailureCounter {
+    return { nrFailures: 0, nrRetries: 0 };
+  }
+
   /**
    * Initialize a map from prompts to completions by inspecting the contents
    * of the "prompts" subdirectory

src/util/promise-utils.ts

@@ -2,11 +2,13 @@
  * This function provides supports for retrying the creation of a promise
  * up to a given number of times in case the promise is rejected.
  * This is useful for, e.g., retrying a request to a server that is temporarily unavailable.
+ * Invokes notifyFun after each rejection.
  *
  */
 export async function retry<T>(
   f: () => Promise<T>,
-  howManyTimes: number
+  howManyTimes: number,
+  notifyFun: () => void
 ): Promise<T> {
   let i = 1;
   let promise: Promise<T> = f(); // create the promise, but don't wait for its fulfillment yet..
@@ -19,6 +21,9 @@ export async function retry<T>(
       return val; // if the promise was fulfilled, return another promise that is fulfilled with the same value
     } catch (e) {
       i++;
+      if (notifyFun !== undefined) {
+        notifyFun();
+      }
       console.log(`Promise rejected with ${e}.`);
       promise = f(); // next attempt: create the promise, but don't wait for its fulfillment yet..
     }

test/expected/summary.json

@@ -21,5 +21,7 @@
     "ignore": "src/**/*.spec.ts",
     "rateLimit": 1000,
     "benchmark": false
-  }
+  },
+  "nrRetries": 0,
+  "nrFailures": 0
 }