diff --git a/benchmark/generateCostsTable.ts b/benchmark/generateCostsTable.ts new file mode 100644 index 0000000..4a0269f --- /dev/null +++ b/benchmark/generateCostsTable.ts @@ -0,0 +1,157 @@ +import * as fs from "fs"; + +function createTableFooter(dirName: string, runNr: number): string { + // get meta-info from first benchmark + const results = fs.readdirSync(dirName); + const firstBenchmarkName = results[0]; + const firstBenchmarkFile = fs.readFileSync( + `${dirName}/${firstBenchmarkName}/summary.json`, + "utf8" + ); + const firstSummary = JSON.parse(firstBenchmarkFile); + const modelName = firstSummary.metaInfo.modelName; + let temperature = firstSummary.metaInfo.temperature; + if (temperature === "0" || temperature === 0) { + temperature = "0.0"; + } + if (temperature === "1" || temperature === 1) { + temperature = "1.0"; + } + const maxTokens = firstSummary.metaInfo.maxTokens; + const maxNrPrompts = firstSummary.metaInfo.maxNrPrompts; + let template = firstSummary.metaInfo.template; + template = template.substring(template.indexOf("/") + 1); + const systemPrompt = firstSummary.metaInfo.systemPrompt; + const rateLimit = firstSummary.metaInfo.rateLimit; + const nrAttempts = firstSummary.metaInfo.nrAttempts; + + return `\\end{tabular} + } + \\\\[2mm] + \\caption{Results from LLMorpheus experiment \\ChangedText{(run \\#${runNr})}. + Model: \\textit{${modelName}}, + temperature: ${temperature}, + maxTokens: ${maxTokens}, + maxNrPrompts: ${maxNrPrompts}, + template: \\textit{${template}}, + systemPrompt: \\textit{${systemPrompt}}, + rateLimit: ${rateLimit}, + nrAttempts: ${nrAttempts}. + } + \\label{table:Cost:run${runNr}:${modelName}:${template}:${temperature}} +\\end{table*}`; +} + +/** + * Convert a string like "19m16.114s" to seconds. + * Use commas to separate thousands. + */ +function convertToSeconds(time: string): number { + const timeParts = time.split(/m|s/); + const minutes = parseInt(timeParts[0]); + const seconds = parseFloat(timeParts[1]); + return minutes * 60 + seconds; +} + +function formatFixedNr(nr: number): string { + const fixedString = nr.toFixed(2).toString(); + const dotIndex = fixedString.indexOf("."); + const beforeDot = fixedString.slice(0, dotIndex); + const afterDot = fixedString.slice(dotIndex); + return `${numberWithCommas(parseInt(beforeDot))}${afterDot}`; +} + +/** + * Use commas to separate thousands. + */ +function numberWithCommas(x: number): string { + return x.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ","); +} + +// create table with columns: time (LLMorpheus), time (Stryker), prompt tokens, completion tokens, total tokens +export function generateCostsTable(dirName: string, runNr: number): string { + const results = fs.readdirSync(dirName); + + let totalLLMorpheusTime = 0; + let totalStrykerTime = 0; + let totalPromptTokens = 0; + let totalCompletionTokens = 0; + let totalTotalTokens = 0; + + let result = ` +% table generated using command: "node benchmark/generateCostsTable.js ${dirName} ${runNr}" +\\begin{table*}[hbt!] +\\centering +\{\\scriptsize +\\begin{tabular}{l||r|r|r|r|r} +\\multicolumn{1}{c|}{\\bf project} & \\multicolumn{2}{|c|}{\\bf time (sec)} & \\multicolumn{3}{|c|}{\\bf \\#tokens} \\\\ + & {\\it LLMorpheus} & {\\it StrykerJS} & {\\bf prompt} & {\\bf compl.} & {\\bf total} \\\\ +\\hline + `; + for (const benchmarkName of results) { + if (benchmarkName.startsWith(".")) continue; + if (benchmarkName.endsWith(".zip")) continue; + const file = fs.readFileSync( + `${dirName}/${benchmarkName}/summary.json`, + "utf8" + ); + const summary = JSON.parse(file); + const promptTokens = numberWithCommas(parseInt(summary.totalPromptTokens)); + const completionTokens = numberWithCommas( + parseInt(summary.totalCompletionTokens) + ); + const totalTokens = numberWithCommas(parseInt(summary.totalTokens)); + const strykerInfo = JSON.parse( + fs.readFileSync(`${dirName}/${benchmarkName}/StrykerInfo.json`, "utf8") + ); + const strykerTime: number = convertToSeconds(strykerInfo.time); + + // retrieve LLMorpheus time from the third to last line of file LLMorpheusOutput.txt after the word "real" + const LLMorpheusOutput = fs.readFileSync( + `${dirName}/${benchmarkName}/LLMorpheusOutput.txt`, + "utf8" + ); + const lines = LLMorpheusOutput.split("\n"); + const summaryLine = lines[lines.length - 4]; + const summaryLineParts = summaryLine.split("real"); + const summaryLineTime = summaryLineParts[1].trim(); + const LLMorpheusTime: number = convertToSeconds(summaryLineTime); + + result += `${benchmarkName} & ${formatFixedNr( + LLMorpheusTime + )} & ${formatFixedNr( + strykerTime + )} & ${promptTokens} & ${completionTokens} & ${totalTokens} \\\\ \n`; + + totalLLMorpheusTime += LLMorpheusTime; + totalStrykerTime += strykerTime; + totalPromptTokens += parseInt(summary.totalPromptTokens); + totalCompletionTokens += parseInt(summary.totalCompletionTokens); + totalTotalTokens += parseInt(summary.totalTokens); + } + result += `\\hline + \\textit{Total} & ${formatFixedNr(totalLLMorpheusTime)} & ${formatFixedNr( + totalStrykerTime + )} & ${numberWithCommas(totalPromptTokens)} & ${numberWithCommas( + totalCompletionTokens + )} & ${numberWithCommas(totalTotalTokens)} \\\\ + `; + result += createTableFooter(dirName, runNr); + return result; +} + +// to be executed from the command line only +if (require.main === module) { + const dirName = process.argv[2]; // read dirName from command line + const pathEntries = dirName.split("/"); + const lastEntry = pathEntries[pathEntries.length - 1]; + if (!lastEntry.startsWith("run")) { + throw new Error( + "Usage: node /benchmark/generateCostsTable.js " + ); + } + const runNr = parseInt(lastEntry.substring(3)); + const table = generateCostsTable(dirName + "/zip", runNr); + + console.log(table); +} diff --git a/benchmark/generateMutantsTable.ts b/benchmark/generateMutantsTable.ts new file mode 100644 index 0000000..42b08fa --- /dev/null +++ b/benchmark/generateMutantsTable.ts @@ -0,0 +1,194 @@ +import * as fs from "fs"; +import { unzip } from "zlib"; + +/** + * Create header of latex table as shown above. + */ +function createTableHeader(dirName: string, runNr: number): string { + return ` +% table generated using command: "node benchmark/generateMutantsTable.js ${dirName} ${runNr}" +\\begin{table*}[hbt!] +\\centering +{\\scriptsize +\\begin{tabular}{l||r|r|r|r|r|r|r|r|r|r} + {\\bf application} & {\\bf \\#prompts} & \\multicolumn{4}{|c|}{\\bf \\ChangedText{mutant candidates}} & {\\bf \\#mutants} & {\\bf \\#killed} & {\\bf \\#survived} & {\\bf \\#timeout} & {\\bf mut.} \\\\ + & & {\\bf \\ChangedText{total}} & {\\bf \\ChangedText{invalid}} & {\\bf \\ChangedText{identical}} & {\\bf \\ChangedText{duplicate}} & & & & & {\\bf score} \\\\ + \\hline + `; +} + +export function getTemperature(dirName: string) { + const firstBenchmarkName = "delta"; + const firstBenchmarkFile = fs.readFileSync( + `${dirName}/${firstBenchmarkName}/summary.json`, + "utf8" + ); + const firstSummary = JSON.parse(firstBenchmarkFile); + let temperature = firstSummary.metaInfo.temperature; + if (temperature === "0" || temperature === 0) { + temperature = "0.0"; + } + if (temperature === "1" || temperature === 1) { + temperature = "1.0"; + } + return temperature; +} + +export function getTemplate(dirName: string) { + // const results = fs.readdirSync(dirName); + const firstBenchmarkName = "delta"; + const firstBenchmarkFile = fs.readFileSync( + `${dirName}/${firstBenchmarkName}/summary.json`, + "utf8" + ); + const firstSummary = JSON.parse(firstBenchmarkFile); + let template = firstSummary.metaInfo.template; + template = template.substring(template.indexOf("/") + 1); + // remove ".hb" at the end + template = template.substring(0, template.length - 3); + return template; +} + +function createTableFooter(dirName: string, runNr: number): string { + // get meta-info from first benchmark + const firstBenchmarkName = "delta"; + const firstBenchmarkFile = fs.readFileSync( + `${dirName}/${firstBenchmarkName}/summary.json`, + "utf8" + ); + const firstSummary = JSON.parse(firstBenchmarkFile); + const modelName = firstSummary.metaInfo.modelName; + let temperature = firstSummary.metaInfo.temperature; + if (temperature === "0" || temperature === 0) { + temperature = "0.0"; + } + if (temperature === "1" || temperature === 1) { + temperature = "1.0"; + } + const maxTokens = firstSummary.metaInfo.maxTokens; + const maxNrPrompts = firstSummary.metaInfo.maxNrPrompts; + let template = firstSummary.metaInfo.template; + template = template.substring(template.indexOf("/") + 1); + const systemPrompt = firstSummary.metaInfo.systemPrompt; + const rateLimit = firstSummary.metaInfo.rateLimit; + const nrAttempts = firstSummary.metaInfo.nrAttempts; + + return `\\end{tabular} + } + \\\\[2mm] + \\caption{Results from LLMorpheus experiment \\ChangedText{(run \\#${runNr})}. + Model: \\textit{${modelName}}, + temperature: ${temperature}, + maxTokens: ${maxTokens}, + maxNrPrompts: ${maxNrPrompts}, + template: \\textit{${template}}, + systemPrompt: \\textit{${systemPrompt}}, + rateLimit: ${rateLimit}, + nrAttempts: ${nrAttempts}. + } + \\label{table:Mutants:run${runNr}:${modelName}:${template}:${temperature}} +\\end{table*}`; +} + +function unzipDirIfNeccessary(dirName: string) { + const results = fs.readdirSync(dirName); + if (!results.includes("delta")) { + // unzip "mutants.zip" and "results.zip" in dirName + for (const zipFile of ["mutants.zip", "results.zip"]) { + // execute shell command to unzip + const execSync = require("child_process").execSync; + execSync(`unzip ${dirName}/${zipFile} -d ${dirName}`, { + stdio: "inherit", + }); + } + } +} + +export function generateMutantsTable(dirName: string, runNr: number): string { + unzipDirIfNeccessary(dirName); + let result = createTableHeader(dirName, runNr); + const results = fs.readdirSync(dirName); + let totalNrPrompts = 0; + let totalNrCandidates = 0; + let totalNrSyntacticallyInvalid = 0; + let totalNrIdentical = 0; + let totalNrDuplicate = 0; + let totalNrMutants = 0; + let totalNrKilled = 0; + let totalNrSurvived = 0; + let totalNrTimedOut = 0; + + for (const projectName of results) { + // skip directories and zip files + if (projectName.endsWith(".zip")) continue; + if (!fs.lstatSync(`${dirName}/${projectName}`).isDirectory()) continue; + result += `\\hline\n`; + if (!fs.existsSync(`${dirName}/${projectName}/summary.json`)) { + throw new Error( + `summary.json file not found in ${dirName}/${projectName}` + ); + } + if (!fs.existsSync(`${dirName}/${projectName}/StrykerInfo.json`)) { + throw new Error( + `StrykerInfo.json file not found in ${dirName}/${projectName}` + ); + } + + const jsonLLMorpheusObj = JSON.parse( + fs.readFileSync(`${dirName}/${projectName}/summary.json`, "utf8") + ); + const nrPrompts = parseInt(jsonLLMorpheusObj.nrPrompts); + const nrCandidates = parseInt( + jsonLLMorpheusObj.nrCandidates + jsonLLMorpheusObj.nrDuplicate + ); + const nrSyntacticallyInvalid = parseInt( + jsonLLMorpheusObj.nrSyntacticallyInvalid + ); + const nrIdentical = parseInt(jsonLLMorpheusObj.nrIdentical); + const nrDuplicate = parseInt(jsonLLMorpheusObj.nrDuplicate); + + const jsonStrykerObj = JSON.parse( + fs.readFileSync(`${dirName}/${projectName}/StrykerInfo.json`, "utf8") + ); + + const nrKilled = parseInt(jsonStrykerObj.nrKilled); + const nrSurvived = parseInt(jsonStrykerObj.nrSurvived); + const nrTimedOut = parseInt(jsonStrykerObj.nrTimedOut); + const nrMutants = nrKilled + nrSurvived + nrTimedOut; + const mutScore = jsonStrykerObj.mutationScore; + + result += `\\textit{${projectName}} & ${nrPrompts} & \\ChangedText\{${nrCandidates}\} & \\ChangedText\{${nrSyntacticallyInvalid}\} & \\ChangedText\{${nrIdentical}\} & \\ChangedText\{${nrDuplicate}\} & ${nrMutants} & ${nrKilled} & ${nrSurvived} & ${nrTimedOut} & ${parseFloat( + mutScore + ).toFixed(2)} \\\\ \n`; + + totalNrPrompts += nrPrompts; + totalNrCandidates += nrCandidates; + totalNrSyntacticallyInvalid += nrSyntacticallyInvalid; + totalNrIdentical += nrIdentical; + totalNrDuplicate += nrDuplicate; + totalNrMutants += nrMutants; + totalNrKilled += nrKilled; + totalNrSurvived += nrSurvived; + totalNrTimedOut += nrTimedOut; + } + result += `\\hline\n`; + result += `\\textit{Total} & ${totalNrPrompts} & \\ChangedText\{${totalNrCandidates}\} & \\ChangedText\{${totalNrSyntacticallyInvalid}\} & \\ChangedText\{${totalNrIdentical}\} & \\ChangedText\{${totalNrDuplicate}\} & ${totalNrMutants} & ${totalNrKilled} & ${totalNrSurvived} & ${totalNrTimedOut} & --- \\\\ \n`; + result += createTableFooter(dirName, runNr); + return result; +} + +// to be executed from the command line only +if (require.main === module) { + const dirName = process.argv[2]; // read dirName from command line + const pathEntries = dirName.split("/"); + const lastEntry = pathEntries[pathEntries.length - 1]; + if (!lastEntry.startsWith("run")) { + throw new Error( + "Usage: node /benchmark/generateMutantsTable.js " + ); + } + const runNr = parseInt(lastEntry.substring(3)); + const table = generateMutantsTable(dirName + "/zip", runNr); + + console.log(table); +} diff --git a/benchmark/generateTables.ts b/benchmark/generateTables.ts new file mode 100644 index 0000000..e94efdd --- /dev/null +++ b/benchmark/generateTables.ts @@ -0,0 +1,107 @@ +import * as fs from "fs"; +import path from "path"; +import { + generateMutantsTable, + getTemperature, + getTemplate, +} from "./generateMutantsTable"; +import { generateCostsTable } from "./generateCostsTable"; +import { generateVariabilityTable } from "./generateVariabilityTable"; + +const dataDir = process.argv[2]; // read dataDir from command line +const outputDir = process.argv[3]; // read outputDir from command line + +// check if both directories exist +if (!fs.existsSync(dataDir)) { + throw new Error(`Directory ${dataDir} does not exist`); +} +if (!fs.existsSync(outputDir)) { + console.log(`Directory ${outputDir} does not exist, creating it`); + fs.mkdirSync(outputDir); +} + +console.log( + `Generating tables from data in ${dataDir} and saving them in ${outputDir}` +); + +for (const modelName of [ + "codellama-34b-instruct", + "codellama-13b-instruct", + "mixtral-8x7b-instruct", +]) { + const modelNameDir = `${dataDir}/${modelName}`; + const configurations = fs.readdirSync(modelNameDir); + // console.log(`configurations for ${modelName} = ${configurations}`); + for (const configuration of configurations) { + // console.log(`*** configuration = ${configuration}`); + const configurationDir = `${modelNameDir}/${configuration}`; + console.log(`generating tables for ${modelName} with ${configuration}`); + for (const run of fs.readdirSync(configurationDir)) { + if (!run.startsWith("run")) continue; + const runDir = `${configurationDir}/${run}`; + const runNr = parseInt(run.substring(3)); + const mutantsTable: string = generateMutantsTable(`${runDir}/zip`, runNr); + const costsTable: string = generateCostsTable(`${runDir}/zip`, runNr); + + // write table to the appropriate file in the output directory + const subDir = `${configuration}`; + + const mutantsTableFileName = `${run}-mutants-table.tex`; + const mutantsTablePath = path.join( + outputDir, + modelName, + subDir, + run, + mutantsTableFileName + ); + + const costsTableFileName = `${run}-costs-table.tex`; + const costsTablePath = path.join( + outputDir, + modelName, + subDir, + run, + costsTableFileName + ); + + // check if the directory exists, create if necessary + if (!fs.existsSync(path.join(outputDir, modelName))) { + fs.mkdirSync(path.join(outputDir, modelName)); + } + if (!fs.existsSync(path.join(outputDir, modelName, subDir))) { + fs.mkdirSync(path.join(outputDir, modelName, subDir)); + } + if (!fs.existsSync(path.join(outputDir, modelName, subDir, run))) { + fs.mkdirSync(path.join(outputDir, modelName, subDir, run)); + } + console.log( + ` -- writing mutants table for run #${runNr} of ${modelName} with configuration ${configuration} to ${mutantsTablePath}` + ); + fs.writeFileSync(mutantsTablePath, mutantsTable); + console.log( + ` -- writing costs table for run #${runNr} of ${modelName} with configuration ${configuration} to ${costsTablePath}` + ); + fs.writeFileSync(costsTablePath, costsTable); + } + // generate the variability table + const runs = fs + .readdirSync(configurationDir) + .filter((run) => run.startsWith("run")); + const variabilityTable = generateVariabilityTable(configurationDir, runs); + const temperature = getTemperature(`${configurationDir}/${runs[0]}/zip`); + const tableFileName = `table-variability-${modelName}-${configuration}.tex`; + const variabilityTablePath = path.join( + outputDir, + "variability", + tableFileName + ); + // console.log(`variability table for ${modelName} with ${configuration} = ${variabilityTable}`); + if (!fs.existsSync(path.join(outputDir, "variability"))) { + fs.mkdirSync(path.join(outputDir, "variability")); + } + console.log( + ` -- writing variability table for runs# ${runs} of ${modelName} with ${configuration} to ${variabilityTablePath}` + ); + fs.writeFileSync(variabilityTablePath, variabilityTable); + } +} diff --git a/benchmark/computeVariability.ts b/benchmark/generateVariabilityTable.ts similarity index 78% rename from benchmark/computeVariability.ts rename to benchmark/generateVariabilityTable.ts index 0019b3d..f7f0452 100644 --- a/benchmark/computeVariability.ts +++ b/benchmark/generateVariabilityTable.ts @@ -41,7 +41,7 @@ function retrieveMutantsForProject( projectName: string ): Set { const data = fs.readFileSync( - path.join(baseDir, run, "projects", projectName, "mutants.json"), + path.join(baseDir, run, "zip", projectName, "mutants.json"), "utf8" ); return new Set(JSON.parse(data).map((x: any) => getMutantInfo(x))); @@ -89,9 +89,7 @@ function findCommonMutants( function getModelName(baseDir: string, run: string): string { const file = fs.readFileSync( - path.join( - path.join(baseDir, run, "projects", projectNames[0], "summary.json") - ), + path.join(path.join(baseDir, run, "zip", projectNames[0], "summary.json")), "utf8" ); const json = JSON.parse(file); @@ -100,9 +98,7 @@ function getModelName(baseDir: string, run: string): string { function getTemperature(baseDir: string, run: string): string { const file = fs.readFileSync( - path.join( - path.join(baseDir, run, "projects", projectNames[0], "summary.json") - ), + path.join(path.join(baseDir, run, "zip", projectNames[0], "summary.json")), "utf8" ); const json = JSON.parse(file); @@ -116,17 +112,20 @@ function getTemperature(baseDir: string, run: string): string { /** * Generate a LaTeX table that shows the variability of mutants across runs. */ -function generateVariabilityTable(baseDir: string, runs: string[]): void { - let latexTable = - `% table generated using command: "node benchmark/computeVariability.js ${baseDir.substring( - baseDir.lastIndexOf("/") + 1 - )}${runs.join(" ")}"\n` + - "\\begin{table}\n" + - "\\centering\n" + - "{\\footnotesize\n" + - "\\begin{tabular}{l|r|r|r|r}\n" + - "{\\bf application} & {\\bf \\#min} & {\\bf \\#max} & {\\bf \\#distinct} & {\\bf \\#common}\\\\\n" + - "\\hline\n"; +export function generateVariabilityTable( + baseDir: string, + runs: string[] +): string { + let latexTable = ` +% table generated using command: "node benchmark/computeVariability.js ${baseDir} ${runs.join( + " " + )}" +\\begin{table}[hbt!] +\\centering +{\\footnotesize +\\begin{tabular}{l|r|r|r|r}\n +{\\bf application} & {\\bf \\#min} & {\\bf \\#max} & {\\bf \\#distinct} & {\\bf \\#common}\\\\ +\\hline\n`; for (const projectName of projectNames) { const allMutants = findAllMutants(baseDir, runs, projectName); const allMutantsSize = allMutants.size; @@ -148,8 +147,9 @@ function generateVariabilityTable(baseDir: string, runs: string[]): void { "\\end{tabular}\n}\n" + "\\caption{\n" + ` Variability of the mutants generated in 5 runs of \\ToolName using the \\textit{${modelName}} LLM - at temperature ${temperature}. The columns of the table show,\n` + - " from left to right: \n" + + at temperature ${temperature} \\ChangedText{(run ${runs.map((s) => + s.replace("run", "\\#") + )})}. The columns of the table show, from left to right:\n` + " (i) the minimum number of mutants observed in any of the runs,\n" + " (ii) the maximum number of mutants observed in any of the runs,\n" + " (iii) the total number of distinct mutants observed in all runs, and\n" + @@ -157,11 +157,14 @@ function generateVariabilityTable(baseDir: string, runs: string[]): void { "}\n" + `\\label{table:Variability_${modelName}_${temperature}}\n` + "\\end{table}"; - console.log(latexTable); + return latexTable; } -// usage: node benchmark/computeVariability.js +if (require.main === module) { + const baseDir = process.argv[2]; + const runs = process.argv.slice(3); + const table = generateVariabilityTable(baseDir, runs); + console.log(table); +}