diff --git a/discojs/src/validator.ts b/discojs/src/validator.ts index cb1c3970f..0e4890c61 100644 --- a/discojs/src/validator.ts +++ b/discojs/src/validator.ts @@ -14,17 +14,27 @@ export class Validator { /** infer every line of the dataset and check that it is as labelled */ async *test( dataset: Dataset, - ): AsyncGenerator<{ result: boolean; predicted: DataFormat.ModelEncoded[D][1]; truth : DataFormat.ModelEncoded[D][1] }, void> { + ): AsyncGenerator<{ result: boolean; predicted: DataFormat.Inferred[D]; truth : number }, void> { const results = (await processing.preprocess(this.task, dataset)) .batch(this.task.trainingInformation.batchSize) .map(async (batch) => (await this.#model.predict(batch.map(([inputs, _]) => inputs))) - .zip(batch.map(([_, outputs]) => outputs)) - .map(([inferred, truth]) => ({ result: inferred === truth, predicted: inferred, truth : truth })), - ) - .flatten(); + .zip(batch.map(([_, outputs]) => outputs)) + .map(([inferred, truth]) => ({ result: inferred === truth, predicted: inferred, truth : truth })), + ) + .flatten(); + + const predictions = await processing.postprocess( + this.task, + results.map(({ predicted }) => predicted), + ); + + const finalResults = results.zip(predictions).map(([result, predicted]) => ({ + ...result, + predicted, + })); - for await (const e of results) yield e; + for await (const e of finalResults) yield e; } /** use the model to predict every line of the dataset */ diff --git a/webapp/src/components/testing/TestSteps.vue b/webapp/src/components/testing/TestSteps.vue index e07f90793..fc3f88de9 100644 --- a/webapp/src/components/testing/TestSteps.vue +++ b/webapp/src/components/testing/TestSteps.vue @@ -58,33 +58,34 @@ -
+ +

- Confusion Matrix -

- - - - - - - - - - - - - -
- Label \ Prediction - - {{ confusionMatrix.labels.get(index) }} -
- {{ confusionMatrix.labels.get(rowIndex) }} - - {{ value }} -
-
+ Confusion Matrix + + + + + + + + + + + + + + +
+ Label \ Prediction + + {{ label }} +
+ {{ rowLabel }} + + {{ confusionMatrix.matrix[rowLabel][colLabel] }} +
+
@@ -157,7 +158,7 @@ import ImageCard from "@/components/containers/ImageCard.vue"; import LabeledDatasetInput from "@/components/dataset_input/LabeledDatasetInput.vue"; import TableLayout from "@/components/containers/TableLayout.vue"; import type { LabeledDataset } from "@/components/dataset_input/types.js"; -import { Map, Set } from 'immutable'; +import { Map } from 'immutable'; const debug = createDebug("webapp:testing:TestSteps"); const toaster = useToaster(); @@ -171,7 +172,7 @@ const props = defineProps<{ interface Tested { image: List<{ input: { filename: string; image: ImageData }; - output: { truth: number; correct: boolean; predicted : number, label : string }; + output: { truth: number; correct: boolean; predicted : string, label : string }; }>; tabular: { labels: { @@ -188,7 +189,7 @@ interface Tested { } const dataset = ref(); -const generator = ref>(); +const generator = ref>(); const tested = ref(); const visitedSamples = computed(() => { @@ -207,10 +208,10 @@ const visitedSamples = computed(() => { } }); -const confusionMatrix = computed<{labels : Map, matrix : number[][]} | undefined>(() => { - if (tested.value === undefined) return undefined; - // const labels = Set(); // l'idéal serait de tous les avoir en one try, sinon c'est dégueulasse - // const mapLabels = Map(); +const confusionMatrix = computed<{ labels: Map; matrix: { [key: string]: { [key: string]: number } } }>(() => { + if (tested.value === undefined) { + return { labels: Map(), matrix: {} }; + } let labels : string[] = []; switch (props.task.trainingInformation.dataType) { case "image" : @@ -220,21 +221,28 @@ const confusionMatrix = computed<{labels : Map, matrix : number[ labels = ["0", "1"]; // binary classification break; case "text" : - return undefined; - default: { - const _: never = props.task.trainingInformation; + return { labels: Map(), matrix: {} }; default: { + const _: never = props.task.trainingInformation; throw new Error("should never happen"); } } const size = labels.length; // Initialize the confusion matrix - const matrix = Array.from({ length: size }, () => Array(size).fill(0)); - + const matrix: { [key: string]: { [key: string]: number } } = {}; + + // Initialize the confusion matrix + labels.forEach((label) => { + matrix[label.toString()] = {}; + labels.forEach((innerLabel) => { + matrix[label.toString()][innerLabel.toString()] = 0; + }); + }); + switch (props.task.trainingInformation.dataType) { case "image": (tested.value as Tested["image"]).map( - ( {output} ) => matrix[output.truth][output.predicted] = matrix[output.truth][output.predicted] + 1, + ( {output} ) => matrix[output.label][output.predicted] = matrix[output.label][output.predicted] + 1, ); break; //case "text": @@ -250,6 +258,8 @@ const confusionMatrix = computed<{labels : Map, matrix : number[ } } const mapLabels = Map(labels.map((label, index) => [index, label])); + console.log(mapLabels) + console.log(matrix) return {labels : mapLabels, matrix : matrix}; }) @@ -341,7 +351,7 @@ async function startImageTest( output: { label: label, correct: result, - predicted: predicted, + predicted: String(predicted), truth : truth, }, }); @@ -388,7 +398,7 @@ async function startTabularTest( output: { truth: truth, correct: result, - predicted : predicted, + predicted : Number(predicted), label : truth_label, }, });