From 0ba8a7effb074782e712ed6816a2ed5003a3b0a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Boull=C3=A9?= Date: Tue, 10 Oct 2023 10:22:56 +0200 Subject: [PATCH] Fix data grid dimensioning bug Correction d'un bug de dimensionnement detecte sur LearningTest\TestKhiops\BugsMultiTables\BugDeployedCoclustering - le deploiement d'un petit coclustering demande etonamment beaucoup trop de RAM - modification du jeu de test avec MemoryLimit a 500 Mb pour detecter ce probleme plus tot - correction - affichage du dimensionnement selon GetPreparationTraceMode dans PLST|MTDatabaseTextFile::ComputeOpenInformation - KWDGAttribute::GetUsedMemory: pour tenir compte des valeurs Symbol ou VarPart - KWDGValueSet::GetUsedMemory: prise en compte uniquement du sizeoff de la classe, pas des valeurs Test complets valides sur LearningTest --- src/Learning/KWDataPreparation/KWDataGrid.cpp | 16 +++------------- src/Learning/KWDataPreparation/KWDataGrid.h | 4 ++-- .../KWDataUtils/KWDatabaseSlicerTask.cpp | 10 +++++----- .../KWDataUtils/PLMTDatabaseTextFile.cpp | 2 +- .../KWDataUtils/PLSTDatabaseTextFile.cpp | 2 +- src/Learning/MODL/MODL.cpp | 1 + 6 files changed, 13 insertions(+), 22 deletions(-) diff --git a/src/Learning/KWDataPreparation/KWDataGrid.cpp b/src/Learning/KWDataPreparation/KWDataGrid.cpp index 71facb3fd..345a603eb 100644 --- a/src/Learning/KWDataPreparation/KWDataGrid.cpp +++ b/src/Learning/KWDataPreparation/KWDataGrid.cpp @@ -3141,8 +3141,8 @@ longint KWDGAttribute::GetUsedMemory() const // Prise en compte des parties et des valeurs if (headPart != NULL) lUsedMemory *= nPartNumber * headPart->GetUsedMemory(); - if (GetAttributeType() == KWType::Symbol) - lUsedMemory += nGranularizedValueNumber * sizeof(KWDGValue); + if (KWType::IsCoclusteringGroupableType(GetAttributeType())) + lUsedMemory += nGranularizedValueNumber * headPart->GetValueSet()->GetHeadValue()->GetUsedMemory(); // Prise en compte de la structure d'indexation lUsedMemory += oaIntervals.GetUsedMemory(); @@ -4448,17 +4448,7 @@ void KWDGValueSet::WriteValues(ostream& ost) const longint KWDGValueSet::GetUsedMemory() const { - longint lUsedMemory; - KWDGValue* value; - - lUsedMemory = sizeof(KWDGValueSet); - value = GetHeadValue(); - while (value != NULL) - { - lUsedMemory += value->GetUsedMemory(); - GetNextValue(value); - } - return lUsedMemory; + return sizeof(KWDGValueSet); } const ALString KWDGValueSet::GetClassLabel() const diff --git a/src/Learning/KWDataPreparation/KWDataGrid.h b/src/Learning/KWDataPreparation/KWDataGrid.h index cdccad4e3..971b75b45 100644 --- a/src/Learning/KWDataPreparation/KWDataGrid.h +++ b/src/Learning/KWDataPreparation/KWDataGrid.h @@ -829,7 +829,7 @@ class KWDGPart : public Object // Controle d'integrite local a la partie (valeurs, cellules de la partie) boolean Check() const override; - // Memoire utilisee + // Memoire utilisee par la partie longint GetUsedMemory() const override; // Affichage @@ -926,7 +926,7 @@ class KWDGPartValues : public Object // Affichage void Write(ostream& ost) const override = 0; - // Memoire utilisee + // Memoire utilisee par la partie, sans ses valeurs longint GetUsedMemory() const override = 0; }; diff --git a/src/Learning/KWDataUtils/KWDatabaseSlicerTask.cpp b/src/Learning/KWDataUtils/KWDatabaseSlicerTask.cpp index f891a51a8..b7bb05961 100644 --- a/src/Learning/KWDataUtils/KWDatabaseSlicerTask.cpp +++ b/src/Learning/KWDataUtils/KWDatabaseSlicerTask.cpp @@ -190,8 +190,8 @@ longint KWDatabaseSlicerTask::GetMinOutputNecessaryMemory() lNecessaryMemorySpace = GetEmptyOutputNecessaryMemory(); // On rajoute au minimum 64 KB par tranche plus deux tailles de buffer pour la tranche courante - lNecessaryMemorySpace += 2 * BufferedFile::nDefaultBufferSize + - shared_DataTableSliceSet.GetDataTableSliceSet()->GetSliceNumber() * 64 * lKB; + lNecessaryMemorySpace += 2 * (longint)BufferedFile::nDefaultBufferSize + + (longint)shared_DataTableSliceSet.GetDataTableSliceSet()->GetSliceNumber() * 64 * lKB; return lNecessaryMemorySpace; } @@ -202,9 +202,9 @@ longint KWDatabaseSlicerTask::GetMaxOutputNecessaryMemory() // On rajoute au maximum une taille de buffer standard par tranche plus deux tailles de buffer pour la tranche // courante - lNecessaryMemorySpace += - 2 * BufferedFile::nDefaultBufferSize + shared_DataTableSliceSet.GetDataTableSliceSet()->GetSliceNumber() * - longint(BufferedFile::nDefaultBufferSize); + lNecessaryMemorySpace += 2 * (longint)BufferedFile::nDefaultBufferSize + + shared_DataTableSliceSet.GetDataTableSliceSet()->GetSliceNumber() * + longint(BufferedFile::nDefaultBufferSize); return lNecessaryMemorySpace; } diff --git a/src/Learning/KWDataUtils/PLMTDatabaseTextFile.cpp b/src/Learning/KWDataUtils/PLMTDatabaseTextFile.cpp index 50f192227..971d38988 100644 --- a/src/Learning/KWDataUtils/PLMTDatabaseTextFile.cpp +++ b/src/Learning/KWDataUtils/PLMTDatabaseTextFile.cpp @@ -54,7 +54,7 @@ boolean PLMTDatabaseTextFile::ComputeOpenInformation(boolean bRead, boolean bInc PLMTDatabaseTextFile* outputDatabaseTextFile) { boolean bOk = true; - boolean bDisplay = false; + boolean bDisplay = GetPreparationTraceMode(); boolean bCurrentVerboseMode; KWMTDatabaseMapping* mapping; KWMTDatabaseMapping* outputMapping; diff --git a/src/Learning/KWDataUtils/PLSTDatabaseTextFile.cpp b/src/Learning/KWDataUtils/PLSTDatabaseTextFile.cpp index 1686895cd..caebadd1d 100644 --- a/src/Learning/KWDataUtils/PLSTDatabaseTextFile.cpp +++ b/src/Learning/KWDataUtils/PLSTDatabaseTextFile.cpp @@ -36,7 +36,7 @@ boolean PLSTDatabaseTextFile::ComputeOpenInformation(boolean bRead, boolean bInc PLSTDatabaseTextFile* outputDatabaseTextFile) { boolean bOk = true; - boolean bDisplay = false; + boolean bDisplay = GetPreparationTraceMode(); boolean bCurrentVerboseMode; KWClass* kwcUsedHeaderLineClass; longint lDatabaseClassNecessaryMemory; diff --git a/src/Learning/MODL/MODL.cpp b/src/Learning/MODL/MODL.cpp index c379e44c4..d2dd00629 100644 --- a/src/Learning/MODL/MODL.cpp +++ b/src/Learning/MODL/MODL.cpp @@ -35,6 +35,7 @@ int main(int argc, char** argv) // Choix du repertoire de lancement pour le debugage sous Windows (a commenter apres fin du debug) // SetWindowsDebugDir("Standard", "IrisLight"); + SetWindowsDebugDir("BugsMultiTables", "BugDeployedCoclustering"); // Parametrage des logs memoires depuis les variables d'environnement, pris en compte dans KWLearningProject // KhiopsMemStatsLogFileName, KhiopsMemStatsLogFrequency, KhiopsMemStatsLogToCollect