Merge pull request #90 from KhiopsML/73-bug-descriptives-stats-on-hug…

…e-dataset-400-gb Fix bug for descriptive stats with huge dataset (400 Gb)
KhiopsML · Oct 9, 2023 · 92fabda · 92fabda
2 parents ebcd607 + 49aaec1
commit 92fabda
Show file tree

Hide file tree

Showing 8 changed files with 42 additions and 1,947 deletions.
diff --git a/src/Learning/KWDRRuleLibrary/KWDRTokenCounts.cpp b/src/Learning/KWDRRuleLibrary/KWDRTokenCounts.cpp
diff --git a/src/Learning/KWDRRuleLibrary/KWDRTokenCounts.h b/src/Learning/KWDRRuleLibrary/KWDRTokenCounts.h
diff --git a/src/Learning/KWData/KWCDUniqueString.cpp b/src/Learning/KWData/KWCDUniqueString.cpp
@@ -131,9 +131,8 @@ KWCDUniqueStringDataPtr KWCDUniqueStringDictionary::AsUniqueString(const ALStrin
 		pvUniqueStringDatas.SetAt(nHashPosition, pUniqueStringData);
 
 		// Retaillage dynamique
-		assert(GetHashTableSize() < INT_MAX / sizeof(void*));
-		if (GetCount() > GetHashTableSize() / 2)
-			ReinitHashTable(DictionaryGetNextTableSize(2 * GetHashTableSize()));
+		if (GetCount() > GetHashTableSize() / 2 and GetHashTableSize() < INT_MAX)
+			ReinitHashTable(DictionaryGetNextTableSize(2 * min(GetHashTableSize(), INT_MAX / 2)));
 	}
 	return pUniqueStringData;
 }

diff --git a/src/Learning/KWData/KWSymbol.cpp b/src/Learning/KWData/KWSymbol.cpp
@@ -556,7 +556,7 @@ void KWSymbolDictionary::ReinitHashTable(int nNewHashSize)
 	// Affichage du debut de la methode
 	if (bDisplay)
 	{
-		cout << "ReinitHashTable (" << GetCount() << "," << GetHashTableSize() << ")";
+		cout << "Symbol ReinitHashTable (" << GetCount() << "," << GetHashTableSize() << ")";
 		cout << " -> " << nNewHashSize << ": " << flush;
 		timer.Start();
 	}
@@ -769,9 +769,8 @@ KWSymbolDataPtr KWSymbolDictionary::AsSymbol(const char* key, int nLength)
 		pvSymbolDatas.SetAt(nHashPosition, pSymbolData);
 
 		// Retaillage dynamique
-		assert(GetHashTableSize() < INT_MAX / sizeof(void*));
-		if (GetCount() > GetHashTableSize() / 2)
-			ReinitHashTable(DictionaryGetNextTableSize(2 * GetHashTableSize()));
+		if (GetCount() > GetHashTableSize() / 2 and GetHashTableSize() < INT_MAX)
+			ReinitHashTable(DictionaryGetNextTableSize(2 * min(GetHashTableSize(), INT_MAX / 2)));
 	}
 	return pSymbolData;
 }