Merge pull request #221 from bab2min/dev_130

Dev 0.13.0

.github/workflows/deploy.yml

@@ -106,13 +106,13 @@ jobs:
           auditwheel repair dist/*-${{ matrix.cp }}-linux_*.whl
           TWINE_USERNAME=${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD=${{ secrets.PYPI_PASSWORD }} /opt/python/${{ matrix.cp }}/bin/python -m twine upload wheelhouse/*.whl
 
-  build_macos_11:
+  build_macos_13:
     name: Build for macOS 13
     runs-on: macOS-13
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.8, 3.9, "3.10", 3.11]
+        python-version: [3.8, 3.9, "3.10", 3.11, 3.12]
         cpu-arch: [x86_64, arm64]
 
     steps:
@@ -123,7 +123,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        python -m pip install --upgrade pip setuptools
         wget https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz
         tar -zxvf eigen-3.4.0.tar.gz
         rm eigen-3.4.0.tar.gz

.github/workflows/deploy_test.yml

@@ -105,13 +105,13 @@ jobs:
           auditwheel repair dist/*-${{ matrix.cp }}-linux_*.whl
           TWINE_USERNAME=${{ secrets.TEST_PYPI_USERNAME }} TWINE_PASSWORD=${{ secrets.TEST_PYPI_PASSWORD }} /opt/python/${{ matrix.cp }}/bin/python -m twine upload --repository testpypi wheelhouse/*.whl
 
-  build_macos_11:
+  build_macos_13:
     name: Build for macOS 13
     runs-on: macOS-13
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.8, 3.9, "3.10", 3.11]
+        python-version: [3.8, 3.9, "3.10", 3.11, 3.12]
         cpu-arch: [x86_64, arm64]
 
     steps:
@@ -122,7 +122,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        python -m pip install --upgrade pip setuptools
         wget https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz
         tar -zxvf eigen-3.4.0.tar.gz
         rm eigen-3.4.0.tar.gz

.github/workflows/generate_documentation.yml

@@ -41,15 +41,15 @@ jobs:
         mv variant/include/mapbox include/
     - name: build
       run: |
-        /opt/python/cp38-cp38/bin/python -m pip install numpy==`/opt/python/cp38-cp38/bin/python .github/workflows/numpy_version.py`
-        /opt/python/cp38-cp38/bin/python -m pip install pdoc3==0.8.4
+        /opt/python/cp39-cp39/bin/python -m pip install numpy==`/opt/python/cp39-cp39/bin/python .github/workflows/numpy_version.py`
+        /opt/python/cp39-cp3/bin/python -m pip install pdoc3==0.8.4
         export TOMOTOPY_LANG=${{ matrix.language }}
-        /opt/python/cp38-cp38/bin/python setup.py install
+        /opt/python/cp39-cp39/bin/python setup.py install
     - name: gen doc
       run: |
-        export TOMOTOPY_VER="`/opt/python/cp38-cp38/bin/python -m pip show tomotopy | grep Version | cut -d' ' -f2`"
+        export TOMOTOPY_VER="`/opt/python/cp39-cp39/bin/python -m pip show tomotopy | grep Version | cut -d' ' -f2`"
         export TOMOTOPY_LANG=${{ matrix.language }}
-        /opt/python/cp38-cp38/bin/python -m pdoc --html tomotopy
+        /opt/python/cp39-cp39/bin/python -m pdoc --html tomotopy
         sed -i -E "s/documentation<\/title>/documentation (v${TOMOTOPY_VER})<\/title>/" html/tomotopy/*.html
         sed -i -E 's/<\/title>/<\/title><link rel="icon" type="image\/png" sizes="192x192" href="\/tomotopy\/favicon-192x192.png">/' html/tomotopy/*.html
         sed -i -E 's/(<p><img alt="" src="https:\/\/badge.fury.io\/py\/tomotopy.svg"><\/p>)/<script async defer src="https:\/\/buttons.github.io\/buttons.js"><\/script> <p><img alt="" src="https:\/\/badge.fury.io\/py\/tomotopy.svg"> <a class="github-button" href="https:\/\/github.com\/bab2min\/tomotopy" data-size="large" data-show-count="true" aria-label="Star bab2min\/tomotopy on GitHub">Star<\/a> <a class="github-button" href="https:\/\/github.com\/bab2min\/tomotopy\/issues" data-size="large" data-show-count="true" aria-label="Issue bab2min\/tomotopy on GitHub">Issue<\/a> <\/p>/' html/tomotopy/*.html

.github/workflows/pull_request_test.yml

@@ -102,13 +102,13 @@ jobs:
         name: Artifacts ${{ matrix.cp }}
         path: artifacts/build.tgz
 
-  build_macos_11:
+  build_macos_13:
     name: Build for macOS 13
     runs-on: macOS-13
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.8, 3.9, "3.10", 3.11]
+        python-version: [3.9, "3.10", 3.11, 3.12]
         cpu-arch: ["x86_64", "arm64"]
 
     steps:
@@ -119,7 +119,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        python -m pip install --upgrade pip setuptools
         wget https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz
         tar -zxvf eigen-3.4.0.tar.gz
         rm eigen-3.4.0.tar.gz

MANIFEST.in

@@ -1,4 +1,4 @@
 recursive-include src *.c *.cpp *.h *.hpp
-recursive-include include *.h *.hpp
+recursive-include include *
 recursive-include tomotopy *.py *.rst *.html
 include LICENSE.txt

README.kr.rst

@@ -7,7 +7,7 @@ tomotopy
 .. image:: https://zenodo.org/badge/186155463.svg
    :target: https://zenodo.org/badge/latestdoi/186155463
 
-🎌
+🌐
 `English`_,
 **한국어**.
 
@@ -305,6 +305,15 @@ tomotopy의 Python3 예제 코드는 https://github.com/bab2min/tomotopy/blob/ma
 
 역사
 -------
+* 0.13.0 (2024-08-05)
+    * 신규 기능
+        * 토픽 모델 뷰어인 `tomotopy.viewer.open_viewer()`의 주요 기능이 완성되었습니다.
+        * `tomotopy.LDAModel.get_hash()`가 추가되었습니다. 모델의 128bit 해시를 구해줍니다.
+        * `ngram_list` 인자가 `tomotopy.utils.SimpleTokenizer`에 추가되었습니다.
+    * Bug fixes
+        * `Corpus.concat_ngrams` 호출 후에 `spans`이 비일관적인 버그가 수정되었습니다.
+        * `tomotopy.LDAModel.load()`와 `tomotopy.LDAModel.save()`의 병목을 최적화하여 속도를 10배 이상 개선했습니다.
+
 * 0.12.7 (2023-12-19)
     * 신규 기능
         * 토픽 모델 뷰어인 `tomotopy.viewer.open_viewer()`가 추가되었습니다.

README.rst

@@ -7,7 +7,7 @@ tomotopy
 .. image:: https://zenodo.org/badge/186155463.svg
    :target: https://zenodo.org/badge/latestdoi/186155463
 
-🎌
+🌐
 **English**,
 `한국어`_.
 
@@ -309,6 +309,15 @@ meaning you can use it for any reasonable purpose and remain in complete ownersh
 
 History
 -------
+* 0.13.0 (2024-08-05)
+    * New features
+        * Major features of Topic Model Viewer `tomotopy.viewer.open_viewer()` are ready now.
+        * `tomotopy.LDAModel.get_hash()` is added. You can get 128bit hash value of the model.
+        * Add an argument `ngram_list` to `tomotopy.utils.SimpleTokenizer`.
+    * Bug fixes
+        * Fixed inconsistent `spans` bug after `Corpus.concat_ngrams` is called.
+        * Optimized the bottleneck of `tomotopy.LDAModel.load()` and `tomotopy.LDAModel.save()` and improved its speed more than 10 times.
+
 * 0.12.7 (2023-12-19)
     * New features
         * Added Topic Model Viewer `tomotopy.viewer.open_viewer()`

document/document_header.html

@@ -10,8 +10,9 @@
 <a class="homelink" rel="home" title="tomotopy Home" href="/tomotopy" style="display:block; font-size:2em; font-weight:bold; color:#555; padding-bottom:.5em; border-bottom:1px solid silver;"> <img src="/tomotopy/tomoto.png" alt="" style="height:1.5em;"> tomotopy </a>
 <a id='lang-en' href="../en/index.html">English</a> <a id='lang-kr' href="../kr/index.html">한국어</a> 
 <div id="version-link">
-<span>v0.12.7</span>
+<span>v0.13.0</span>
 <ul>
+	<li><a href='/tomotopy/v0.13.0/en'>v0.13.0</a></li>
 	<li><a href='/tomotopy/v0.12.7/en'>v0.12.7</a></li>
 	<li><a href='/tomotopy/v0.12.6/en'>v0.12.6</a></li>
 	<li><a href='/tomotopy/v0.12.5/en'>v0.12.5</a></li>

setup.py

@@ -199,14 +199,14 @@ def build_extension(self, ext):
     arch_levels = ['none', 'sse2', 'avx2']
 elif platform.system() == 'Darwin': 
     if tomotopy_cpu_arch == 'arm64':
-        arch_levels = ['aarch64']
+        arch_levels = ['arm64']
     elif build_for_manyplatform: 
         arch_levels = ['sse2', 'avx', 'avx2']
     else:
         arch_levels = ['sse2']
 elif build_for_manyplatform:
     if tomotopy_cpu_arch in ('arm64', 'aarch64'):
-        arch_levels = ['aarch64']
+        arch_levels = ['arm64']
 else:
     arch_levels = ['native']
 
@@ -237,7 +237,7 @@ def build_extension(self, ext):
 
 for arch in arch_levels:
     if len(arch_levels) > 1:
-        if arch in ('none', 'native', 'aarch64'):
+        if arch in ('none', 'native', 'arm64'):
             module_name = '_tomotopy_none'
         else:
             module_name = '_tomotopy_' + arch

src/TopicModel/CTModel.hpp

@@ -243,6 +243,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, numBetaSample, numTMNSample, topicPrior);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, numBetaSample, numTMNSample, topicPrior);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, numBetaSample, numTMNSample, topicPrior);
 
 		CTModel(const CTArgs& args)
 			: BaseClass(args)

src/TopicModel/DMRModel.hpp

@@ -364,6 +364,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, sigma, alphaEps, metadataDict, lambda);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, sigma, alphaEps, metadataDict, lambda, multiMetadataDict);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, sigma, alphaEps, metadataDict, lambda, multiMetadataDict);
 
 		DMRModel(const DMRArgs& args)
 			: BaseClass(args), sigma(args.sigma), alphaEps(args.alphaEps)

src/TopicModel/DTModel.hpp

@@ -22,6 +22,7 @@ namespace tomoto
 		Eigen::Matrix<WeightType, -1, -1> numByTopicWord; // Dim: (Topic * Time, Vocabs)
 		//ShareableMatrix<WeightType, -1, -1> numByTopicWord; // Dim: (Topic * Time, Vocabs)
 		DEFINE_SERIALIZER(numByTopic, numByTopicWord);
+		DEFINE_HASHER(numByTopic, numByTopicWord);
 	};
 
 	template<TermWeight _tw, typename _RandGen,
@@ -496,6 +497,8 @@ namespace tomoto
 			T, shapeA, shapeB, shapeC, alphaVar, etaVar, phiVar, alphas, etaByDoc, phi);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, 
 			T, shapeA, shapeB, shapeC, alphaVar, etaVar, phiVar, alphas, etaByDoc, phi);
+		DEFINE_HASHER_AFTER_BASE(BaseClass,
+			T, shapeA, shapeB, shapeC, alphaVar, etaVar, phiVar, alphas, etaByDoc, phi);
 
 		GETTER(T, size_t, T);
 		GETTER(NumDocsByT, std::vector<uint32_t>, numDocsByTime);

src/TopicModel/GDMRModel.hpp

@@ -412,6 +412,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, sigma0, degreeByF, mdCoefs, mdIntercepts);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, sigma0, orderDecay, degreeByF, mdCoefs, mdIntercepts, mdMax);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, sigma0, orderDecay, degreeByF, mdCoefs, mdIntercepts, mdMax);
 
 		GDMRModel(const GDMRArgs& args)
 			: BaseClass(args), sigma0(args.sigma0), orderDecay(args.orderDecay), degreeByF(args.degrees)

src/TopicModel/HDPModel.hpp

@@ -19,6 +19,7 @@ namespace tomoto
 		size_t totalTable = 0;
 
 		DEFINE_SERIALIZER_AFTER_BASE(ModelStateLDA<_tw>, numTableByTopic, totalTable);
+		DEFINE_HASHER_AFTER_BASE(ModelStateLDA<_tw>, numTableByTopic, totalTable);
 	};
 
 	template<TermWeight _tw, typename _RandGen,
@@ -457,6 +458,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, gamma);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, gamma);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, gamma);
 
 		HDPModel(const HDPArgs& args)
 			: BaseClass(args), gamma(args.gamma)

src/TopicModel/HLDAModel.hpp

@@ -18,6 +18,7 @@ namespace tomoto
 			int32_t parent = 0, sibling = 0, child = 0;
 
 			DEFINE_SERIALIZER(numCustomers, level, parent, sibling, child);
+			DEFINE_HASHER(numCustomers, level, parent, sibling, child);
 
 			NCRPNode* getParent() const
 			{
@@ -118,6 +119,7 @@ namespace tomoto
 			Vector nodeWLikelihoods; //
 
 			DEFINE_SERIALIZER(nodes, levelBlocks);
+			DEFINE_HASHER(nodes, levelBlocks);
 
 			template<bool _makeNewPath = true>
 			void calcNodeLikelihood(Float gamma, size_t levelDepth)
@@ -317,6 +319,12 @@ namespace tomoto
 			ModelStateLDA<_tw>::serializerWrite(ostr);
 			nt->serializerWrite(ostr);
 		}
+
+		uint64_t computeHash(uint64_t seed) const
+		{
+			seed = ModelStateLDA<_tw>::computeHash(seed);
+			return nt->computeHash(seed);
+		}
 	};
 
 	template<TermWeight _tw, typename _RandGen,
@@ -596,6 +604,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, gamma);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, gamma);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, gamma);
 
 		HLDAModel(const HLDAArgs& args)
 			: BaseClass(args), gamma(args.gamma)

src/TopicModel/HPAModel.hpp

@@ -21,6 +21,7 @@ namespace tomoto
 		Eigen::Matrix<WeightType, -1, -1> numByTopic1_2;
 
 		DEFINE_SERIALIZER_AFTER_BASE(ModelStateLDA<_tw>, numByTopicWord, numByTopic, numByTopic1_2);
+		DEFINE_HASHER_AFTER_BASE(ModelStateLDA<_tw>, numByTopicWord, numByTopic, numByTopic1_2);
 	};
 
 	template<TermWeight _tw, typename _RandGen,
@@ -439,6 +440,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, K2, subAlphas, subAlphaSum);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, K2, subAlphas, subAlphaSum);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, K2, subAlphas, subAlphaSum);
 
 		HPAModel(const HPAArgs& args)
 			: BaseClass(args, false), K2(args.k2)

src/TopicModel/LDA.h

@@ -93,6 +93,12 @@ namespace tomoto
 			if (!ostr.write((const char*)this->data(), sizeof(_Scalar) * this->size()))
 				throw std::ios_base::failure(std::string("writing type '") + typeid(_Scalar).name() + std::string("' is failed"));
 		}
+
+		uint64_t computeHash(uint64_t seed) const
+		{
+			seed = serializer::computeHashMany(seed, (uint32_t)this->rows(), (uint32_t)this->cols());
+			return serializer::computeFastHash(this->data(), sizeof(_Scalar) * this->size(), seed);
+		}
 	};
 
 	template<typename _Base, TermWeight _tw>

src/TopicModel/LDAModel.hpp

@@ -65,6 +65,7 @@ namespace tomoto
 		//Eigen::Matrix<WeightType, -1, -1> numByTopicWord; // Dim: (Topic, Vocabs)
 		ShareableMatrix<WeightType, -1, -1> numByTopicWord; // Dim: (Topic, Vocabs)
 		DEFINE_SERIALIZER(numByTopic, numByTopicWord);
+		DEFINE_HASHER(numByTopic, numByTopicWord);
 	};
 
 	namespace flags
@@ -958,6 +959,8 @@ namespace tomoto
 		DEFINE_TAGGED_SERIALIZER_WITH_VERSION(1, 0x00010001, vocabWeights, alpha, alphas, eta, K, etaByWord,
 			burnIn, optimInterval);
 
+		DEFINE_HASHER(vocabWeights, alpha, alphas, eta, K, /*etaByWord,*/ burnIn, optimInterval);
+
 		LDAModel(const LDAArgs& args, bool checkAlpha = true)
 			: BaseClass(args.seed), K(args.k), alpha(args.alpha[0]), eta(args.eta)
 		{

src/TopicModel/LLDAModel.hpp

@@ -107,6 +107,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, topicLabelDict);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, topicLabelDict);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, topicLabelDict);
 
 		LLDAModel(const LDAArgs& args)
 			: BaseClass(args)

src/TopicModel/MGLDAModel.hpp

@@ -370,6 +370,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, alphaL, alphaM, alphaML, etaL, gamma, KL, T);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, alphaL, alphaM, alphaML, etaL, gamma, KL, T);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, alphaL, alphaM, alphaML, etaL, gamma, KL, T);
 
 		MGLDAModel(const MGLDAArgs& args)
 			: BaseClass(args), KL(args.kL), T(args.t),

src/TopicModel/PAModel.hpp

@@ -19,6 +19,7 @@ namespace tomoto
 		Vector subTmp;
 
 		DEFINE_SERIALIZER_AFTER_BASE(ModelStateLDA<_tw>, numByTopic1_2, numByTopic2);
+		DEFINE_HASHER_AFTER_BASE(ModelStateLDA<_tw>, numByTopic1_2, numByTopic2);
 	};
 
 	template<TermWeight _tw, typename _RandGen,
@@ -364,6 +365,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, K2, subAlphas, subAlphaSum);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, K2, subAlphas, subAlphaSum);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, K2, subAlphas, subAlphaSum);
 
 		PAModel(const PAArgs& args)
 			: BaseClass(args), K2(args.k2)

src/TopicModel/PLDAModel.hpp

@@ -111,6 +111,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, topicLabelDict, numLatentTopics, numTopicsPerLabel);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, topicLabelDict, numLatentTopics, numTopicsPerLabel);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, topicLabelDict, numLatentTopics, numTopicsPerLabel);
 
 		PLDAModel(const PLDAArgs& args)
 			: BaseClass(args.setK(1)),

src/TopicModel/PTModel.hpp

@@ -266,6 +266,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, numPDocs, lambda);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, numPDocs, lambda);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, numPDocs, lambda);
 
 		GETTER(P, size_t, numPDocs);
 

src/TopicModel/SLDAModel.hpp

@@ -348,6 +348,7 @@ namespace tomoto
 	public:
 		DEFINE_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 0, F, responseVars, mu, nuSq);
 		DEFINE_TAGGED_SERIALIZER_AFTER_BASE_WITH_VERSION(BaseClass, 1, 0x00010001, F, responseVars, mu, nuSq);
+		DEFINE_HASHER_AFTER_BASE(BaseClass, F, mu, nuSq);
 
 		SLDAModel(const SLDAArgs& args)
 			: BaseClass(args), F(args.vars.size()), varTypes(args.vars),