* set the default encoding of stdout to utf-8

modelscope · Dec 30, 2024 · 8981b63 · 8981b63
1 parent d464b79
commit 8981b63
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 4 deletions.
diff --git a/data_juicer/ops/mapper/nlpcda_zh_mapper.py b/data_juicer/ops/mapper/nlpcda_zh_mapper.py
@@ -1,4 +1,3 @@
-import sys
 from copy import deepcopy
 
 from loguru import logger
@@ -9,9 +8,6 @@
 
 from ..base_op import OPERATORS, Mapper
 
-# set the default encoding to utf-8 explicitly
-sys.stdout.reconfigure(encoding='utf-8')
-
 nlpcda = LazyLoader('nlpcda', 'nlpcda')
 
 OP_NAME = 'nlpcda_zh_mapper'

diff --git a/tests/ops/mapper/test_nlpcda_zh_mapper.py b/tests/ops/mapper/test_nlpcda_zh_mapper.py
@@ -13,6 +13,9 @@
 class NlpaugEnMapperTest(DataJuicerTestCaseBase):
 
     def setUp(self):
+        import sys
+        # set the default encoding to utf-8 explicitly
+        sys.stdout.reconfigure(encoding='utf-8')
         self.samples = Dataset.from_dict({
             'text': ['这里一共有5种不同的数据增强方法', '这是不带数字的测试样例'],
             'meta': ['meta information', 'meta information without numbers'],