* set the default encoding of stdout to utf-8

modelscope · Dec 30, 2024 · d464b79 · d464b79
1 parent 6020a41
commit d464b79
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/data_juicer/__init__.py b/data_juicer/__init__.py
@@ -10,9 +10,6 @@
 
 from data_juicer.utils.availability_utils import _is_package_available
 
-# set the default encoding to utf-8 explicitly
-sys.stdout.reconfigure(encoding='utf-8')
-
 ImageFile.LOAD_TRUNCATED_IMAGES = True
 
 # For now, only INFO will be shown. Later the severity level will be changed

diff --git a/data_juicer/ops/mapper/nlpcda_zh_mapper.py b/data_juicer/ops/mapper/nlpcda_zh_mapper.py
@@ -1,3 +1,4 @@
+import sys
 from copy import deepcopy
 
 from loguru import logger
@@ -8,6 +9,9 @@
 
 from ..base_op import OPERATORS, Mapper
 
+# set the default encoding to utf-8 explicitly
+sys.stdout.reconfigure(encoding='utf-8')
+
 nlpcda = LazyLoader('nlpcda', 'nlpcda')
 
 OP_NAME = 'nlpcda_zh_mapper'