Fix POSTagTransformer, TextPreprocessingTransformer recipes (#203)

* `nltk.pos_tag` function expects a `list[str]` for the `token` param * Get the same behaviour as with nltk v3.4.3
h2oai · Jun 21, 2024 · deed860 · deed860
1 parent e1647fd
commit deed860
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 4 deletions.
diff --git a/transformers/nlp/text_pos_tagging_transformer.py b/transformers/nlp/text_pos_tagging_transformer.py
@@ -23,7 +23,7 @@ def set_tagger(self):
         nltk.download('averaged_perceptron_tagger', download_dir=nltk_data_path)
         try:
             self.pos_tagger = nltk.pos_tag
-            self.pos_tagger("test")
+            self.pos_tagger(list("test"))
         except LookupError:
             os.makedirs(nltk_data_path, exist_ok=True)
             os.makedirs(nltk_temp_path, exist_ok=True)
@@ -40,7 +40,7 @@ def set_tagger(self):
             self.atomic_copy(file1, tagger_path)
             self.atomic_copy(file2, tagger_path)
             self.pos_tagger = nltk.pos_tag
-            self.pos_tagger("test")
+            self.pos_tagger(list("test"))
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)

diff --git a/transformers/nlp/text_preprocessing_transformer.py b/transformers/nlp/text_preprocessing_transformer.py
@@ -65,7 +65,7 @@ def __init__(self, **kwargs):
                 self.lemmatizer = nltk.stem.WordNetLemmatizer()
                 self.pos_tagger = nltk.pos_tag
                 self.lemmatizer.lemmatize("test", wordnet.NOUN)
-                self.pos_tagger("test")
+                self.pos_tagger(list("test"))
             except LookupError:
                 os.makedirs(nltk_data_path, exist_ok=True)
                 os.makedirs(nltk_temp_path, exist_ok=True)
@@ -92,7 +92,7 @@ def __init__(self, **kwargs):
                 self.lemmatizer = nltk.stem.WordNetLemmatizer()
                 self.pos_tagger = nltk.pos_tag
                 self.lemmatizer.lemmatize("test", wordnet.NOUN)
-                self.pos_tagger("test")
+                self.pos_tagger(list("test"))
             self.wordnet_map = {"N": wordnet.NOUN,
                                 "V": wordnet.VERB,
                                 "J": wordnet.ADJ,