From c6776a37948e080898a95b55b1d772da97d9e99d Mon Sep 17 00:00:00 2001 From: vTuanpham Date: Tue, 14 Nov 2023 01:11:54 +0700 Subject: [PATCH] chore: add more test cases --- tests/eli5_test.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/eli5_test.py b/tests/eli5_test.py index 6ab5461..ef97fa0 100644 --- a/tests/eli5_test.py +++ b/tests/eli5_test.py @@ -1,6 +1,9 @@ import os import unittest import warnings + +from datasets import load_dataset + from examples.ELI5.ELI5_10docs_Parser import ELI5Val @@ -28,6 +31,17 @@ def step4(self): self.assertTrue(os.path.exists(self.output_path), f"File '{self.output_path}' does not exist") def step5(self): + try: + self.translated_dataset = load_dataset("json", self.output_path, keep_in_memory=False) + except Exception as e: + raise SyntaxError("Invalid syntax for save function, the data output must be in the form of" + f"line-delimited json,\n Error message: {e}") + + def step6(self): + self.assertEqual(len(self.translated_dataset['train']), len(self.parser.converted_data), + "The parsed translated dataset does not match the length of the parsed dataset") + + def step7(self): if os.path.exists(self.output_path): os.remove(self.output_path)