Handle records with dict rather than array affil.

Issue: #51
dspinellis · Jun 30, 2024 · c5b040f · c5b040f
1 parent 13286a0
commit c5b040f
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 13 deletions.
diff --git a/src/alexandria3k/data_sources/datacite.py b/src/alexandria3k/data_sources/datacite.py
@@ -72,6 +72,13 @@ def current_row_value(self):
                 self.item_index
             ]
             self.json_data = json.loads(json_string)
+            # Record 10.17031/637b5e4a8d3ae of file10.17031/part_00001.jsonl
+            # and others have affiliation as a dict, rather than an array
+            # containing a dict.  Detect and fix.
+            for relation in ["creators", "contributors"]:
+                for creator in self.json_data[relation]:
+                    if isinstance(creator.get("affiliation"), dict):
+                        creator["affiliation"] = [creator["affiliation"]]
             self.cached_json_item_index = self.item_index
         return self.json_data
 

diff --git a/tests/data/datacite.tar.gz b/tests/data/datacite.tar.gz
diff --git a/tests/data_sources/test_datacite.py b/tests/data_sources/test_datacite.py
@@ -50,19 +50,19 @@ def tearDownClass(cls):
         os.unlink(DATABASE_PATH)
 
     def test_counts(self):
-        self.assertEqual(self.record_count("dc_works"), 7)
-        self.assertEqual(self.record_count("dc_work_creators"), 26)
+        self.assertEqual(self.record_count("dc_works"), 9)
+        self.assertEqual(self.record_count("dc_work_creators"), 29)
         self.assertEqual(self.record_count("dc_creator_name_identifiers"), 8)
-        self.assertEqual(self.record_count("dc_creator_affiliations"), 14)
-        self.assertEqual(self.record_count("dc_work_titles"), 7)
-        self.assertEqual(self.record_count("dc_work_subjects"), 25)
-        self.assertEqual(self.record_count("dc_work_contributors"), 0)
+        self.assertEqual(self.record_count("dc_creator_affiliations"), 17)
+        self.assertEqual(self.record_count("dc_work_titles"), 9)
+        self.assertEqual(self.record_count("dc_work_subjects"), 27)
+        self.assertEqual(self.record_count("dc_work_contributors"), 1)
         self.assertEqual(self.record_count("dc_contributor_name_identifiers"), 0)
-        self.assertEqual(self.record_count("dc_contributor_affiliations"), 0)
+        self.assertEqual(self.record_count("dc_contributor_affiliations"), 1)
         self.assertEqual(self.record_count("dc_work_dates"), 7)
         self.assertEqual(self.record_count("dc_work_related_identifiers"), 20)
-        self.assertEqual(self.record_count("dc_work_descriptions"), 10)
-        self.assertEqual(self.record_count("dc_work_geo_locations"), 0)
+        self.assertEqual(self.record_count("dc_work_descriptions"), 12)
+        self.assertEqual(self.record_count("dc_work_geo_locations"), 2)
         self.assertEqual(self.record_count("dc_work_funding_references"), 2)
 
 
@@ -79,13 +79,13 @@ def test_counts(self):
                 """(SELECT DISTINCT work_id
           FROM dc_work_creators)"""
             ),
-            7,
+            9,
         )
 
         self.assertEqual(self.record_count(
                 """(SELECT DISTINCT container_id FROM dc_works)"""
             ),
-            3,
+            4,
         )
 
     def test_work_contents(self):
@@ -229,7 +229,7 @@ def tearDownClass(cls):
         os.unlink(DATABASE_PATH)
 
     def test_counts(self):
-        self.assertEqual(self.record_count("dc_works"), 7)
+        self.assertEqual(self.record_count("dc_works"), 9)
 
     def test_no_extra_fields(self):
         with self.assertRaises(sqlite3.OperationalError):
@@ -262,7 +262,7 @@ def tearDownClass(cls):
         os.unlink(DATABASE_PATH)
 
     def test_counts(self):
-        self.assertEqual(self.record_count("dc_works"), 4)
+        self.assertEqual(self.record_count("dc_works"), 6)
 
     def test_no_extra_fields(self):
         with self.assertRaises(sqlite3.OperationalError):