Merge pull request #9 from ror-community/20231110_data_update

Updating data and mapping script for Zenodo changes
ror-community · Nov 11, 2023 · a8905a8 · a8905a8
2 parents 9c3d6c4 + 58f3c31
commit a8905a8
Show file tree

Hide file tree

Showing 12 changed files with 63,069 additions and 31,317 deletions.
diff --git a/data/aggregate_mapped.csv b/data/aggregate_mapped.csv
diff --git a/data/aggregate_unmapped.csv b/data/aggregate_unmapped.csv
diff --git a/data/crossref_funders.json b/data/crossref_funders.json
diff --git a/data/crossref_overlap.png b/data/crossref_overlap.png
diff --git a/data/datacite_funders.json b/data/datacite_funders.json
diff --git a/data/datacite_overlap.png b/data/datacite_overlap.png
diff --git a/data/funders.json b/data/funders.json
diff --git a/data/members.json b/data/members.json
diff --git a/data/ror_funder_registry_mapping.json b/data/ror_funder_registry_mapping.json
diff --git a/main.py b/main.py
@@ -11,9 +11,9 @@
     "DataCite - Aggregrate overlap": DataCite_view
 }
 
-funder_registry_version = '1.52'
-ror_registry_version = '1.33'
-works_count_date = '2023/09/28'
+funder_registry_version = '1.54'
+ror_registry_version = '1.36'
+works_count_date = '2023/11/11'
 
 def main():
     sidebar_title = st.sidebar.title("Views")

diff --git a/requirements.txt b/requirements.txt
@@ -1,5 +1,6 @@
 altair==5.0.1
 attrs==23.1.0
+beautifulsoup4==4.12.2
 blinker==1.6.2
 cachetools==5.3.1
 certifi==2023.5.7
@@ -13,9 +14,11 @@ gitdb==4.0.10
 GitPython==3.1.31
 idna==3.4
 importlib-metadata==6.7.0
+importlib-resources==6.1.1
 Jinja2==3.1.2
 jsonschema==4.17.3
 kiwisolver==1.4.4
+lxml==4.9.3
 markdown-it-py==3.0.0
 MarkupSafe==2.1.3
 matplotlib==3.7.1
@@ -38,6 +41,7 @@ requests==2.31.0
 rich==13.4.2
 six==1.16.0
 smmap==5.0.0
+soupsieve==2.5
 streamlit==1.24.0
 tabulate==0.9.0
 tenacity==8.2.2

diff --git a/utilities/create_funder_id_mapping_w_dl.py b/utilities/create_funder_id_mapping_w_dl.py
@@ -6,7 +6,12 @@
 import os
 
 
+import os
+import requests
+import zipfile
+
 def download_and_unzip(record_id, path='.'):
+    # Downloading the record from Zenodo
     response = requests.get(f'https://zenodo.org/api/records/{record_id}')
     record = response.json()
     download_link = record['files'][0]['links']['self']
@@ -16,9 +21,14 @@ def download_and_unzip(record_id, path='.'):
     with open(file_path, 'wb') as out_file:
         for chunk in response.iter_content(chunk_size=1024):
             out_file.write(chunk)
+    extracted_file_names = []
     with zipfile.ZipFile(file_path, 'r') as zip_ref:
         zip_ref.extractall(path)
-    return file_name.split('.zip')[0]
+        extracted_file_names = zip_ref.namelist()
+    if extracted_file_names:
+        return os.path.splitext(extracted_file_names[0])[0]
+    return None
+
 
 
 def create_mapping_and_output_json(ror_data_file, json_output_file):