Simplify curated paper loading

src/bioregistry/analysis/paper_ranking.py

@@ -65,60 +65,40 @@ def load_bioregistry_json(file_path):
     return pd.DataFrame(publications)
 
 
-def load_curated_pmids(file_path):
-    """Load curated papers data from TSV file, extracting PMIDs in a list.
-
-    :param file_path: Path to the curated_papers.tsv file.
-    :type file_path: str
-    :return: List containing already curated PMIDs
-    :rtype: list
-    """
-    try:
-        curated_papers_df = pd.read_csv(file_path, sep="\t")
-        return curated_papers_df["pubmed"].tolist()
-    except FileNotFoundError:
-        click.echo(f"Could not find file {file_path}.")
-        return list()
-
-
-def load_curated_papers(file_path):
+def load_curated_papers(file_path=CURATED_PAPERS_PATH):
     """Load curated papers data from TSV file, and fetch titles and abstracts for PMIDs.
 
     :param file_path: Path to the curated_papers.tsv file.
     :type file_path: str
     :return: DataFrame containing curated publication details.
     :rtype: pd.DataFrame
     """
-    try:
-        curated_df = pd.read_csv(file_path, sep="\t")
-        curated_df = curated_df.rename(columns={"pmid": "pubmed", "relevant": "label"})
-        curated_df["title"] = ""
-        curated_df["abstract"] = ""
-
-        pmids_to_fetch = curated_df["pubmed"].tolist()
-        fetched_metadata = {}
-        for chunk in [pmids_to_fetch[i : i + 200] for i in range(0, len(pmids_to_fetch), 200)]:
-            fetched_metadata.update(pubmed_client.get_metadata_for_ids(chunk, get_abstracts=True))
-
-        for index, row in curated_df.iterrows():
-            if row["pubmed"] in fetched_metadata:
-                curated_df.at[index, "title"] = fetched_metadata[row["pubmed"]].get("title", "")
-                curated_df.at[index, "abstract"] = fetched_metadata[row["pubmed"]].get(
-                    "abstract", ""
-                )
+    curated_df = pd.read_csv(file_path, sep="\t")
+    curated_df = curated_df.rename(columns={"pmid": "pubmed", "relevant": "label"})
+    curated_df["title"] = ""
+    curated_df["abstract"] = ""
 
-        click.echo(f"Got {len(curated_df)} curated publications from the curated_papers.tsv file")
-        return curated_df
-    except FileNotFoundError:
-        click.echo(f"Could not find file {file_path}.")
-        return pd.DataFrame()
+    pmids_to_fetch = curated_df["pubmed"].tolist()
+    fetched_metadata = {}
+    for chunk in [pmids_to_fetch[i : i + 200] for i in range(0, len(pmids_to_fetch), 200)]:
+        fetched_metadata.update(pubmed_client.get_metadata_for_ids(chunk, get_abstracts=True))
+
+    for index, row in curated_df.iterrows():
+        if row["pubmed"] in fetched_metadata:
+            curated_df.at[index, "title"] = fetched_metadata[row["pubmed"]].get("title", "")
+            curated_df.at[index, "abstract"] = fetched_metadata[row["pubmed"]].get(
+                "abstract", ""
+            )
+
+    click.echo(f"Got {len(curated_df)} curated publications from the curated_papers.tsv file")
+    return curated_df
 
 
 def fetch_pubmed_papers(curated_pmids):
     """Fetch PubMed papers from the last 30 days using specific search terms, excluding curated papers.
 
     :param curated_pmids: List containing already curated PMIDs
-    :type curated_pmids: list
+    :type curated_pmids: Iterable
     :return: DataFrame containing PubMed paper details.
     :rtype: pd.DataFrame
     """
@@ -410,7 +390,8 @@ def main(bioregistry_file, start_date, end_date):
     click.echo(f"Writing feature (word) importances to {importance_path}")
     importances_df.to_csv(importance_path, sep="\t", index=False)
 
-    curated_pmids = load_curated_pmids(CURATED_PAPERS_PATH)
+    # These have already been curated and will therefore be filtered out
+    curated_pmids = set(curated_papers_df["pubmed"])
 
     new_pub_df = fetch_pubmed_papers(curated_pmids)
     if not new_pub_df.empty: