ss_not_open.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# import packages "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import dask.dataframe as dd\n",
    " \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# import OpenAIRE's DOIs\n",
    "(need a CSV with all OpenAIRES's DOIs, named 'all_dois.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_dois =  dd.read_csv('all_dois.csv', sep='\\t')\n",
    "all_dois['doi'] = all_dois['doi'].str.lower()\n",
    "all_dois['doi'] = all_dois['doi'].map_partitions(lambda part: part.str.strip())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# import Semantic Scholar's DOIs\n",
    "(create a subfolder named 'ss_unique_publications' and save there the Semantic Scholar's Parquet files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "subfolder_path = 'ss_unique_publications'\n",
    "\n",
    "file_names1 = os.listdir(subfolder_path)\n",
    "file_names = [file for file in file_names1 if file[0] != '.']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "99"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(file_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# construct a dask dataframe with Semantic Scholar's data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dois_df = []\n",
    "\n",
    "for file in file_names:\n",
    "    df = dd.read_parquet(subfolder_path+'/'+file)\n",
    "    df = df.dropna(subset=['doi'])\n",
    "    df = df.reset_index()\n",
    "    df['doi'] = df['doi'].str.lower()\n",
    "    df['doi'] = df['doi'].map_partitions(lambda part: part.str.strip())\n",
    "\n",
    "    df = df[['doi','title']]\n",
    "\n",
    "    dois_df.append(df)\n",
    "    \n",
    "ss_df = dd.concat(dois_df)\n",
    "ss_df = ss_df.reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# find which DOIs are in Semantic Scholar but not in OpenAIRE\n",
    "(it will need some time to run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "common_df = dd.merge(ss_df, all_dois, on = 'doi')\n",
    "\n",
    "common_dois_set = set(common_df['doi'].compute()) \n",
    "not_in_common = ss_df[~ss_df['doi'].isin(common_dois_set)]\n",
    "not_common_dois_set = set(not_in_common['doi'].compute())\n",
    "with open(\"ss_not_open.txt\", \"w\") as file:\n",
    "    # Iterate through the list\n",
    "        for item in not_common_dois_set:\n",
    "            # Write each item to the file\n",
    "            file.write(item + \"\\n\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4767729"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(not_common_dois_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}