havard_difference.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f0275d9d-0e5f-4840-bd20-3025748ed37d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6de4d707-e3f2-4b64-a1a8-793f1461a74f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ASA Botanist ID</th>\n",
       "      <th>Author Teams:</th>\n",
       "      <th>Author name</th>\n",
       "      <th>B &amp; P Author Abbrev.</th>\n",
       "      <th>Citation as Author</th>\n",
       "      <th>Citation as Collector</th>\n",
       "      <th>Collections in</th>\n",
       "      <th>Collector Teams:</th>\n",
       "      <th>labelName</th>\n",
       "      <th>GUID</th>\n",
       "      <th>...</th>\n",
       "      <th>deathYear</th>\n",
       "      <th>deathYearIsApprox</th>\n",
       "      <th>geographyCombined</th>\n",
       "      <th>geographyISO</th>\n",
       "      <th>title</th>\n",
       "      <th>firstName</th>\n",
       "      <th>middleName</th>\n",
       "      <th>lastName</th>\n",
       "      <th>herbariaCode</th>\n",
       "      <th>institutionUUID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>127603 botanist</td>\n",
       "      <td>L. T. Lu &amp; BrachChandani Gupta, S. S. Dash &amp; A...</td>\n",
       "      <td>Brach</td>\n",
       "      <td>Brach</td>\n",
       "      <td>Novon 12(4): 495. 2002</td>\n",
       "      <td>pers. comm.</td>\n",
       "      <td>1977 (34),2012 (1),2015 (51),2016 (13),2017 (2...</td>\n",
       "      <td>L. T. Lu &amp; BrachA. R. Brach &amp; J. M. BrachA. R....</td>\n",
       "      <td>Brach, Anthony Robert</td>\n",
       "      <td>http://purl.oclc.org/net/edu.harvard.huh/guid/...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States of America|United States of America</td>\n",
       "      <td>['US']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Anthony</td>\n",
       "      <td>Robert</td>\n",
       "      <td>Brach</td>\n",
       "      <td>{'SYRF'}</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>114612 botanist</td>\n",
       "      <td>W. R. Buck &amp; B. C. TanNoguchi &amp; B. C. TanB. C....</td>\n",
       "      <td>B. C. Tan</td>\n",
       "      <td>B.C.Tan</td>\n",
       "      <td>['Authors of Plant Names. Royal Botanic Garden...</td>\n",
       "      <td>['Index Herbariorum - Collectors T-Z', 'Collec...</td>\n",
       "      <td>[no date] (3),1974 (1),1975 (2),1977 (1),1984 ...</td>\n",
       "      <td>B. C. Tan, E. A. Brown &amp; R. G. CovenyB. C. Tan...</td>\n",
       "      <td>Tan, Benito Ching</td>\n",
       "      <td>http://purl.oclc.org/net/edu.harvard.huh/guid/...</td>\n",
       "      <td>...</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>False</td>\n",
       "      <td>Philippines,  Singapore,  United States of Ame...</td>\n",
       "      <td>['PH', 'SG', 'CN', 'MY', 'US', 'ID']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Benito</td>\n",
       "      <td>Ching</td>\n",
       "      <td>Tan</td>\n",
       "      <td>{'UBC', 'H', 'SINU', 'B', 'FH'}</td>\n",
       "      <td>['5d6c8b57-9bfb-46e8-960f-f57343ca9729', '4cf9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>125914 botanist</td>\n",
       "      <td>Ormerod &amp; E.W.Wood</td>\n",
       "      <td>E. W. Wood</td>\n",
       "      <td>E.W.Wood</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Index Herbariorum - Collectors T-Z</td>\n",
       "      <td>1974 (2),1976 (1),1979 (6),1980 (1),1982 (5),1...</td>\n",
       "      <td>D. E. Boufford &amp; E. W. WoodD. E. Boufford, V. ...</td>\n",
       "      <td>Wood, Emily Walker</td>\n",
       "      <td>http://purl.oclc.org/net/edu.harvard.huh/guid/...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States of America|Japan</td>\n",
       "      <td>['US', 'JP']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Emily</td>\n",
       "      <td>Walker</td>\n",
       "      <td>Wood</td>\n",
       "      <td>{'TI', 'NCU', 'KYO', 'PE', 'CM', 'MO', 'A', 'C...</td>\n",
       "      <td>['ac8eb8fc-0703-4408-8388-686764b75877', 'ea90...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>144144 botanist</td>\n",
       "      <td>Garay &amp; W. KittredgeP. T. Li &amp; W. Kittredge</td>\n",
       "      <td>W. Kittredge</td>\n",
       "      <td>W. Kittr.</td>\n",
       "      <td>['Authors of Plant Names. Royal Botanic Garden...</td>\n",
       "      <td>Index Herbariorum, ed. 8</td>\n",
       "      <td>[no date] (6),1973 (1),1974 (18),1975 (8),1981...</td>\n",
       "      <td>Garay &amp; W. KittredgeP. T. Li &amp; W. KittredgeW. ...</td>\n",
       "      <td>Kittredge, Walter Tobey</td>\n",
       "      <td>http://purl.oclc.org/net/edu.harvard.huh/guid/...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States of America|United States of America</td>\n",
       "      <td>['US']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Walter</td>\n",
       "      <td>Tobey</td>\n",
       "      <td>Kittredge</td>\n",
       "      <td>{'NEBC', 'GH', 'AMES'}</td>\n",
       "      <td>['648a5b5f-0421-44fd-8dd3-82add535bc54', 'd56b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>126030 botanist</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Index Herbariorum, ed. 8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M. A. Sherwood, D. T. Plas, D. Gregory &amp; G. A....</td>\n",
       "      <td>Cacavio, Gennaro J.</td>\n",
       "      <td>http://purl.oclc.org/net/edu.harvard.huh/guid/...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States of America</td>\n",
       "      <td>['US']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Gennaro</td>\n",
       "      <td>J.</td>\n",
       "      <td>Cacavio</td>\n",
       "      <td>{'GH'}</td>\n",
       "      <td>['648a5b5f-0421-44fd-8dd3-82add535bc54']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 54 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   ASA Botanist ID                                      Author Teams:  \\\n",
       "0  127603 botanist  L. T. Lu & BrachChandani Gupta, S. S. Dash & A...   \n",
       "1  114612 botanist  W. R. Buck & B. C. TanNoguchi & B. C. TanB. C....   \n",
       "2  125914 botanist                                 Ormerod & E.W.Wood   \n",
       "3  144144 botanist        Garay & W. KittredgeP. T. Li & W. Kittredge   \n",
       "4  126030 botanist                                                NaN   \n",
       "\n",
       "    Author name B & P Author Abbrev.  \\\n",
       "0         Brach                Brach   \n",
       "1     B. C. Tan              B.C.Tan   \n",
       "2    E. W. Wood             E.W.Wood   \n",
       "3  W. Kittredge            W. Kittr.   \n",
       "4           NaN                  NaN   \n",
       "\n",
       "                                  Citation as Author  \\\n",
       "0                             Novon 12(4): 495. 2002   \n",
       "1  ['Authors of Plant Names. Royal Botanic Garden...   \n",
       "2                                                NaN   \n",
       "3  ['Authors of Plant Names. Royal Botanic Garden...   \n",
       "4                                                NaN   \n",
       "\n",
       "                               Citation as Collector  \\\n",
       "0                                        pers. comm.   \n",
       "1  ['Index Herbariorum - Collectors T-Z', 'Collec...   \n",
       "2                 Index Herbariorum - Collectors T-Z   \n",
       "3                           Index Herbariorum, ed. 8   \n",
       "4                           Index Herbariorum, ed. 8   \n",
       "\n",
       "                                      Collections in  \\\n",
       "0  1977 (34),2012 (1),2015 (51),2016 (13),2017 (2...   \n",
       "1  [no date] (3),1974 (1),1975 (2),1977 (1),1984 ...   \n",
       "2  1974 (2),1976 (1),1979 (6),1980 (1),1982 (5),1...   \n",
       "3  [no date] (6),1973 (1),1974 (18),1975 (8),1981...   \n",
       "4                                                NaN   \n",
       "\n",
       "                                    Collector Teams:                labelName  \\\n",
       "0  L. T. Lu & BrachA. R. Brach & J. M. BrachA. R....    Brach, Anthony Robert   \n",
       "1  B. C. Tan, E. A. Brown & R. G. CovenyB. C. Tan...        Tan, Benito Ching   \n",
       "2  D. E. Boufford & E. W. WoodD. E. Boufford, V. ...       Wood, Emily Walker   \n",
       "3  Garay & W. KittredgeP. T. Li & W. KittredgeW. ...  Kittredge, Walter Tobey   \n",
       "4  M. A. Sherwood, D. T. Plas, D. Gregory & G. A....      Cacavio, Gennaro J.   \n",
       "\n",
       "                                                GUID  ... deathYear  \\\n",
       "0  http://purl.oclc.org/net/edu.harvard.huh/guid/...  ...       NaN   \n",
       "1  http://purl.oclc.org/net/edu.harvard.huh/guid/...  ...    2016.0   \n",
       "2  http://purl.oclc.org/net/edu.harvard.huh/guid/...  ...       NaN   \n",
       "3  http://purl.oclc.org/net/edu.harvard.huh/guid/...  ...       NaN   \n",
       "4  http://purl.oclc.org/net/edu.harvard.huh/guid/...  ...       NaN   \n",
       "\n",
       "  deathYearIsApprox                                  geographyCombined  \\\n",
       "0               NaN  United States of America|United States of America   \n",
       "1             False  Philippines,  Singapore,  United States of Ame...   \n",
       "2               NaN                     United States of America|Japan   \n",
       "3               NaN  United States of America|United States of America   \n",
       "4               NaN                           United States of America   \n",
       "\n",
       "                           geographyISO title firstName middleName   lastName  \\\n",
       "0                                ['US']   NaN   Anthony     Robert      Brach   \n",
       "1  ['PH', 'SG', 'CN', 'MY', 'US', 'ID']   NaN    Benito      Ching        Tan   \n",
       "2                          ['US', 'JP']   NaN     Emily     Walker       Wood   \n",
       "3                                ['US']   NaN    Walter      Tobey  Kittredge   \n",
       "4                                ['US']   NaN   Gennaro         J.    Cacavio   \n",
       "\n",
       "                                        herbariaCode  \\\n",
       "0                                           {'SYRF'}   \n",
       "1                    {'UBC', 'H', 'SINU', 'B', 'FH'}   \n",
       "2  {'TI', 'NCU', 'KYO', 'PE', 'CM', 'MO', 'A', 'C...   \n",
       "3                             {'NEBC', 'GH', 'AMES'}   \n",
       "4                                             {'GH'}   \n",
       "\n",
       "                                     institutionUUID  \n",
       "0                                                 []  \n",
       "1  ['5d6c8b57-9bfb-46e8-960f-f57343ca9729', '4cf9...  \n",
       "2  ['ac8eb8fc-0703-4408-8388-686764b75877', 'ea90...  \n",
       "3  ['648a5b5f-0421-44fd-8dd3-82add535bc54', 'd56b...  \n",
       "4           ['648a5b5f-0421-44fd-8dd3-82add535bc54']  \n",
       "\n",
       "[5 rows x 54 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file1 = '~/Documents/coding/PKB-main/PKB-main/data/intermediate/harvard-index/collectors.csv'  ## macOS file location, harvard index collector\n",
    "harvard_data = pd.read_csv(file1,chunksize=10000,encoding='utf-8',on_bad_lines='skip',engine='python')\n",
    "harvard_data = pd.concat(harvard_data)\n",
    "harvard_columns = harvard_data.columns.tolist()\n",
    "harvard_data.rename(columns={harvard_columns[8]:'labelName'}, inplace=True) ## to avoid special character in colNames\n",
    "\n",
    "harvard_columns = harvard_data.columns.tolist()   ## load content into list\n",
    "harvard_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e3ddd729-c0d2-4afe-81f6-7541f09a266c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "76391"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(harvard_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "16809594-7768-4da2-a621-0acc67d1ac92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ASA Botanist ID',\n",
       " 'Author Teams:',\n",
       " 'Author name',\n",
       " 'B & P Author Abbrev.',\n",
       " 'Citation as Author',\n",
       " 'Citation as Collector',\n",
       " 'Collections in',\n",
       " 'Collector Teams:',\n",
       " 'labelName',\n",
       " 'GUID',\n",
       " 'Geography Author',\n",
       " 'Geography Collector',\n",
       " 'Holdings',\n",
       " 'Member of teams/groups',\n",
       " 'Name',\n",
       " 'Remarks',\n",
       " 'Specialty Author',\n",
       " 'Specialty Collector',\n",
       " 'Standard/Label Name',\n",
       " 'Start Date',\n",
       " 'asa_category',\n",
       " 'id',\n",
       " 'Date of birth',\n",
       " 'Date of death',\n",
       " 'Author in',\n",
       " 'First date flourished',\n",
       " 'Geography Determiner',\n",
       " 'Specialty Determiner',\n",
       " 'URL',\n",
       " 'End Date',\n",
       " 'Last date flourished',\n",
       " 'Vernacular name',\n",
       " 'First date collected',\n",
       " 'Last date collected',\n",
       " 'Image',\n",
       " 'First date received',\n",
       " 'Last date received',\n",
       " 'Citation as Determiner',\n",
       " 'Agent type',\n",
       " 'Team Members:',\n",
       " 'Geography',\n",
       " 'Specialty',\n",
       " 'birthYear',\n",
       " 'birthYearIsApprox',\n",
       " 'deathYear',\n",
       " 'deathYearIsApprox',\n",
       " 'geographyCombined',\n",
       " 'geographyISO',\n",
       " 'title',\n",
       " 'firstName',\n",
       " 'middleName',\n",
       " 'lastName',\n",
       " 'herbariaCode',\n",
       " 'institutionUUID']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "harvard_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "77368c98-1d84-4576-98e6-a7a1281935a3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>Standard/Label Name</th>\n",
       "      <th>birthYear</th>\n",
       "      <th>deathYear</th>\n",
       "      <th>birthYearIsApprox</th>\n",
       "      <th>geographyISO</th>\n",
       "      <th>firstName</th>\n",
       "      <th>middleName</th>\n",
       "      <th>lastName</th>\n",
       "      <th>B &amp; P Author Abbrev.</th>\n",
       "      <th>Name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>89438</td>\n",
       "      <td>Shin. Sato</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DE</td>\n",
       "      <td>Shinya</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sato</td>\n",
       "      <td>Shin. Sato</td>\n",
       "      <td>Sato, Shinya, Shinya Sato</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>64680</td>\n",
       "      <td>E. J. Hoffman</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Emily</td>\n",
       "      <td>J.</td>\n",
       "      <td>Hoffman</td>\n",
       "      <td>E. J. Hoffman</td>\n",
       "      <td>Hoffman, Emily J., Emily J. Hoffman</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>34653</td>\n",
       "      <td>D. W. Stevenson</td>\n",
       "      <td>1942.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>US</td>\n",
       "      <td>Dennis</td>\n",
       "      <td>William</td>\n",
       "      <td>Stevenson</td>\n",
       "      <td>D. W. Stev.</td>\n",
       "      <td>Stevenson, Dennis William, Dennis William Stev...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>42819</td>\n",
       "      <td>D. Müller-Doblies</td>\n",
       "      <td>1938.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>DE</td>\n",
       "      <td>Dietrich</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Müller-Doblies</td>\n",
       "      <td>D. Müll.-Doblies</td>\n",
       "      <td>Müller-Doblies, Dietrich, Dietrich Müller-Do...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>82862</td>\n",
       "      <td>Boutroux</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Boutroux</td>\n",
       "      <td>Boutroux</td>\n",
       "      <td>Boutroux, A., A. Boutroux</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id Standard/Label Name  birthYear  deathYear birthYearIsApprox  \\\n",
       "0  89438          Shin. Sato        NaN        NaN               NaN   \n",
       "1  64680       E. J. Hoffman        NaN        NaN               NaN   \n",
       "2  34653     D. W. Stevenson     1942.0        NaN             False   \n",
       "3  42819  D. Müller-Doblies     1938.0        NaN             False   \n",
       "4  82862            Boutroux        NaN        NaN               NaN   \n",
       "\n",
       "  geographyISO firstName middleName         lastName B & P Author Abbrev.  \\\n",
       "0           DE    Shinya        NaN             Sato           Shin. Sato   \n",
       "1          NaN     Emily         J.          Hoffman        E. J. Hoffman   \n",
       "2           US    Dennis    William        Stevenson          D. W. Stev.   \n",
       "3           DE  Dietrich        NaN  Müller-Doblies    D. Müll.-Doblies   \n",
       "4          NaN        A.        NaN         Boutroux             Boutroux   \n",
       "\n",
       "                                                Name  \n",
       "0                          Sato, Shinya, Shinya Sato  \n",
       "1                Hoffman, Emily J., Emily J. Hoffman  \n",
       "2  Stevenson, Dennis William, Dennis William Stev...  \n",
       "3  Müller-Doblies, Dietrich, Dietrich Müller-Do...  \n",
       "4                          Boutroux, A., A. Boutroux  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file = '~/Documents/coding/pkb/b.csv'  ## macOS file location, harvard index collector\n",
    "b = pd.read_csv(file,chunksize=10000,encoding='utf-8',on_bad_lines='skip',engine='python')\n",
    "b = pd.concat(b)\n",
    "b_columns = b.columns.tolist()\n",
    "b.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3e59bf6a-dd44-4c28-a511-a5f8e7784229",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "76496"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9f479755-22b9-4ecf-8b90-7218d985fe39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{2404,\n",
       " 3696,\n",
       " 3757,\n",
       " 3898,\n",
       " 4531,\n",
       " 4880,\n",
       " 5776,\n",
       " 5902,\n",
       " 6781,\n",
       " 7312,\n",
       " 7451,\n",
       " 7899,\n",
       " 8591,\n",
       " 8603,\n",
       " 8690,\n",
       " 9626,\n",
       " 10161,\n",
       " 10470,\n",
       " 11364,\n",
       " 11598,\n",
       " 11882,\n",
       " 12534,\n",
       " 12661,\n",
       " 12914,\n",
       " 13006,\n",
       " 14298,\n",
       " 14363,\n",
       " 14448,\n",
       " 15369,\n",
       " 16332,\n",
       " 17798,\n",
       " 19095,\n",
       " 19261,\n",
       " 19761,\n",
       " 20197,\n",
       " 20488,\n",
       " 20747,\n",
       " 21656,\n",
       " 21763,\n",
       " 22270,\n",
       " 23383,\n",
       " 24707,\n",
       " 24818,\n",
       " 25218,\n",
       " 27013,\n",
       " 27927,\n",
       " 29278,\n",
       " 29870,\n",
       " 30892,\n",
       " 30956,\n",
       " 34177,\n",
       " 34870,\n",
       " 38348,\n",
       " 39242,\n",
       " 39496,\n",
       " 39841,\n",
       " 39953,\n",
       " 40058,\n",
       " 40072,\n",
       " 40544,\n",
       " 41405,\n",
       " 42089,\n",
       " 42175,\n",
       " 42189,\n",
       " 42266,\n",
       " 43628,\n",
       " 44136,\n",
       " 44661,\n",
       " 46478,\n",
       " 46913,\n",
       " 47249,\n",
       " 47738,\n",
       " 48024,\n",
       " 48178,\n",
       " 64326,\n",
       " 70285,\n",
       " 72152,\n",
       " 75156,\n",
       " 75223,\n",
       " 75858,\n",
       " 76020,\n",
       " 77275,\n",
       " 77542,\n",
       " 77759,\n",
       " 79567,\n",
       " 80027,\n",
       " 80410,\n",
       " 86540,\n",
       " 86639,\n",
       " 86664,\n",
       " 86695,\n",
       " 86750,\n",
       " 86760,\n",
       " 86766,\n",
       " 86802,\n",
       " 86809,\n",
       " 88158,\n",
       " 88307,\n",
       " 88443,\n",
       " 88445,\n",
       " 89062,\n",
       " 89203,\n",
       " 90351,\n",
       " 90777,\n",
       " 90855}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(b['id']).difference(set(harvard_data['id']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f524c809-9fee-448c-baa3-fe1a6b783a11",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.20"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}