demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import metadata_ingest modules\n",
    "\n",
    "from metadata_ingest.form_parsers import ITSMetadataQuestionnaire, PDFQuestionnaire\n",
    "from metadata_ingest.socrata_ingestor import SocrataDataset\n",
    "\n",
    "\n",
    "# set sample questionnaire file path\n",
    "fp = 'forms/ITSJPO_MetadataQuestionnaire_fillable_sample.pdf'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run generic PDF parser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parsing forms/ITSJPO_MetadataQuestionnaire_fillable_sample.pdf using generic PDF parser\n",
      "===== Content ====\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'title': 'Sample Dataset Title',\n",
       " 'accessLevel': 'Restricted Public',\n",
       " 'rights': 'Sample text.',\n",
       " 'describedByAccessLevel': 'Public',\n",
       " 'describedByRights': 'N/A',\n",
       " 'distribution': {'downloadURL': 'N/A',\n",
       "  'accessURL': 'N/A',\n",
       "  'describedBy': 'N/A',\n",
       "  'describedByType': 'N/A'},\n",
       " 'contactPoint': {'dataOwner': 'Jane Doe: jane@dot.gov',\n",
       "  'dataSteward': 'ITS JPO: data.itsjpo@dot.gov, Joan Smith: joan@dot.gov',\n",
       "  'federalSponsor': 'John Smith: john@dot.gov'},\n",
       " 'creator': 'Wyoming Department of Transportation (WYDOT)',\n",
       " 'publisher': {'name': 'USDOT'},\n",
       " 'bureauCode': '021:15',\n",
       " 'programCode': '021:013',\n",
       " 'license': 'https://creativecommons.org/licenses/by-sa/4.0/',\n",
       " 'description': 'Sample description of the dataset.',\n",
       " 'spatial': 'Cambridge, MA',\n",
       " 'temporal': '2018-01-01/2018-12-31',\n",
       " 'accrualPeriodicity': 'R/P1D',\n",
       " 'theme': 'Automobiles, Railroads',\n",
       " 'keyword': 'tag1, tag2, tag3',\n",
       " 'version': '1.0',\n",
       " 'issued': '2019-01-01',\n",
       " 'landingPage': 'http://its.dot.gov/data/',\n",
       " 'identifiersExtended': 'ORCID: 12345, DOI: 54321, ISBN: 135246',\n",
       " 'awardIdentifier': 'id1, id2',\n",
       " 'columnarOrComplex': 'nested and complex',\n",
       " 'dataIngestType': 'scheduled batch uploads',\n",
       " 'dataVolume': '100000',\n",
       " 'dataSize': '5 MB',\n",
       " 'dataIngestNotes': 'Sample description.',\n",
       " 'dataReadyDate': '2019-01-01',\n",
       " 'fileAttachments': 'its_jpo_tampa_cvpilot_sampledoc.pdf: Sample Doc, its_jpo_tampa_cvpilot_sampledoc2.pdf: Sample Doc 2'}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Parsing {} using generic PDF parser'.format(fp))\n",
    "questionnaire = PDFQuestionnaire(fp)\n",
    "\n",
    "print('===== Content ====')\n",
    "questionnaire.content"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run ITS DataHub Metadata Questionnaire PDF parser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parsing forms/ITSJPO_MetadataQuestionnaire_fillable_sample.pdf using generic ITS DataHub Metadata Questionnaire PDF parser\n",
      "===== Content ====\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'title': 'Sample Dataset Title',\n",
       " 'accessLevel': 'Restricted Public',\n",
       " 'rights': 'Sample text.',\n",
       " 'describedByAccessLevel': 'Public',\n",
       " 'describedByRights': 'N/A',\n",
       " 'distribution': [{'downloadURL': 'N/A',\n",
       "   'accessURL': 'N/A',\n",
       "   'describedBy': 'N/A',\n",
       "   'describedByType': 'N/A'}],\n",
       " 'contactPoint': [{'fn': 'Jane Doe',\n",
       "   'hasEmail': 'jane@dot.gov',\n",
       "   'hasRole': 'dataOwner'},\n",
       "  {'fn': 'ITS JPO',\n",
       "   'hasEmail': 'data.itsjpo@dot.gov',\n",
       "   'hasRole': 'dataSteward'},\n",
       "  {'fn': 'Joan Smith', 'hasEmail': 'joan@dot.gov', 'hasRole': 'dataSteward'},\n",
       "  {'fn': 'John Smith',\n",
       "   'hasEmail': 'john@dot.gov',\n",
       "   'hasRole': 'federalSponsor'}],\n",
       " 'creator': 'Wyoming Department of Transportation (WYDOT)',\n",
       " 'publisher': {'name': 'USDOT'},\n",
       " 'bureauCode': '021:15',\n",
       " 'programCode': '021:013',\n",
       " 'license': 'https://creativecommons.org/licenses/by-sa/4.0/',\n",
       " 'description': 'Sample description of the dataset.',\n",
       " 'spatial': 'Cambridge, MA',\n",
       " 'temporal': '2018-01-01/2018-12-31',\n",
       " 'accrualPeriodicity': 'R/P1D',\n",
       " 'theme': 'Automobiles, Railroads',\n",
       " 'keyword': ['tag1', 'tag2', 'tag3'],\n",
       " 'version': '1.0',\n",
       " 'issued': '2019-01-01',\n",
       " 'landingPage': 'http://its.dot.gov/data/',\n",
       " 'identifiersExtended': [{'type': 'orcid', 'uid': '12345'},\n",
       "  {'type': 'doi', 'uid': '54321'},\n",
       "  {'type': 'isbn', 'uid': '135246'}],\n",
       " 'awardIdentifier': 'id1, id2',\n",
       " 'columnarOrComplex': 'nested and complex',\n",
       " 'dataIngestType': 'scheduled batch uploads',\n",
       " 'dataVolume': '100000',\n",
       " 'dataSize': '5 MB',\n",
       " 'dataIngestNotes': 'Sample description.',\n",
       " 'dataReadyDate': '2019-01-01',\n",
       " 'fileAttachments': 'its_jpo_tampa_cvpilot_sampledoc.pdf: Sample Doc, its_jpo_tampa_cvpilot_sampledoc2.pdf: Sample Doc 2'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Parsing {} using generic ITS DataHub Metadata Questionnaire PDF parser'.format(fp))\n",
    "mq = ITSMetadataQuestionnaire(fp)\n",
    "\n",
    "print('===== Content ====')\n",
    "mq.content"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Format metadata to fit Socrata's schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===== DTG metadataUpsert ====\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'name': 'Sample Dataset Title',\n",
       " 'attribution': 'U.S. Department of Transportation Intelligent Transportation Systems Joint Program Office (JPO)',\n",
       " 'description': 'Sample description of the dataset.',\n",
       " 'privateMetadata': {'contactEmail': 'RDAE_Support@bah.com'},\n",
       " 'tags': ['tag1',\n",
       "  'tag3',\n",
       "  'intelligent transportation systems (its)',\n",
       "  'its joint program office (jpo)',\n",
       "  'tag2'],\n",
       " 'customFields': {'Common Core': {'Contact Email': 'data.itsjpo@dot.gov',\n",
       "   'Contact Name': 'ITS JPO',\n",
       "   'Language': 'English',\n",
       "   'Update Frequency': 'R/P1D',\n",
       "   'License': 'Other',\n",
       "   'Program Code': '021:013',\n",
       "   'Bureau Code': '021:15',\n",
       "   'Geographic Coverage': 'Cambridge, MA',\n",
       "   'Publisher': 'USDOT',\n",
       "   'Temporal Applicability': '2018-01-01/2018-12-31',\n",
       "   'Is Quality Data': 'True',\n",
       "   'Public Access Level': 'restricted public',\n",
       "   'Homepage': 'http://its.dot.gov/data/'}},\n",
       " 'category': 'Automobiles'}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('===== DTG metadataUpsert ====')\n",
    "mq.generate_dtg_metadata()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create new dataset in Socrata using content from a questionnaire"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = SocrataDataset(uuid=None, mq_fp=fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job progress: initializing\n",
      "Job progress: initializing\n",
      "Job progress: initializing\n",
      "Job progress: initializing\n",
      "Job progress: initializing\n",
      "Job progress: finishing\n",
      "Job progress: finishing\n",
      "Job progress: finishing\n",
      "Job progress: finishing\n",
      "Job progress: successful\n",
      "Dataset has been created at https://datahub.transportation.gov/d/xd74-fkkc\n"
     ]
    }
   ],
   "source": [
    "dataset.create_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'xd74-fkkc'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ID of newly created dataset\n",
    "dataset.uuid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Update existing dataset in Socrata using content from a questionnaire"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metadata updated for https://datahub.transportation.gov/d/xd74-fkkc\n"
     ]
    }
   ],
   "source": [
    "updated_dataset = SocrataDataset(uuid=dataset.uuid, mq_fp=fp)\n",
    "updated_dataset.update_metadata()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}