Xtract-Crawl-Fetch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fair_research_login import NativeClient\n",
    "import requests\n",
    "import pickle\n",
    "import json\n",
    "import time \n",
    "\n",
    "xtract_base_url = \"http://xtract-crawler-4.eba-ghixpmdf.us-east-1.elasticbeanstalk.com\"\n",
    "\n",
    "# MDF Materials Data at NCSA \n",
    "# source_ep_id = \"82f1b5c6-6e9b-11e5-ba47-22000b92c6ec\"\n",
    "source_ep_id = \"e38ee745-6d04-11e5-ba46-22000b92c6ec\"\n",
    "base_url = \"https://data.materialsdatafacility.org\"\n",
    "folder_to_crawl = \"/MDF/mdf_connect/prod\"\n",
    "\n",
    "# This only matters if you want files grouped together. \n",
    "grouper = \"matio\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Headers: {'Authorization': 'Bearer Ag2j63gD27qkB9PD74Nl103DE9N8ejrqvVWgMOaBn5M99kkqVWtkC6kvO1Xn1YM2NVwd1g7be0oNepHm0KKoeTdk5n', 'Transfer': 'Ag3lkkvrlEmQpjpbdpyQ14EY8JzMjzzJDeOnn0zWQo9DerV4mNsVCw2832rG27g6jj4ll0pXbB4kemtXg77eJUgB6z', 'FuncX': 'Ag8rBWbWKPlO4dd9kOq34r15z5mrbxw4Dz6r8mk9zjxOv9283Wi8CJGy4bYmxVYvqPbgbrB9DpnM7kF7QDDgeSoekw', 'Petrel': 'Ag2j63gD27qkB9PD74Nl103DE9N8ejrqvVWgMOaBn5M99kkqVWtkC6kvO1Xn1YM2NVwd1g7be0oNepHm0KKoeTdk5n'}\n"
     ]
    }
   ],
   "source": [
    "# Do Globus NativeClient Authentication, save the headers.\n",
    "# I've included all the relevant ones to be safe. \n",
    "client = NativeClient(client_id='7414f0b4-7d05-4bb6-bb00-076fa3f17cf5')\n",
    "tokens = client.login(\n",
    "    requested_scopes=['https://auth.globus.org/scopes/56ceac29-e98a-440a-a594-b41e7a084b62/all', \n",
    "                      'urn:globus:auth:scope:transfer.api.globus.org:all',\n",
    "                     \"https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all\", \n",
    "                     'email', 'openid'],\n",
    "    no_local_server=True,\n",
    "    no_browser=True)\n",
    "\n",
    "auth_token = tokens[\"petrel_https_server\"]['access_token']\n",
    "transfer_token = tokens['transfer.api.globus.org']['access_token']\n",
    "funcx_token = tokens['funcx_service']['access_token']\n",
    "\n",
    "headers = {'Authorization': f\"Bearer {auth_token}\", 'Transfer': transfer_token, 'FuncX': funcx_token, 'Petrel': auth_token}\n",
    "print(f\"Headers: {headers}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Crawl URL is : http://xtract-crawler-4.eba-ghixpmdf.us-east-1.elasticbeanstalk.com/crawl\n",
      "Crawl ID: 6f889b2f-3c5c-4289-bc64-ede2def32a63\n"
     ]
    }
   ],
   "source": [
    "# Initialize the crawl. This kicks off the Globus EP crawling service on the backend. \n",
    "crawl_url = f'{xtract_base_url}/crawl'\n",
    "print(f\"Crawl URL is : {crawl_url}\")\n",
    "crawl_req = requests.post(crawl_url, json={'repo_type': \"GLOBUS\", 'eid': source_ep_id, 'dir_path': folder_to_crawl, 'Transfer': transfer_token, 'Authorization': funcx_token,'grouper': grouper, 'https_info': {'base_url':base_url}})\n",
    "crawl_id = json.loads(crawl_req.content)['crawl_id']\n",
    "print(f\"Crawl ID: {crawl_id}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 0, 'crawl_id': '6f889b2f-3c5c-4289-bc64-ede2def32a63', 'crawl_status': 'STARTING', 'files_crawled': 0, 'groups_crawled': 0}\n",
      "Sleeping before re-polling...\n",
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 0, 'crawl_id': '6f889b2f-3c5c-4289-bc64-ede2def32a63', 'crawl_status': 'STARTING', 'files_crawled': 0, 'groups_crawled': 0}\n",
      "Sleeping before re-polling...\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-076ce3e08ef1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Sleeping before re-polling...\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m         \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Wait for the crawl to finish before we can start fetching our metadata. \n",
    "while True: \n",
    "    crawl_status = requests.get(f'{xtract_base_url}/get_crawl_status', json={'crawl_id': crawl_id})\n",
    "    print(crawl_status)\n",
    "    crawl_content = json.loads(crawl_status.content)\n",
    "    print(f\"Crawl Status: {crawl_content}\")\n",
    "\n",
    "    if crawl_content['crawl_status'] == 'SUCCEEDED':\n",
    "        files_crawled = crawl_content['files_crawled']\n",
    "        print(\"Our crawl has succeeded!\")\n",
    "        break\n",
    "    else:\n",
    "        print(\"Sleeping before re-polling...\")\n",
    "        time.sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queue is empty!\n",
      "Continuing...\n",
      "All files have been fetched!\n",
      "Files: ['/thurston_selfassembled_peptide_spectra_v1.1/DFT/MoleculeConfigs/di_30_-10.xyz/POSCAR', '/thurston_selfassembled_peptide_spectra_v1.1/DFT/MoleculeConfigs/di_30_-10.xyz/INCAR', '/thurston_selfassembled_peptide_spectra_v1.1/DFT/MoleculeConfigs/di_30_-10.xyz/DOSCAR', '/thurston_selfassembled_peptide_spectra_v1.1/DFT/MoleculeConfigs/di_30_-10.xyz/OUTCAR', '/thurston_selfassembled_peptide_spectra_v1.1/DFT/MoleculeConfigs/di_30_-10.xyz/KPOINTS', '/thurston_selfassembled_peptide_spectra_v1.1/DFT/MoleculeConfigs/di_30_-10.xyz/vasprun.xml', '/thurston_selfassembled_peptide_spectra_v1.1/DFT/MoleculeConfigs/di_30_-10.xyz/Transmatrix', '/thurston_selfassembled_peptide_spectra_v1.1/DFT/MoleculeConfigs/di_30_-10.xyz/PROCAR']\n"
     ]
    }
   ],
   "source": [
    "# Now we fetch our metadata. Here you can configure n to be maximum number of \n",
    "# messages you want at once. \n",
    "\n",
    "file_ls = []\n",
    "fetched_files = 0\n",
    "while fetched_files < files_crawled: \n",
    "    fetch_mdata = requests.get(f'{xtract_base_url}/fetch_crawl_mdata', json={'crawl_id': crawl_id, 'n': 2})\n",
    "    fetch_content = json.loads(fetch_mdata.content)\n",
    "    \n",
    "    for file_path in fetch_content['file_ls']:\n",
    "        file_ls.append(file_path)\n",
    "        fetched_files += 1\n",
    "        \n",
    "    if fetch_content['queue_empty']:\n",
    "        print(\"Queue is empty!\")\n",
    "        print(\"Continuing...\")\n",
    "        time.sleep(2)\n",
    "        \n",
    "print(\"All files have been fetched!\")\n",
    "print(f\"Files: {file_ls}\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (funcx-jan)",
   "language": "python",
   "name": "funcx-jan"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}