matthew_and_erica_poster.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import pickle\n",
    "import json\n",
    "import mdf_toolbox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Headers: {'Authorization': 'Bearer Agd16y9rjKX2KzgqmGMml5G0qzzmJW65gmOb7K6eBd3Y660VgkiXCo2o8EGXlJDK2PonD3aqOOD2BBig8wy07T70ma', 'Search': 'Agrx31XQ7WMKNrGvKM1oE30Vko9M64XEbYD7O0Y7mygwGbYN0ECXCEjxwllEnkBgea65v955JjlP4pilx5p42hvo2P', 'Openid': 'AgyBJoEbYbgwyz1WvoOYzDDBwgyMW0jklp8Vl7yXYE8mm0Wv54sVCwPWD1vz9ovDwm10MlooODDqOmIOKxJy9sp6Vn'}\n"
     ]
    }
   ],
   "source": [
    "eb_url = \"http://crawlerdev2-env.eba-afcdn59k.us-east-1.elasticbeanstalk.com/\"\n",
    "grouper = \"file_is_group\"\n",
    "\n",
    "auths = mdf_toolbox.login(\n",
    "    services=[\n",
    "        \"openid\",\n",
    "        \"data_mdf\",\n",
    "        \"search\",\n",
    "        \"petrel\",\n",
    "        \"transfer\",\n",
    "        \"dlhub\",\n",
    "        \"https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all\",\n",
    "    ],\n",
    "    app_name=\"Foundry\",\n",
    "    make_clients=True,\n",
    "    no_browser=False,\n",
    "    no_local_server=False,\n",
    "    # force=True\n",
    ")\n",
    "\n",
    "# print(auths['search'])\n",
    "# print(dir(auths['search']))\n",
    "\n",
    "# print(auths)\n",
    "fx_scope = 'https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all'\n",
    "headers = {'Authorization': f\"Bearer {auths['petrel']}\", 'Transfer': auths['transfer'], 'FuncX': auths[fx_scope], 'Petrel': auths['petrel']}\n",
    "fx_headers = {'Authorization': f\"Bearer {auths[fx_scope].access_token}\", \n",
    "             'Search': auths['search'].authorizer.access_token, \n",
    "             'Openid': auths['openid'].access_token}\n",
    "print(f\"Headers: {str(fx_headers)}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Crawl URL is : http://crawlerdev2-env.eba-afcdn59k.us-east-1.elasticbeanstalk.com//crawl\n",
      "{'Transfer': 'AgKgE5n1g5vo3DDY2WMQa68pbKl7dpV4dDVJ2nwrg2m5rG4YXBieCx2zrebe3a31BKEdgJkmByY679f10Qv7rSaqvo', 'Authorization': 'Bearer AgxQe2zMlezbK20GG5dq2OJGVKd82E6zkmnWnJy22KN04Q9K1PS8COKze9N7DQ9n7xWpo4z3WGWqGXiq0QdPlUPwEv', 'FuncX': 'Agd16y9rjKX2KzgqmGMml5G0qzzmJW65gmOb7K6eBd3Y660VgkiXCo2o8EGXlJDK2PonD3aqOOD2BBig8wy07T70ma'}\n",
      "b'{\"crawl_id\":\"fbe29541-7b30-41aa-84fb-7a73a170bbef\",\"status\":\"200 (OK)\"}\\n'\n",
      "Crawl ID: fbe29541-7b30-41aa-84fb-7a73a170bbef\n"
     ]
    }
   ],
   "source": [
    "# Now do the actual crawl \n",
    "globus_eid = \"4f99675c-ac1f-11ea-bee8-0e716405a293\"\n",
    "pub8_path = \"/cdiac/cdiac.ornl.gov/pub8old/pub8\"\n",
    "\n",
    "crawl_url = f'{eb_url}/crawl'\n",
    "print(f\"Crawl URL is : {crawl_url}\")\n",
    "\n",
    "first_ep_dict = {\n",
    "    'repo_type': 'GLOBUS',\n",
    "    'eid': globus_eid,\n",
    "    'dir_paths': [pub8_path],# , source_ep_path_2],\n",
    "    'grouper': grouper, \n",
    "    # 'prefetch_list':\n",
    "    #     [{'dest_ep_id': dest_ep_id, 'path': '/project2/chard/skluzacek/data_to_process'}]\n",
    "}\n",
    "\n",
    "tokens = {'Transfer': auths['transfer'].authorizer.access_token, \n",
    "          'Authorization': f\"Bearer {auths['petrel'].access_token}\", \n",
    "          'FuncX': auths[fx_scope].access_token}\n",
    "print(tokens)\n",
    "\n",
    "crawl_req = requests.post(f'{eb_url}/crawl', json={'endpoints': [first_ep_dict], 'tokens': tokens})\n",
    "print(crawl_req.content)\n",
    "crawl_id = json.loads(crawl_req.content)['crawl_id']\n",
    "print(f\"Crawl ID: {crawl_id}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n",
      "Crawl Status: {'bytes_crawled': 67502539984, 'crawl_id': 'fbe29541-7b30-41aa-84fb-7a73a170bbef', 'crawl_status': 'complete', 'files_crawled': 20427, 'groups_crawled': 20427}\n"
     ]
    }
   ],
   "source": [
    "# Now monitor status until crawl is complete\n",
    "crawl_status = requests.get(f'{eb_url}/get_crawl_status', json={'crawl_id': crawl_id})\n",
    "print(crawl_status)\n",
    "crawl_content = json.loads(crawl_status.content)\n",
    "print(f\"Crawl Status: {crawl_content}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-74-a8fa5c64abe4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     19\u001b[0m             \u001b[0mcsv_writer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwriterow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcrawl_timestamp\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0;31m# Sleep so I don't crash my service.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m         \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Fetch all of the metadata from SQS\n",
    "import time\n",
    "import csv\n",
    "#print(dir(auths['transfer'].authorizer))\n",
    "\n",
    "with open(\"/Users/tylerskluzacek/Desktop/filename_crawl_t_map.csv\", 'w') as f:\n",
    "    field_names = ['petrel_path', 'crawl_timestamp']\n",
    "    csv_writer = csv.writer(f)\n",
    "    \n",
    "    csv_writer.writerow(field_names)\n",
    "    \n",
    "    while True: \n",
    "        poller = requests.post(f'{eb_url}/fetch_crawl_mdata', json={'crawl_id': crawl_id, 'Transfer': auths['transfer'].authorizer.access_token, 'n': 1000})\n",
    "        file_batch = json.loads(poller.content)['file_ls']\n",
    "        for file in file_batch:\n",
    "            #print(file)\n",
    "            path = file['path']\n",
    "            crawl_timestamp = file['crawl_timestamp']\n",
    "            csv_writer.writerow([path, crawl_timestamp])\n",
    "        # Sleep so I don't crash my service.\n",
    "        time.sleep(1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (plz)",
   "language": "python",
   "name": "plz"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}