acm-pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "afab57ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "047b55be",
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import os \n",
    "import pickle\n",
    "import json\n",
    "import requests\n",
    "from googleapiclient.discovery import build\n",
    "from google.auth.transport.requests import Request\n",
    "from google_auth_oauthlib.flow import InstalledAppFlow\n",
    "import google_auth_httplib2  # This gotta be installed for build() to work\n",
    "import requests_oauthlib\n",
    "import xml.etree.ElementTree as ET\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11085caa",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d5482d76",
   "metadata": {},
   "outputs": [],
   "source": [
    "# EVENT NAME FOR THESE PHOTOS!!\n",
    "event_name = \"testing 3.8 system test\"\n",
    "\n",
    "\n",
    "#Google Auth\n",
    "SCOPES = ['https://www.googleapis.com/auth/photoslibrary.readonly']\n",
    "google_secrets = 'client_secret.json'\n",
    "pickle_google = \"token.pickle\"\n",
    "\n",
    "# FLicker Auth\n",
    "api_key = ...#HIDDEN\n",
    "api_secret = ... #HIDDEN\n",
    "\n",
    "# pipeline params\n",
    "google_album_name = \"full test\" \n",
    "flicker_album_name = \"FULL_PIPLEINE_TEST_3.8\"\n",
    "is_public = 0 # 1 means yes, 0 means no. \n",
    "\n",
    "#metadata csv\n",
    "metadata_filename = \"file_metadata.csv\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bec3b3cf",
   "metadata": {},
   "source": [
    "## Authorization Work"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a29e2633",
   "metadata": {},
   "source": [
    "### Google Auth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bce0282d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<google.oauth2.credentials.Credentials object at 0x000001570BB6F908>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<googleapiclient.discovery.Resource at 0x1570bbabb08>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Load cred from https://console.cloud.google.com/apis/credentials?authuser=3&project=acm-pipeline-tes-1662055073652\n",
    "creds = None\n",
    "if(os.path.exists(pickle_google)):\n",
    "    with open(pickle_google, \"rb\") as tokenFile:\n",
    "        creds = pickle.load(tokenFile)\n",
    "\n",
    "print(creds)\n",
    "if not creds or not creds.valid:\n",
    "    if (creds and creds.expired and creds.refresh_token):\n",
    "        creds.refresh(Request())\n",
    "    else:\n",
    "        flow = InstalledAppFlow.from_client_secrets_file(google_secrets, SCOPES)\n",
    "        creds = flow.run_local_server(port = 0)\n",
    "    with open(pickle_google, \"wb\") as tokenFile:\n",
    "        pickle.dump(creds, tokenFile)\n",
    "        \n",
    "service = build('photoslibrary', 'v1', credentials = creds, static_discovery=False)\n",
    "service"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a72668f",
   "metadata": {},
   "source": [
    "### Flicker Auth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4a69ff3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# OAuth URLs\n",
    "request_token_url = 'https://www.flickr.com/services/oauth/request_token'\n",
    "access_token_url = 'https://www.flickr.com/services/oauth/access_token'\n",
    "authorization_url = 'https://www.flickr.com/services/oauth/authorize'\n",
    "\n",
    "callback_uri = \"https://www.flickr.com\"\n",
    "\n",
    "oauth_session = requests_oauthlib.OAuth1Session(client_key=api_key, client_secret=api_secret, signature_method=u'HMAC-SHA1', signature_type=u'AUTH_HEADER', callback_uri=callback_uri)\n",
    "\n",
    "# First step, fetch the request token:\n",
    "request_token = oauth_session.fetch_request_token(request_token_url)\n",
    "\n",
    "# Second step, follow this link and authorize:\n",
    "print(oauth_session.authorization_url(authorization_url))\n",
    "\n",
    "# Third step, fetch the access token:\n",
    "redirect_response = input('Paste the full redirect URL here.')\n",
    "print(oauth_session.parse_authorization_response(redirect_response))\n",
    "access_token = oauth_session.fetch_access_token(access_token_url)\n",
    "access_token"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddbdfdb4",
   "metadata": {},
   "source": [
    "## Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a2809ba",
   "metadata": {},
   "source": [
    "### Get Google Photos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6296d1e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "google_albums = service.albums().list().execute()\n",
    "albums = google_albums.get('albums', [])\n",
    "for album in albums:\n",
    "    if(album[\"title\"] == google_album_name):\n",
    "        break\n",
    "album"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7790c8e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_album_photos(id, next_page_token=None):\n",
    "    url = 'https://photoslibrary.googleapis.com/v1/mediaItems:search'\n",
    "    if(next_page_token==None):\n",
    "        payload = {\n",
    "                      \"albumId\": id,\n",
    "                    \"pageSize\": \"5\"\n",
    "        }\n",
    "    else:\n",
    "        payload = {\n",
    "                      \"albumId\": id,\n",
    "                    \"pageSize\": \"5\",\n",
    "                      \"pageToken\": next_page_token\n",
    "        }\n",
    "    \n",
    "    #print(payload)\n",
    "    headers = {\n",
    "        'content-type': 'application/json',\n",
    "        'Authorization': 'Bearer {}'.format(creds.token)\n",
    "    }\n",
    "    \n",
    "    try:\n",
    "        res = requests.request(\"POST\", url, headers=headers, params=payload)\n",
    "    except:\n",
    "        print('Request error') \n",
    "    \n",
    "    return(res)\n",
    "\n",
    "photos = []\n",
    "res = get_album_photos(album['id'])\n",
    "photo_meta_data = res.json()\n",
    "photos = photos + photo_meta_data[\"mediaItems\"]\n",
    "if (\"nextPageToken\" in list(photo_meta_data.keys())):\n",
    "    next_page = photo_meta_data[\"nextPageToken\"]\n",
    "else:\n",
    "    next_page= False\n",
    "\n",
    "while(next_page):\n",
    "    res = get_album_photos(album['id'], next_page)\n",
    "    photo_meta_data = res.json()\n",
    "    photos = photos + photo_meta_data[\"mediaItems\"]\n",
    "    next_page = False\n",
    "    if (\"nextPageToken\" in list(photo_meta_data.keys())):\n",
    "        next_page = photo_meta_data[\"nextPageToken\"]\n",
    "    else:\n",
    "        next_page= False\n",
    "\n",
    "    \n",
    "photos"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "979b3621",
   "metadata": {},
   "source": [
    "## Metadata saving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "43931506",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "metadata_df = pd.DataFrame(columns=[\"filename\", \"event name\", \"google photos link\", \"raw image link\", \"mediaType\", \"Metadata\", \"tags\", \"google album\", \"flickr album\"])\n",
    "for media in photos:\n",
    "    row = {\n",
    "        \"filename\": media[\"filename\"],\n",
    "        \"event name\": event_name,\n",
    "        \"google photos link\": media[\"productUrl\"],\n",
    "        \"raw image link\": media[\"baseUrl\"],\n",
    "        \"mediaType\": media['mimeType'],\n",
    "        \"Metadata\":  media[\"mediaMetadata\"],\n",
    "        \"tags\": \"tbd\",\n",
    "        \"google album\": album[\"productUrl\"],\n",
    "        \"flickr album\": \"not uploaded yet\"\n",
    "    }\n",
    "    metadata_df.loc[metadata_df.shape[0]] = row"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bffe3f2",
   "metadata": {},
   "source": [
    "### Create Flickr Album"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5730a328",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://api.flickr.com/services/rest/'\n",
    "\n",
    "headers = {'Authorization': \"Bearer {}\".format(access_token[\"oauth_token\"])}\n",
    "params = {\n",
    "    \"api_key\": api_key,\n",
    "    \"title\": flicker_album_name,\n",
    "    \"primary_photo_id\": \"52328002434\", #TEMP PHOTO\n",
    "    \"method\": \"flickr.photosets.create\"\n",
    "}\n",
    "method=\"flickr.photosets.create\"\n",
    "\n",
    "try:\n",
    "    res1 = oauth_session.post(url, params=params)\n",
    "except Exception as e:\n",
    "    print(e)\n",
    "    print('Request error') \n",
    "\n",
    "mytree = ET.fromstring(res1.text)\n",
    "id_of_photoset = mytree.getchildren()[0].attrib[\"id\"]\n",
    "url_of_photoset = mytree.getchildren()[0].attrib[\"url\"]\n",
    "id_of_photoset, url_of_photoset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3cbfe6f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "res1.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "10af07b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "is_public"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09936180",
   "metadata": {},
   "source": [
    "### Add Each File To Flickr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6def8457",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import shutil\n",
    "from tqdm import tqdm\n",
    "for media in tqdm(photos):\n",
    "    ##DOWNLOAD IMAGE\n",
    "    r = requests.get(media['baseUrl'], stream = True)\n",
    "    filename = media['filename']\n",
    "    if r.status_code == 200:\n",
    "        # Set decode_content value to True, otherwise the downloaded image file's size will be zero.\n",
    "        r.raw.decode_content = True\n",
    "\n",
    "        # Open a local file with wb ( write binary ) permission.\n",
    "        with open(filename,'wb') as f:\n",
    "            shutil.copyfileobj(r.raw, f)\n",
    "\n",
    "        #print('Image sucessfully Downloaded: ',filename)\n",
    "    else:\n",
    "        #print('Image Couldn\\'t be retreived')\n",
    "        continue\n",
    "    \n",
    "    \n",
    "    ##TODO HERE:\n",
    "    # ADD IN CSV STUFF FOR CATALOGING\n",
    "    \n",
    "    \n",
    "    ##UPLOAD TO FLICKER\n",
    "    url = 'https://api.flickr.com/services/upload/'\n",
    "\n",
    "    headers = {'Authorization': \"Bearer {}\".format(access_token[\"oauth_token\"])}\n",
    "\n",
    "    params = {\n",
    "        \"is_public\": is_public,\n",
    "        \"is_friend\": is_public, \n",
    "        \"is_family\": is_public\n",
    "    }\n",
    "\n",
    "    try:\n",
    "        with open(filename,'rb') as f:\n",
    "            files = {'photo': f, \"is_public\": is_public,\n",
    "        \"is_friend\": is_public, \n",
    "        \"is_family\": is_public}\n",
    "            res = oauth_session.request(\"POST\", url,  files=files)\n",
    "            print(res.text)\n",
    "        f.close()\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        #print('Request error')\n",
    "        continue\n",
    "\n",
    "    id_of_photo = None\n",
    "    try:\n",
    "        mytree = ET.fromstring(res.text)\n",
    "        id_of_photo = mytree.getchildren()[0].text\n",
    "    except Exception as e:\n",
    "        #print(e)\n",
    "        #print('File no exists')\n",
    "        continue\n",
    "\n",
    "    #print(res.text)\n",
    "    # hold for data caps\n",
    "    time.sleep(2)\n",
    "    break\n",
    "    \n",
    "    ##ADD PHOTO TO FLICKER\n",
    "    url = 'https://api.flickr.com/services/rest/'\n",
    "    params = {\n",
    "        \"api_key\": api_key,\n",
    "        \"photoset_id\": id_of_photoset,\n",
    "        \"photo_id\": id_of_photo,\n",
    "        \"method\": \"flickr.photosets.addPhoto\"\n",
    "    }\n",
    "\n",
    "    try:\n",
    "        res1 = oauth_session.request(\"POST\", url, params=params)\n",
    "        #print(res1.text)\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        #print('ADDING TO FLICKER ALBUM FAILED') \n",
    "        continue\n",
    "   \n",
    "    f.close()\n",
    "    del files\n",
    "    os.remove(filename)\n",
    "    metadata_df.loc[metadata_df[\"filename\"]==filename, \"flickr album\"] = url_of_photoset\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08dd606c",
   "metadata": {},
   "source": [
    "## Clean Up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e734d80c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n",
      "<rsp stat=\"ok\">\n",
      "</rsp>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## Remove Temp Photo Used to Create Flicker Photo\n",
    "url = 'https://api.flickr.com/services/rest/'\n",
    "params = {\n",
    "    \"api_key\": api_key,\n",
    "    \"photoset_id\": id_of_photoset,\n",
    "    \"photo_id\": 52328002434,\n",
    "    \"method\": \"flickr.photosets.removePhoto\"\n",
    "}\n",
    "\n",
    "try:\n",
    "    res1 = oauth_session.request(\"POST\", url, params=params)\n",
    "    print(res1.text)\n",
    "except Exception as e:\n",
    "    print(e)\n",
    "    print('ADDING TO FLICKER ALBUM FAILED') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e86a8aa9",
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata_df.to_csv(metadata_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "345f931f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d073bb88",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}