Add links and script to download 2015 HD presenter & presentation videos

This adds links to 2015 HD videos of both presenter and presentation.
If you want non-HD videos, you need to modify hd-2015-video-pages.txt file
and extract video links again.

First lecture is missing because I couldn't extract it.

fix #7
fix #9

README.md

@@ -10,7 +10,27 @@ This is a basic shell script that downloads all the lecture/section videos assoc
 
 Usage
 ---
-
+
+## 2015
+
+`hd-2015-video-files.txt` file contains links to both presenter and presentaion
+videos. They are HD and without captions. Lectures will have meaningful
+filenames after download. To download:
+
+```
+cat hd-2015-video-files.txt | ./get-2015.sh
+```
+
+If you want SD/Mobile/HD-Captions videos, you need to put the links from
+CS109 Data Science videos page into `hg-2015-video-pages.txt` file and
+extract video links again:`
+
+```
+cat hd-2015-video-pages.txt | ./extract-2015.py > hd-video-files-2015.txt
+```
+
+## 2014
+
 ```
 ./get.sh hq-urls.txt # for high quality versions
 ```

add.py

@@ -0,0 +1,10 @@
+import sys
+import xmlrpclib
+from pprint import pprint
+
+name, url = sys.argv[1:]
+print('%s -> %s' % (name, url))
+
+s = xmlrpclib.ServerProxy("http://localhost:6800/rpc")
+r = s.aria2.addUri([url], {"dir":".", "out": name})
+pprint(r)

extract-2015.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+
+from pyvirtualdisplay import Display
+from selenium import webdriver
+from selenium.webdriver.support.ui import WebDriverWait
+import time
+import fileinput
+import sys
+
+display = Display(visible=0, size=(800, 600))
+display.start()
+
+def grab(name_url):
+    name, url = name_url.split(',')
+    browser = webdriver.Chrome()
+    browser.get(url)
+
+    TIMEOUT = 30
+    WebDriverWait(browser, TIMEOUT).until(
+        lambda session: len(session.find_elements_by_tag_name('source')) > 1)
+
+    # from ipdb import set_trace; set_trace(context=20)
+    # with open('2.html', 'w') as file_object:
+    #     file_object.write(browser.page_source)
+
+    sources = browser.find_elements_by_tag_name('source')
+    for source in sources:
+        src = source.get_attribute('src')
+        src = src[:src.find('?')]
+        if 'presenter' in src:
+            suffix = '-presenter'
+        elif 'presentation' in src:
+            suffix = '-presentation'
+        else:
+            suffix = '-unknown'
+        print('%s,%s' % (name + suffix, src))
+    browser.quit()
+
+for line in fileinput.input():
+   sys.stdout.flush()
+   try:
+       grab(line)
+   except Exception as e:
+       print(line, e)
+
+display.stop()

get-2015.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# To extract files:
+# cat hd-2015-video-pages.txt | ./extract-2015.py > hd-video-files-2015.txt
+
+# To downloaded extracted files:
+# cat hd-2015-video-files.txt | ./get-2015.sh
+
+#cat name.url.txt | while read line
+#cat $1 | while read line
+while read line
+do
+    echo $line
+    name="`echo $line | cut -f1 -d,`.mp4"
+    url="`echo $line | cut -f2 -d,`"
+    echo "$name -> $url"
+    wget $* -O "$name" "$url"
+    #python2 add.py "$name" "$url"
+    #aria2c $* -o "$name" "$url"
+done

hd-2015-video-files.txt

@@ -0,0 +1,76 @@
+week02-lab01-pandas-python-github-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/e15f221c-5275-4f7f-b486-759a7d483bc8/6cd35dfd-6c3d-46e8-bc58-9d8d4d27e28a/presenter_delivery.mp4
+week02-lab01-pandas-python-github-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/e15f221c-5275-4f7f-b486-759a7d483bc8/7888ca17-0938-4e49-8526-902b47623d78/presentation_delivery.mp4
+week02-lecture02-web-scraping-regexp-reshaping-cleanup-pandas-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/f7ff1893-fbf7-4909-b44e-12e61a98a677/4f389044-2704-47b0-8c1a-73b52d4289a1/presenter_delivery.mp4
+week02-lecture02-web-scraping-regexp-reshaping-cleanup-pandas-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/f7ff1893-fbf7-4909-b44e-12e61a98a677/02a2e2e7-e4ff-4230-becc-026e46e67d42/presentation_delivery.mp4
+week02-lecture03-exploratory-data-analysis-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/a4e81697-fd86-415c-9b29-c14ea7ec15f2/8c767060-50c3-4b1d-bbee-76dcb6ff76ab/presenter_delivery.mp4
+week02-lecture03-exploratory-data-analysis-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/a4e81697-fd86-415c-9b29-c14ea7ec15f2/34b1205a-497b-4ee0-a7b4-7dc8424635bc/presentation_delivery.mp4
+week03-lab02-scraping-pandas-python-viz-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/b89de485-9e30-4783-9287-23701c5f95f7/5073437e-574b-4e83-9923-dc2393ab4ecc/presenter_delivery.mp4
+week03-lab02-scraping-pandas-python-viz-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/b89de485-9e30-4783-9287-23701c5f95f7/5c3f9f35-6652-43e5-a7c6-938e95056cca/presentation_delivery.mp4
+week03-lecture04-pandas-sql-grammar-of-data-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/f8a832cb-56e7-401b-b485-aec3c9928069/f9a65e8c-903f-4823-979e-6adf967301cd/presenter_delivery.mp4
+week03-lecture04-pandas-sql-grammar-of-data-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/f8a832cb-56e7-401b-b485-aec3c9928069/96cd9f92-8eac-4429-bc83-f1a9c4491772/presentation_delivery.mp4
+week03-lecture05-statistical-models-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/873964c6-d345-4f46-a8bc-727b96432d63/a7b54f51-b071-4b46-b6bb-b09f131cc949/presenter_delivery.mp4
+week03-lecture05-statistical-models-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/873964c6-d345-4f46-a8bc-727b96432d63/3d7b7722-e16f-422f-addf-b3ab22ebf0f7/presentation_delivery.mp4
+week04-lab03-probability-distributions-frequentist-statistics-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/8af4418a-7f5b-4738-9c72-6fc2ba1fc499/84ed1353-3d21-45d1-bca6-9da394ca6a9f/presenter_delivery.mp4
+week04-lab03-probability-distributions-frequentist-statistics-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/8af4418a-7f5b-4738-9c72-6fc2ba1fc499/caeed6e9-2bbb-4da1-b5c7-1b99a7992ffa/presentation_delivery.mp4
+week04-lecture06-story-telling-effective-communication-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/7f968df9-404a-46a2-ae5f-e35479875f95/8c95d0ef-c686-4e91-8dd8-80984605ff30/presenter_delivery.mp4
+week04-lecture06-story-telling-effective-communication-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/7f968df9-404a-46a2-ae5f-e35479875f95/c3867e01-4fd6-4f29-851b-df5e90a15df8/presentation_delivery.mp4
+week04-lecture07-bias-regression-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/afe70053-b8b7-43d3-9c2f-f482f479baf7/931974da-6254-4cf4-bd57-139c3e2eab5b/presenter_delivery.mp4
+week04-lecture07-bias-regression-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/afe70053-b8b7-43d3-9c2f-f482f479baf7/4de8b07f-8af4-4f68-abb8-3b7c069187b5/presentation_delivery.mp4
+week05-lab04-regression-logistic-regression-sklearn-statsmodels-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/483c8b93-3700-4ee8-80ed-aad7f3da7ac2/ebd231b0-9e64-4dea-b7d7-5ac4e96ecef6/presenter_delivery.mp4
+week05-lab04-regression-logistic-regression-sklearn-statsmodels-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/483c8b93-3700-4ee8-80ed-aad7f3da7ac2/ebb68ed2-48e5-48fe-99ad-6fc09322a552/presentation_delivery.mp4
+week05-lecture08-more-regression-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/664f668e-e008-4f44-8600-e09ee6d629b0/33adf7fe-5be8-4fa0-87d8-1afc958773e9/presenter_delivery.mp4
+week05-lecture08-more-regression-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/664f668e-e008-4f44-8600-e09ee6d629b0/e9d1921e-0f3f-4d09-b3ac-fb82f6f26c49/presentation_delivery.mp4
+week05-lecture09-classification-knn-cv-dimensionality-reduction-pca-mds-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/c322c0d5-9cf9-4deb-b59f-d6741064ba8a/267fa8c0-f267-43e8-84f2-c9a450a3da3c/presenter_delivery.mp4
+week05-lecture09-classification-knn-cv-dimensionality-reduction-pca-mds-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/c322c0d5-9cf9-4deb-b59f-d6741064ba8a/6a80b293-6bde-4413-ae0c-d97972464228/presentation_delivery.mp4
+week06-lab05-1-machine-learning-learning-a-model-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/e509f996-9633-4b75-a48a-e29246a316db/2080ddd7-24f8-4282-a182-10cd671033be/presenter_delivery.mp4
+week06-lab05-1-machine-learning-learning-a-model-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/e509f996-9633-4b75-a48a-e29246a316db/c59a7205-5cac-4e97-985c-85a0699238b9/presentation_delivery.mp4
+week06-lab05-2-machine-learning-classification-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/90e73c64-855c-4b06-afb2-94da608ecfbf/14564038-5b0c-4df2-882c-f8a2b21b7411/presenter_delivery.mp4
+week06-lab05-2-machine-learning-classification-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/90e73c64-855c-4b06-afb2-94da608ecfbf/92543021-71c4-433b-829a-efd03de83817/presentation_delivery.mp4
+week06-lecture10-svm-evaluation-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/f21fcc8f-93a8-49f6-9ff8-0f339b0728bd/882666dc-6b2f-46f4-a0db-c7fc5c49d5d0/presenter_delivery.mp4
+week06-lecture10-svm-evaluation-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/f21fcc8f-93a8-49f6-9ff8-0f339b0728bd/d8425e3a-838a-4fa4-966a-14a5b92a5b71/presentation_delivery.mp4
+week06-lecture11-decision-trees-random-forests-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/8892a8b7-25eb-4bc5-80b6-47b9cf681a05/79c76b94-78cd-4d17-bef8-af92f3441cc4/presenter_delivery.mp4
+week06-lecture11-decision-trees-random-forests-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/8892a8b7-25eb-4bc5-80b6-47b9cf681a05/346042c6-757e-42e4-b2d0-b9693827c0d3/presentation_delivery.mp4
+week07-lab06-1-machine-learning-2-classification-redux-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/83dfe4c9-fa1b-429c-a84c-839195fbede8/7f40178a-7301-465e-bbf0-05c9e41778f3/presenter_delivery.mp4
+week07-lab06-1-machine-learning-2-classification-redux-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/83dfe4c9-fa1b-429c-a84c-839195fbede8/ec258bfc-23e1-4f22-a732-0b51200e3c7e/presentation_delivery.mp4
+week07-lab06-2-machine-learning-2-comparing-models-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/4676adde-557f-469b-bca8-5ffe868094eb/f9bcf10a-2404-4ff6-9c46-10ee238a570f/presenter_delivery.mp4
+week07-lab06-2-machine-learning-2-comparing-models-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/4676adde-557f-469b-bca8-5ffe868094eb/3cb2251f-a458-443f-b36e-692be8bb7559/presentation_delivery.mp4
+week07-lecture12-ensemble-methods-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/4831ebf0-7832-42c5-9339-5b5e08dd3e92/10497f83-1a6a-46a8-aeb9-fa2976080084/presenter_delivery.mp4
+week07-lecture12-ensemble-methods-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/4831ebf0-7832-42c5-9339-5b5e08dd3e92/f0c76062-5148-418e-b61c-7a9b747bcaaa/presentation_delivery.mp4
+week07-lecture13-best-practices-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/b33eec92-d049-4353-a904-5054eb718aff/7b965558-ed78-4168-b7b1-c226d826f0de/presenter_delivery.mp4
+week07-lecture13-best-practices-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/b33eec92-d049-4353-a904-5054eb718aff/1c932f88-95c9-4429-b07d-4281820109c5/presentation_delivery.mp4
+week08-lab07-ensembles-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/0c645a5c-d262-4bb5-a137-3a78db60f3e7/47164bc8-e734-49ab-ba60-0c1a519b01b8/presenter_delivery.mp4
+week08-lab07-ensembles-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/0c645a5c-d262-4bb5-a137-3a78db60f3e7/35f98864-fd86-4821-bddf-d4bb54f12ffe/presentation_delivery.mp4
+week08-lecture14-best-practices-recommendations-mapreduce-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/afee45b9-dcf5-4f29-bc60-871aa78f1cf8/ba79c1a5-ce50-40bc-ba90-15246de1b49f/presenter_delivery.mp4
+week08-lecture14-best-practices-recommendations-mapreduce-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/afee45b9-dcf5-4f29-bc60-871aa78f1cf8/f9ec3a8d-58cb-4362-a061-a3fe99ed0438/presentation_delivery.mp4
+week08-lecture15-mapreduce-combiners-spark-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/10964b32-dcdc-49d2-b766-039b707a5c37/1bf1a291-609e-442e-8507-1cdf01804cd1/presenter_delivery.mp4
+week08-lecture15-mapreduce-combiners-spark-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/10964b32-dcdc-49d2-b766-039b707a5c37/b43869c1-6c97-46c6-b1ea-db1a9f8c2d50/presentation_delivery.mp4
+week09-lab08-vagrant-virtualbox-aws-spark-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/6f1deb9e-051b-4019-99ec-19aee14be9d9/132b2d51-2ab1-497e-961f-494f1f00932e/presenter_delivery.mp4
+week09-lab08-vagrant-virtualbox-aws-spark-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/6f1deb9e-051b-4019-99ec-19aee14be9d9/de45cd0f-cba4-4188-8a52-66aa30805708/presentation_delivery.mp4
+week09-lecture16-bayes-theorem-bayesian-methods-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/233f6c34-306f-481b-8ea5-be33076eb6a8/b62f812e-4033-49ad-820c-a0102edba91a/presenter_delivery.mp4
+week09-lecture16-bayes-theorem-bayesian-methods-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/233f6c34-306f-481b-8ea5-be33076eb6a8/bdf7a23c-e8a2-401e-988e-a8d160eef556/presentation_delivery.mp4
+week09-lecture17-bayesian-methods-continued-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/e63c650e-4cf6-4bee-b57e-f16e927ba25a/c46b87e1-b742-45ad-b31a-56d48c74ccdc/presenter_delivery.mp4
+week09-lecture17-bayesian-methods-continued-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/e63c650e-4cf6-4bee-b57e-f16e927ba25a/598826f1-fed0-454f-b06a-4e9e255cf5c9/presentation_delivery.mp4
+week10-lab09-1-bayes-bayes-1-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/2819e79e-d6b9-4988-a0ca-8fc3850ab065/8182fb58-1b42-4b62-a9a7-eff41fa2fd04/presenter_delivery.mp4
+week10-lab09-1-bayes-bayes-1-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/2819e79e-d6b9-4988-a0ca-8fc3850ab065/dbcaf7f0-200c-48f0-aa66-d91748e4f0af/presentation_delivery.mp4
+week10-lab09-2-bayes-bayes-2-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/033ddb0d-55c9-4852-8bf1-d73968b20f8e/e03d7c00-11ac-44a0-884e-7e94e32cbcd5/presenter_delivery.mp4
+week10-lab09-2-bayes-bayes-2-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/033ddb0d-55c9-4852-8bf1-d73968b20f8e/8edfceeb-cab3-43f7-9141-701a2dfa8519/presentation_delivery.mp4
+week10-lecture18-bayesian-methods-continued-text-data-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/8b18c591-27e0-4ea9-8834-8bab321b166a/319e6376-e4ec-4831-85c2-9e1e1cbb1ebd/presenter_delivery.mp4
+week10-lecture18-bayesian-methods-continued-text-data-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/8b18c591-27e0-4ea9-8834-8bab321b166a/49287aaf-5be8-431a-92a4-021b3709226d/presentation_delivery.mp4
+week10-lecture18-bonus-interactive-visualization-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/51e980dd-56a4-4b44-aec5-516ed9c53a2c/73b5b728-3793-4ee4-90f8-57b462613796/presenter_delivery.mp4
+week10-lecture18-bonus-interactive-visualization-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/51e980dd-56a4-4b44-aec5-516ed9c53a2c/f1370fc4-e2bb-44eb-9e2f-912863827300/presentation_delivery.mp4
+week11-lab10-text-clustering-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/52e74ae5-f08b-4bf8-b0a2-b0eeae127856/cda6cc53-4f18-4626-ab49-c1817026a29b/presenter_delivery.mp4
+week11-lab10-text-clustering-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/52e74ae5-f08b-4bf8-b0a2-b0eeae127856/3f2b7669-8149-489d-aa0a-30af9d52beec/presentation_delivery.mp4
+week11-lecture19-clustering-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/27aebc37-3c53-4ce8-a7cb-b2be397aa2f0/7de1665a-f91d-4406-bff5-66d6dd730287/presenter_delivery.mp4
+week11-lecture19-clustering-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/27aebc37-3c53-4ce8-a7cb-b2be397aa2f0/5f8bc0af-bb15-4340-81be-2d54c06ec41f/presentation_delivery.mp4
+week11-lecture20-effective-presentations-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/d213b305-65a8-4ee8-95d3-4dfc4a437205/6cf1a3b6-8e96-49db-9651-4f30596481ae/presenter_delivery.mp4
+week11-lecture20-effective-presentations-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/d213b305-65a8-4ee8-95d3-4dfc4a437205/52767408-4b6f-454b-a7e4-bba0a880bb57/presentation_delivery.mp4
+week12-lab11-projects-example-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/534829b6-1773-4e1d-b25d-aa2a2fbd2b95/a6d8477a-5afb-45fb-b7bd-2b9ab493746b/presenter_delivery.mp4
+week12-lab11-projects-example-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/534829b6-1773-4e1d-b25d-aa2a2fbd2b95/e5c78b33-ad6c-4457-92b6-d01733cc701f/presentation_delivery.mp4
+week12-lecture21-experimental-design-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/15117337-3199-4265-a39a-9a0c8772b9df/20efe48c-04f2-43c6-83e8-ab887b31c007/presenter_delivery.mp4
+week12-lecture21-experimental-design-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/15117337-3199-4265-a39a-9a0c8772b9df/cb8de88e-12fa-4e60-82f4-dca38532f61c/presentation_delivery.mp4
+week12-lecture22-deep-networks-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/3c8c045a-7b5b-41a1-9b96-9e07a7cfb8f5/deaf37ed-2813-492d-96ba-3466f1c43552/presenter_delivery.mp4
+week12-lecture22-deep-networks-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/3c8c045a-7b5b-41a1-9b96-9e07a7cfb8f5/fdc279aa-2b87-43b7-b877-5fd6eeedf237/presentation_delivery.mp4
+week13-lecture23-guest-lecture-building-data-science-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/128b8123-a1a6-493c-bac7-a932234374a0/3fd5c68a-b250-4953-92a6-a6a54f71ed50/presenter_delivery.mp4
+week13-lecture23-guest-lecture-building-data-science-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/128b8123-a1a6-493c-bac7-a932234374a0/c79c2a02-dbf2-4aba-9abc-01c3fde1f59d/presentation_delivery.mp4
+week13-lecture24-wrapup-where-to-go-from-here-presenter,https://da4w749qm6awt.cloudfront.net/engage-player/67735f84-d3c9-406e-86cd-4bfbb38ef1cd/2520cc44-76a6-40e7-b24b-7f82ed90d610/presenter_delivery.mp4
+week13-lecture24-wrapup-where-to-go-from-here-presentation,https://da4w749qm6awt.cloudfront.net/engage-player/67735f84-d3c9-406e-86cd-4bfbb38ef1cd/eed1ead5-9d47-46fd-93b7-d41deca68b40/presentation_delivery.mp4