Skip to content

Commit

Permalink
ndvi and lai data were added
Browse files Browse the repository at this point in the history
  • Loading branch information
@orttak
orttak authored Aug 11, 2020
0 parents commit 4b87cd7
Show file tree
Hide file tree
Showing 4 changed files with 707 additions and 0 deletions.
209 changes: 209 additions & 0 deletions ndvi_lai_1km/Copernicus_1km_LAI_Download.ipynb
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"from requests.auth import HTTPBasicAuth\n",
"import urllib.request\n",
"import time\n",
"from bs4 import BeautifulSoup\n",
"import pandas as pd\n",
"import time\n",
"import os"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"# Vito service username and password. we call this information from config.py file.\n",
"import config\n",
"username=config.username\n",
"password=config.password"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"url='https://land.copernicus.vgt.vito.be/PDF/datapool/Vegetation/Properties/LAI_1km_V2/'"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"response = requests.get(url,auth=(username,password))\n",
"soup=BeautifulSoup(response.text,'html.parser')\n",
"years_divs= soup.find_all(\"td\",{\"class\": \"bydate\"})"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"folder='../geoserver_data/2020_Cop_LAI/'"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"for y in years_divs[1:]:\n",
" url_year=y.a['href']\n",
" \n",
" ###########################################\n",
" # we use if blog for downloading spesific year\n",
" #if int(url_year[-5:-1])>2018:\n",
" #continue\n",
" #if int(url_year[-5:-1])<2000:\n",
" #continue\n",
" ###########################################\n",
" \n",
" download_path=folder+str(url_year[-5:-1])\n",
" try:\n",
" os.mkdir(download_path)\n",
" except FileExistsError:\n",
" pass\n",
" \n",
" response_year = requests.get(url_year,auth=(username,password))\n",
" page_year=BeautifulSoup(response_year.text,'html.parser')\n",
" mounth_divs = page_year.find_all(\"td\",{\"class\": \"bydate\"})\n",
" \n",
" if len(mounth_divs)==0:\n",
" print(str(url_year)+' year is empty')\n",
" with open(download_path+'/year_nodata_list', 'a') as t:\n",
" t.write(\"%s\\n\" % str(url_year))\n",
" continue\n",
" time.sleep(1)\n",
" for m in mounth_divs[1:]:\n",
" url_mounth=m.a['href']\n",
" response_mounth = requests.get(url_mounth,auth=(username,password))\n",
" page_mounth=BeautifulSoup(response_mounth.text,'html.parser')\n",
" days_divs = page_mounth.find_all(\"td\",{\"class\": \"bydate\"})\n",
" if len(days_divs)==0:\n",
" print(str(url_mounth)+' mounth is empty')\n",
" with open(download_path+'/mounth_nodata_list', 'a') as t:\n",
" t.write(\"%s\\n\" % str(url_mounth))\n",
" \n",
" continue\n",
" time.sleep(1)\n",
" for d in days_divs[1:]:\n",
" url_day=d.a['href']\n",
" response_day_v2 = requests.get(url_day,auth=(username,password))\n",
" page_day_v2=BeautifulSoup(response_day_v2.text,'html.parser')\n",
" days_divs_v2 = page_day_v2.find_all(\"td\",{\"class\": \"bydate\"})\n",
" if len(days_divs_v2)==0:\n",
" print(str(url_day)+' is empty')\n",
" with open(download_path+'/daily_nodata_list', 'a') as t:\n",
" t.write(\"%s\\n\" % str(url_day)) \n",
" continue\n",
" \n",
" url_day_v2=days_divs_v2[1].a['href']\n",
" response_img = requests.get(url_day_v2,auth=(username,password))\n",
" page_img=BeautifulSoup(response_img.text,'html.parser')\n",
" img_divs = page_img.find_all('td')\n",
" tag=img_divs[2]\n",
" nc_name=tag.text[6:-15]\n",
" nc_url=url_day_v2+nc_name\n",
" image_bool=False\n",
" try:\n",
" while image_bool is False:\n",
" try:\n",
" #print(nc_url)\n",
" response = requests.get(nc_url, stream=True,auth=(username,password))\n",
" if response.status_code == 200:\n",
" with open(download_path+'/'+nc_name, 'wb') as f:\n",
" f.write(response.raw.read())\n",
" image_bool=True\n",
"\n",
" time.sleep(4)\n",
" \n",
" except ProtocolError:\n",
" time.sleep(5)\n",
" image_bool=False\n",
" pass\n",
" except ConnectionError:\n",
" time.sleep(5)\n",
" image_bool=False\n",
" pass\n",
" except:\n",
" time.sleep(5)\n",
" image_bool=False\n",
" pass\n",
" \n",
" \n",
" with open(download_path+'/download_list', 'a') as t:\n",
" t.write(\"%s\\n\" % nc_url) \n",
" \n",
" \n",
" \n",
" except Exception as e:\n",
" with open(download_path+'/error_list', 'a') as t:\n",
" t.write(\"%s\\n\" % nc_url)\n",
" \n",
" continue\n",
" \n",
" \n",
" \n",
" time.sleep(10) \n",
" \n",
" \n",
" \n",
" time.sleep(30)\n",
" print(download_path) \n",
" print('-------------------------------------------')\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.9"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Loading

0 comments on commit 4b87cd7

Please sign in to comment.