From 4b87cd707c029414716cc4a9e0fcb4f6d21fc728 Mon Sep 17 00:00:00 2001 From: orttak <28723856+orttak@users.noreply.github.com> Date: Tue, 11 Aug 2020 18:43:16 +0200 Subject: [PATCH] ndvi and lai data were added --- .../Copernicus_1km_LAI_Download.ipynb | 209 ++++++++++++++ .../Copernicus_1km_NDVI_Download.ipynb | 270 ++++++++++++++++++ .../Copernicus_NDVI_Mean_Calcultion.ipynb | 199 +++++++++++++ ndvi_lai_1km/readme.md | 29 ++ 4 files changed, 707 insertions(+) create mode 100644 ndvi_lai_1km/Copernicus_1km_LAI_Download.ipynb create mode 100644 ndvi_lai_1km/Copernicus_1km_NDVI_Download.ipynb create mode 100644 ndvi_lai_1km/Copernicus_NDVI_Mean_Calcultion.ipynb create mode 100644 ndvi_lai_1km/readme.md diff --git a/ndvi_lai_1km/Copernicus_1km_LAI_Download.ipynb b/ndvi_lai_1km/Copernicus_1km_LAI_Download.ipynb new file mode 100644 index 0000000..b6cf91b --- /dev/null +++ b/ndvi_lai_1km/Copernicus_1km_LAI_Download.ipynb @@ -0,0 +1,209 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "from requests.auth import HTTPBasicAuth\n", + "import urllib.request\n", + "import time\n", + "from bs4 import BeautifulSoup\n", + "import pandas as pd\n", + "import time\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# Vito service username and password. we call this information from config.py file.\n", + "import config\n", + "username=config.username\n", + "password=config.password" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "url='https://land.copernicus.vgt.vito.be/PDF/datapool/Vegetation/Properties/LAI_1km_V2/'" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.get(url,auth=(username,password))\n", + "soup=BeautifulSoup(response.text,'html.parser')\n", + "years_divs= soup.find_all(\"td\",{\"class\": \"bydate\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "folder='../geoserver_data/2020_Cop_LAI/'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for y in years_divs[1:]:\n", + " url_year=y.a['href']\n", + " \n", + " ###########################################\n", + " # we use if blog for downloading spesific year\n", + " #if int(url_year[-5:-1])>2018:\n", + " #continue\n", + " #if int(url_year[-5:-1])<2000:\n", + " #continue\n", + " ###########################################\n", + " \n", + " download_path=folder+str(url_year[-5:-1])\n", + " try:\n", + " os.mkdir(download_path)\n", + " except FileExistsError:\n", + " pass\n", + " \n", + " response_year = requests.get(url_year,auth=(username,password))\n", + " page_year=BeautifulSoup(response_year.text,'html.parser')\n", + " mounth_divs = page_year.find_all(\"td\",{\"class\": \"bydate\"})\n", + " \n", + " if len(mounth_divs)==0:\n", + " print(str(url_year)+' year is empty')\n", + " with open(download_path+'/year_nodata_list', 'a') as t:\n", + " t.write(\"%s\\n\" % str(url_year))\n", + " continue\n", + " time.sleep(1)\n", + " for m in mounth_divs[1:]:\n", + " url_mounth=m.a['href']\n", + " response_mounth = requests.get(url_mounth,auth=(username,password))\n", + " page_mounth=BeautifulSoup(response_mounth.text,'html.parser')\n", + " days_divs = page_mounth.find_all(\"td\",{\"class\": \"bydate\"})\n", + " if len(days_divs)==0:\n", + " print(str(url_mounth)+' mounth is empty')\n", + " with open(download_path+'/mounth_nodata_list', 'a') as t:\n", + " t.write(\"%s\\n\" % str(url_mounth))\n", + " \n", + " continue\n", + " time.sleep(1)\n", + " for d in days_divs[1:]:\n", + " url_day=d.a['href']\n", + " response_day_v2 = requests.get(url_day,auth=(username,password))\n", + " page_day_v2=BeautifulSoup(response_day_v2.text,'html.parser')\n", + " days_divs_v2 = page_day_v2.find_all(\"td\",{\"class\": \"bydate\"})\n", + " if len(days_divs_v2)==0:\n", + " print(str(url_day)+' is empty')\n", + " with open(download_path+'/daily_nodata_list', 'a') as t:\n", + " t.write(\"%s\\n\" % str(url_day)) \n", + " continue\n", + " \n", + " url_day_v2=days_divs_v2[1].a['href']\n", + " response_img = requests.get(url_day_v2,auth=(username,password))\n", + " page_img=BeautifulSoup(response_img.text,'html.parser')\n", + " img_divs = page_img.find_all('td')\n", + " tag=img_divs[2]\n", + " nc_name=tag.text[6:-15]\n", + " nc_url=url_day_v2+nc_name\n", + " image_bool=False\n", + " try:\n", + " while image_bool is False:\n", + " try:\n", + " #print(nc_url)\n", + " response = requests.get(nc_url, stream=True,auth=(username,password))\n", + " if response.status_code == 200:\n", + " with open(download_path+'/'+nc_name, 'wb') as f:\n", + " f.write(response.raw.read())\n", + " image_bool=True\n", + "\n", + " time.sleep(4)\n", + " \n", + " except ProtocolError:\n", + " time.sleep(5)\n", + " image_bool=False\n", + " pass\n", + " except ConnectionError:\n", + " time.sleep(5)\n", + " image_bool=False\n", + " pass\n", + " except:\n", + " time.sleep(5)\n", + " image_bool=False\n", + " pass\n", + " \n", + " \n", + " with open(download_path+'/download_list', 'a') as t:\n", + " t.write(\"%s\\n\" % nc_url) \n", + " \n", + " \n", + " \n", + " except Exception as e:\n", + " with open(download_path+'/error_list', 'a') as t:\n", + " t.write(\"%s\\n\" % nc_url)\n", + " \n", + " continue\n", + " \n", + " \n", + " \n", + " time.sleep(10) \n", + " \n", + " \n", + " \n", + " time.sleep(30)\n", + " print(download_path) \n", + " print('-------------------------------------------')\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/ndvi_lai_1km/Copernicus_1km_NDVI_Download.ipynb b/ndvi_lai_1km/Copernicus_1km_NDVI_Download.ipynb new file mode 100644 index 0000000..887d9bd --- /dev/null +++ b/ndvi_lai_1km/Copernicus_1km_NDVI_Download.ipynb @@ -0,0 +1,270 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "from requests.auth import HTTPBasicAuth\n", + "import urllib.request\n", + "import time\n", + "from bs4 import BeautifulSoup\n", + "import pandas as pd\n", + "import time\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# Vito service username and password. we call this information from config.py file.\n", + "import config\n", + "username=config.username\n", + "password=config.password" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "url='https://land.copernicus.vgt.vito.be/PDF/datapool/Vegetation/Indicators/NDVI_1km_V2/'" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.get(url,auth=(username,password))\n", + "soup=BeautifulSoup(response.text,'html.parser')\n", + "years_divs= soup.find_all(\"td\",{\"class\": \"bydate\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "folder='../geoserver_data/2020_Cop_NDVI/'" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../geoserver_data/2020_Cop_NDVI/2018\n", + "-------------------------------------------\n" + ] + } + ], + "source": [ + "for y in years_divs[1:]:\n", + " url_year=y.a['href']\n", + " # we use if blog for downloading spesific year\n", + " #if int(url_year[-5:-1])>2018:\n", + " #continue\n", + " #if int(url_year[-5:-1])<2018:\n", + " #continue\n", + " \n", + " #create download path according to year \n", + " download_path=folder+str(url_year[-5:-1])\n", + " try:\n", + " os.mkdir(download_path)\n", + " except FileExistsError:\n", + " pass\n", + " \n", + " response_year = requests.get(url_year,auth=(username,password))\n", + " page_year=BeautifulSoup(response_year.text,'html.parser')\n", + " mounth_divs = page_year.find_all(\"td\",{\"class\": \"bydate\"})\n", + " \n", + " if len(mounth_divs)==0:\n", + " # if year path has no data. We save this info and skip this year\n", + " print(str(url_year)+' year is empty')\n", + " with open(download_path+'/year_nodata_list', 'a') as t:\n", + " t.write(\"%s\\n\" % str(url_year))\n", + " continue\n", + " \n", + " for m in mounth_divs[1:]:\n", + " url_mounth=m.a['href']\n", + " response_mounth = requests.get(url_mounth,auth=(username,password))\n", + " page_mounth=BeautifulSoup(response_mounth.text,'html.parser')\n", + " days_divs = page_mounth.find_all(\"td\",{\"class\": \"bydate\"})\n", + " if len(days_divs)==0:\n", + " #same idea as a year control\n", + " print(str(url_mounth)+' mounth is empty')\n", + " with open(download_path+'/mounth_nodata_list', 'a') as t:\n", + " t.write(\"%s\\n\" % str(url_mounth))\n", + " \n", + " continue\n", + " \n", + " for d in days_divs[1:]:\n", + " url_day=d.a['href']\n", + " response_day_v2 = requests.get(url_day,auth=(username,password))\n", + " page_day_v2=BeautifulSoup(response_day_v2.text,'html.parser')\n", + " days_divs_v2 = page_day_v2.find_all(\"td\",{\"class\": \"bydate\"})\n", + " if len(days_divs_v2)==0:\n", + " print(str(url_day)+' is empty')\n", + " with open(download_path+'/daily_nodata_list', 'a') as t:\n", + " t.write(\"%s\\n\" % str(url_day)) \n", + " continue\n", + " \n", + " url_day_v2=days_divs_v2[1].a['href']\n", + " response_img = requests.get(url_day_v2,auth=(username,password))\n", + " page_img=BeautifulSoup(response_img.text,'html.parser')\n", + " img_divs = page_img.find_all('td')\n", + " tag=img_divs[2]\n", + " nc_name=tag.text[6:-15]\n", + " nc_url=url_day_v2+nc_name\n", + " image_bool=False\n", + " try:\n", + " while image_bool is False:\n", + " try:\n", + " #print(nc_url)\n", + " response = requests.get(nc_url, stream=True,auth=(username,password))\n", + " if response.status_code == 200:\n", + " with open(download_path+'/'+nc_name, 'wb') as f:\n", + " f.write(response.raw.read())\n", + " image_bool=True\n", + " else:\n", + " with open(download_path+'/error_list', 'a') as t:\n", + " t.write(\"Response status code is not 200. %s\\n\" % nc_url)\n", + "\n", + " time.sleep(1)\n", + " \n", + " except ProtocolError:\n", + " time.sleep(5)\n", + " image_bool=False\n", + " pass\n", + " except ConnectionError:\n", + " time.sleep(5)\n", + " image_bool=False\n", + " pass\n", + " except:\n", + " time.sleep(5)\n", + " image_bool=False\n", + " pass\n", + " \n", + " \n", + " with open(download_path+'/download_list', 'a') as t:\n", + " t.write(\"%s\\n\" % nc_url) \n", + " \n", + " \n", + " \n", + " except Exception as e:\n", + " with open(download_path+'/error_list', 'a') as t:\n", + " t.write(\"%s\\n\" % nc_url)\n", + " \n", + " continue\n", + " \n", + " \n", + " \n", + " time.sleep(10) \n", + " \n", + " \n", + " \n", + " time.sleep(15)\n", + " print(download_path) \n", + " print('-------------------------------------------')\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/ndvi_lai_1km/Copernicus_NDVI_Mean_Calcultion.ipynb b/ndvi_lai_1km/Copernicus_NDVI_Mean_Calcultion.ipynb new file mode 100644 index 0000000..1223a8d --- /dev/null +++ b/ndvi_lai_1km/Copernicus_NDVI_Mean_Calcultion.ipynb @@ -0,0 +1,199 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import glob\n", + "from osgeo import gdal, osr" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import numpy as np\n", + "import dask.array as da\n", + "import time\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def ListofExtensionAndName(directory,extension):\n", + " \n", + " if len(directory) != None:\n", + " import os\n", + " FilesList = []\n", + " FileName=[]\n", + " for root, subdirectory, files in os.walk(directory):\n", + " for file in files:\n", + " if file.endswith(extension):\n", + " FilesList.append(os.path.join(root,file))\n", + " base=os.path.basename(file)\n", + " FileName.append(os.path.splitext(base)[0])\n", + "\n", + " return sorted(FilesList),sorted(FileName)\n", + " else:\n", + " print(\"no\"+ extension +\"file for this directory\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'../geoserver_data/2020_Cop_LAI/2000/'" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "folder_list=glob.glob('../geoserver_data/2020_NDVI/*/')\n", + "folder_list[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "folder_list2=['../data/cop_ndvi/2005/','../data/cop_ndvi/2019/']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# You should give list of nc files' path like folder_list2 or folder_list" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "for folder in folder_list2:\n", + " image_status=False\n", + " counter=0\n", + " while image_status is False:\n", + " try:\n", + " nc_files=ListofExtensionAndName(folder,'.nc')\n", + " folder_name=folder.split('/')[-2]\n", + " output_folder='../data/cop_ndvi/mean_tif/'\n", + " array_list=[]\n", + " for index,img in enumerate(nc_files[0]):\n", + " # with this block, we read 36 .nc file' ndvi value\n", + " # with dask array\n", + " netcdf_name=img\n", + " # if you work with LAI change the variable\n", + " #layer_name='LAI'\n", + " layer_name='NDVI'\n", + " ds = gdal.Open(\"NETCDF:{0}:{1}\".format(netcdf_name, layer_name))\n", + " array=ds.ReadAsArray()\n", + " dask_array=da.from_array(array,chunks=len(array) // 4)\n", + " array_list.append(dask_array)\n", + " \n", + " \n", + " stack = da.stack(array_list, axis=0)\n", + " mean_array=stack.mean(axis=0).compute()\n", + " s_srs = ds.GetProjectionRef() \n", + " osng = osr.SpatialReference ()\n", + " \n", + " osng.SetFromUserInput ( s_srs ) \n", + " geo_t = ds.GetGeoTransform () \n", + " x_size = ds.RasterXSize # Raster xsize\n", + " y_size = ds.RasterYSize # Raster ysize\n", + " mem_drv = gdal.GetDriverByName( 'MEM')\n", + " dest = mem_drv.Create('', x_size,y_size, 1)\n", + " dest.SetGeoTransform( geo_t )\n", + " dest.SetProjection( osng.ExportToWkt())\n", + " dest.GetRasterBand(1).WriteArray(mean_array)\n", + " output_mean_path=output_folder+'c_gls_NDVI_Mean_'+folder_name+'.tif' \n", + " gdal.Warp(output_mean_path, dest, format = 'GTiff')\n", + " print(output_mean_path)\n", + " mean_array=None\n", + " dest=None\n", + " ds=None\n", + " mem_drv=None\n", + " image_status=True\n", + " except:\n", + " print('except')\n", + " image_status=False\n", + " counter+=1\n", + " if counter==10:\n", + " print(folder + 'couldnt be calculated')\n", + " break\n", + " continue" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/ndvi_lai_1km/readme.md b/ndvi_lai_1km/readme.md new file mode 100644 index 0000000..98c87b3 --- /dev/null +++ b/ndvi_lai_1km/readme.md @@ -0,0 +1,29 @@ +### Copernicus NDVI & LAI 1km Data Process + +With these notebooks, You can download NDVI and LAI 1 km data from Copernicus Global Land Service . Also,You can crete new annual base max,min or mean data with "ndvi_mean_calculatin" notebook. When you use 'mean_calculation' notebook, you should be careful at "layer_name" variable. According your data( NDVI or LAI ), you should change this value. + +##### NDVI + +The Normalized Difference Vegetation Index (NDVI) is an indicator of the greenness of the biomes. +Even though it is not a physical property of the vegetation cover, its very simple formulation NDVI = (REF_nir – REF_red)/(REF_nir + REF_red) +where REF_nir and REF_red are the spectral reflectances measured in the near infrared and red wavebands respectively, makes it widely used for ecosystems monitoring. + +##### Leaf Area Index(LAI) + +The Leaf Area Index is defined as half the total area of green elements of the canopy per unit horizontal ground area. The satellite-derived value corresponds to the total green LAI of all the canopy layers, including the understory which may represent a very significant contribution, particularly for forests. Practically, the LAI quantifies the thickness of the vegetation cover. + +LAI is recognized as an Essential Climate Variable (ECV) by the Global Climate Observing System (GCOS) + + + +ndvi main page: https://land.copernicus.eu/global/products/ndvi +ndvi product page: https://land.copernicus.vgt.vito.be/PDF/datapool/Vegetation/Indicators/NDVI_1km_V2 + +lai main page: https://land.copernicus.eu/global/products/lai +lai prodcut page: https://land.copernicus.vgt.vito.be/PDF/datapool/Vegetation/Properties/LAI_1km_V2 + + + + + +