From 1dec8f5de36311fbfd2b1d67787f5790ce2d58f1 Mon Sep 17 00:00:00 2001 From: bpstewar Date: Thu, 29 Feb 2024 13:42:08 -0500 Subject: [PATCH 1/3] Initiated support to Sri Lanka CCDR --- .../URB_LKA_CCDR_Support/README.md | 11 + .../URB_LKA_CCDR_Support/Untitled.ipynb | 209 ++++++++++++++++++ 2 files changed, 220 insertions(+) create mode 100644 notebooks/Implementations/URB_LKA_CCDR_Support/README.md create mode 100644 notebooks/Implementations/URB_LKA_CCDR_Support/Untitled.ipynb diff --git a/notebooks/Implementations/URB_LKA_CCDR_Support/README.md b/notebooks/Implementations/URB_LKA_CCDR_Support/README.md new file mode 100644 index 0000000..a8b0bef --- /dev/null +++ b/notebooks/Implementations/URB_LKA_CCDR_Support/README.md @@ -0,0 +1,11 @@ +# Urbanization for CCDR +The Sri Lanka CCDR team is looking for support in urban deep dives on 9 provincial capitals. The following analysis will be completed: + +| Analysis | Phase | Notebook | Plan | +| --- | --- | --- | --- | +| Urbanization analysis | 1 | | Compare official and buffered boundaries to quantified boundaries (DoU); summarize population within all boundaries | +| Urban expansion | 1 | | Summariuze GHSL within urban extents | +| Urban Poverty | 3 | | Analyze high-resolution satellite imagery to identify informality in residential settlements | +| Urban economics | 2 | | Summarize monthly nighttime lights | +| Status of infrastructure | 2 | | TBD | +| GHG emissions | 1 | | Summarize Brian B's GHG emissions data | \ No newline at end of file diff --git a/notebooks/Implementations/URB_LKA_CCDR_Support/Untitled.ipynb b/notebooks/Implementations/URB_LKA_CCDR_Support/Untitled.ipynb new file mode 100644 index 0000000..0901847 --- /dev/null +++ b/notebooks/Implementations/URB_LKA_CCDR_Support/Untitled.ipynb @@ -0,0 +1,209 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "9fd5e3dd", + "metadata": {}, + "source": [ + "# Urbanization comparison\n", + "\n", + "Compare the multiple urban extents in the project\n", + "\n", + "1. Official administrative boundaries provided by the project team. \n", + " a. Need to get metadata on these boundaries from Swati \n", + "2. Official administrative boundaries with buffers \n", + " a. Why these specific buffer distances? \n", + "3. DoU boundaries \n", + " a. Which population source \n" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "9c6da02f", + "metadata": {}, + "outputs": [], + "source": [ + "import sys, os\n", + "import rasterio\n", + "\n", + "import pandas as pd\n", + "import geopandas as gpd\n", + "\n", + "import GOSTurban.UrbanRaster as urban\n" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "df1bf2ed", + "metadata": {}, + "outputs": [], + "source": [ + "base_folder = \"C:/WBG/Work/LKA_URB_CCDR/Data\"\n", + "urban_folder = os.path.join(base_folder, \"Urban_extents\")\n", + "population_folder = os.path.join(base_folder, \"Population\")\n", + "\n", + "worldpop_file = os.path.join(population_folder, \"lka_ppp_2020_UNadj.tif\")\n", + "combo_extents = os.path.join(urban_folder, \"combo_extents.shp\")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "b135cd57", + "metadata": {}, + "outputs": [], + "source": [ + "# Combine all the urban extents into a single file\n", + "if not os.path.exists(combo_extents):\n", + " urban_extents = [x for x in os.listdir(os.path.join(urban_folder, \"Final_AoI\")) if x.endswith(\".shp\")]\n", + " all_extents = []\n", + " for urban_extent in urban_extents:\n", + " urban_extent_gdf = gpd.read_file(os.path.join(urban_folder, \"Final_AoI\", urban_extent))\n", + " type = urban_extent.split(\"_\")[1].replace(\".shp\", \"\").lower()\n", + " if type == \"colombo\":\n", + " type = \"city\"\n", + " if \"buffer\" in urban_extent:\n", + " type = \"20km\"\n", + " urban_extent_gdf[\"Type\"] = type\n", + " all_extents.append(urban_extent_gdf)\n", + "\n", + " all_extents_gdf = gpd.GeoDataFrame(pd.concat(all_extents, ignore_index=True))\n", + " all_extents_gdf.to_file(combo_extents)\n", + "else:\n", + " all_extents_gdf = gpd.read_file(combo_extents)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "97add9be", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "13:19:42\t: Read in urban data\n", + "13:19:42\t: Creating Shape 0\n", + "13:19:45\t: Read in urban data\n", + "13:19:45\t: Creating Shape 0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\wbg\\Anaconda3\\envs\\urban_test\\Lib\\site-packages\\GOSTurban\\UrbanRaster.py:404: UserWarning: Geometry is in a geographic CRS. Results from 'buffer' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " xxGeom[\"geometry\"] = xxGeom.buffer((popRaster.res[0] / 2))\n" + ] + }, + { + "ename": "ValueError", + "evalue": "graph should have two dimensions", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[29], line 14\u001b[0m\n\u001b[0;32m 11\u001b[0m urban_extents[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mType\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m name\n\u001b[0;32m 12\u001b[0m urban_extents\u001b[38;5;241m.\u001b[39mto_file(urb_out_file)\n\u001b[1;32m---> 14\u001b[0m hd_urban_extents \u001b[38;5;241m=\u001b[39m urban_calculator\u001b[38;5;241m.\u001b[39mcalculateUrban(\n\u001b[0;32m 15\u001b[0m densVal\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1500\u001b[39m,\n\u001b[0;32m 16\u001b[0m totalPopThresh\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m50000\u001b[39m,\n\u001b[0;32m 17\u001b[0m smooth\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[0;32m 18\u001b[0m queen\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[38;5;66;03m# high density extents use queen's case contiguity, and\u001b[39;00m\n\u001b[0;32m 19\u001b[0m verbose\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[0;32m 20\u001b[0m ) \u001b[38;5;66;03m# High density extents have hole smoothing applied.\u001b[39;00m\n\u001b[0;32m 21\u001b[0m hd_urban_extents[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mType\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m name\n\u001b[0;32m 22\u001b[0m hd_urban_extents\u001b[38;5;241m.\u001b[39mto_file(hd_out_file)\n", + "File \u001b[1;32mc:\\wbg\\Anaconda3\\envs\\urban_test\\Lib\\site-packages\\GOSTurban\\UrbanRaster.py:407\u001b[0m, in \u001b[0;36murbanGriddedPop.calculateUrban\u001b[1;34m(self, densVal, totalPopThresh, smooth, verbose, queen, raster, raster_pop, print_message)\u001b[0m\n\u001b[0;32m 405\u001b[0m s \u001b[38;5;241m=\u001b[39m xxGeom[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgeometry\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[0;32m 406\u001b[0m overlap_matrix \u001b[38;5;241m=\u001b[39m s\u001b[38;5;241m.\u001b[39mapply(\u001b[38;5;28;01mlambda\u001b[39;00m x: s\u001b[38;5;241m.\u001b[39mintersects(x))\u001b[38;5;241m.\u001b[39mvalues\u001b[38;5;241m.\u001b[39mastype(\u001b[38;5;28mint\u001b[39m)\n\u001b[1;32m--> 407\u001b[0m n, ids \u001b[38;5;241m=\u001b[39m connected_components(overlap_matrix)\n\u001b[0;32m 408\u001b[0m xxGeom[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgroup\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m ids\n\u001b[0;32m 409\u001b[0m xxGeom \u001b[38;5;241m=\u001b[39m xxGeom\u001b[38;5;241m.\u001b[39mdissolve(by\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgroup\u001b[39m\u001b[38;5;124m\"\u001b[39m, aggfunc\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msum\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[1;32m_traversal.pyx:97\u001b[0m, in \u001b[0;36mscipy.sparse.csgraph._traversal.connected_components\u001b[1;34m()\u001b[0m\n", + "File \u001b[1;32mc:\\wbg\\Anaconda3\\envs\\urban_test\\Lib\\site-packages\\scipy\\sparse\\csgraph\\_validation.py:46\u001b[0m, in \u001b[0;36mvalidate_graph\u001b[1;34m(csgraph, directed, dtype, csr_output, dense_output, copy_if_dense, copy_if_sparse, null_value_in, null_value_out, infinity_null, nan_null)\u001b[0m\n\u001b[0;32m 44\u001b[0m csgraph[mask] \u001b[38;5;241m=\u001b[39m null_value_out\n\u001b[0;32m 45\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m---> 46\u001b[0m csgraph \u001b[38;5;241m=\u001b[39m csgraph_from_dense(csgraph, null_value\u001b[38;5;241m=\u001b[39mnull_value_in,\n\u001b[0;32m 47\u001b[0m infinity_null\u001b[38;5;241m=\u001b[39minfinity_null,\n\u001b[0;32m 48\u001b[0m nan_null\u001b[38;5;241m=\u001b[39mnan_null)\n\u001b[0;32m 50\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m csgraph\u001b[38;5;241m.\u001b[39mndim \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m2\u001b[39m:\n\u001b[0;32m 51\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcompressed-sparse graph must be 2-D\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[1;32m_tools.pyx:215\u001b[0m, in \u001b[0;36mscipy.sparse.csgraph._tools.csgraph_from_dense\u001b[1;34m()\u001b[0m\n", + "File \u001b[1;32m_tools.pyx:140\u001b[0m, in \u001b[0;36mscipy.sparse.csgraph._tools.csgraph_masked_from_dense\u001b[1;34m()\u001b[0m\n", + "\u001b[1;31mValueError\u001b[0m: graph should have two dimensions" + ] + } + ], + "source": [ + "# extract urban extents from population grids\n", + "for pop_layer, name in [[worldpop_file, \"WorldPop\"]]:\n", + " urb_out_file = os.path.join(urban_folder, \"DOU_urb_extent_%s.shp\" % name)\n", + " hd_out_file = os.path.join(urban_folder, \"DOU_hd_extent_%s.shp\" % name)\n", + " if not os.path.exists(hd_out_file):\n", + " curPop = rasterio.open(pop_layer)\n", + " urban_calculator = urban.urbanGriddedPop(curPop)\n", + " urban_extents = urban_calculator.calculateUrban(\n", + " densVal=3, totalPopThresh=5000, smooth=False, queen=False, verbose=True\n", + " )\n", + " urban_extents['Type'] = name\n", + " urban_extents.to_file(urb_out_file)\n", + "\n", + " hd_urban_extents = urban_calculator.calculateUrban(\n", + " densVal=15,\n", + " totalPopThresh=50000,\n", + " smooth=True,\n", + " queen=True, # high density extents use queen's case contiguity, and\n", + " verbose=True,\n", + " ) # High density extents have hole smoothing applied.\n", + " hd_urban_extents['Type'] = name\n", + " hd_urban_extents.to_file(hd_out_file)\n", + "\n", + "\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "34d495aa", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['__builtins__',\n", + " '__cached__',\n", + " '__doc__',\n", + " '__file__',\n", + " '__loader__',\n", + " '__name__',\n", + " '__package__',\n", + " '__path__',\n", + " '__spec__',\n", + " '__version__',\n", + " '_version']" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dir(GOSTurban)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "50115a0d", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "urban_test", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From a1dfa9de9c5d3d02a3d2c54eb53e214fcdcb94ee Mon Sep 17 00:00:00 2001 From: "Benjamin P. Stewart" Date: Sat, 25 May 2024 11:12:12 +0000 Subject: [PATCH 2/3] LKA urbanization and MENA --- .../MENA_Benchmarking/NTL_zonal_stats.ipynb | 270 ++++++++++++++++++ .../URB_LKA_CCDR_Support/LKA_Urban_Comp.ipynb | 152 ++++++++++ .../URB_LKA_CCDR_Support/Untitled.ipynb | 209 -------------- 3 files changed, 422 insertions(+), 209 deletions(-) create mode 100644 notebooks/Implementations/MENA_Benchmarking/NTL_zonal_stats.ipynb create mode 100644 notebooks/Implementations/URB_LKA_CCDR_Support/LKA_Urban_Comp.ipynb delete mode 100644 notebooks/Implementations/URB_LKA_CCDR_Support/Untitled.ipynb diff --git a/notebooks/Implementations/MENA_Benchmarking/NTL_zonal_stats.ipynb b/notebooks/Implementations/MENA_Benchmarking/NTL_zonal_stats.ipynb new file mode 100644 index 0000000..0426341 --- /dev/null +++ b/notebooks/Implementations/MENA_Benchmarking/NTL_zonal_stats.ipynb @@ -0,0 +1,270 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Benchmarking cities in MENA\n", + "\n", + "In support of an upcoming Urban flagship report, the MENA team is looking for a series of zonal statistics:\n", + "\n", + "- Nighttime Lights, Population, and built-area: \n", + " - Entire FUA \n", + " - Its associated urban center / “core” \n", + " - Associated “periphery” \n", + "\n", + "The unit of analysis is the Functional Urban Areas (FUAs) from the [UCDB Database](https://human-settlement.emergency.copernicus.eu/ghs_stat_ucdb2015mt_r2019a.php). For each FUA, we need to grab the associated urban periphary (lower threshold urban areas)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "import sys, os\n", + "import rasterio\n", + "\n", + "import pandas as pd\n", + "import geopandas as gpd\n", + "import numpy as np\n", + "\n", + "from shapely.geometry import Point\n", + "\n", + "sys.path.append(\"C:/WBG/Work/Code/GOSTrocks/src\")\n", + "\n", + "import GOSTrocks.rasterMisc as rMisc\n", + "from GOSTrocks.misc import tPrint" + ] + }, + { + "cell_type": "code", + "execution_count": 83, + "metadata": {}, + "outputs": [], + "source": [ + "data_folder = \"C:/WBG/Work/data\"\n", + "ucdb_file = os.path.join(data_folder, \"URBAN\",\"GHS_STAT_UCDB2015MT_GLOBE_R2019A\", \"GHS_STAT_UCDB2015MT_GLOBE_R2019A_V1_2.gpkg\")\n", + "fua_file = os.path.join(data_folder, \"URBAN\", \"GHS_FUA_UCDB2015_GLOBE_R2019A_54009_1K_V1_0.gpkg\")\n", + "\n", + "out_folder = \"C:/WBG/Work/MENA_Urban/\"\n", + "urban_res_folder = os.path.join(out_folder, \"urban_data\")\n", + "\n", + "for out_folder in [urban_res_folder]:\n", + " if not os.path.exists(out_folder):\n", + " os.makedirs(out_folder)\n", + "\n", + "urban_periphary_file = os.path.join(urban_res_folder, \"urban_periphary.gpkg\")\n", + "urban_core_file = os.path.join(urban_res_folder, \"urban_core.gpkg\")" + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "metadata": {}, + "outputs": [], + "source": [ + "inD = gpd.read_file(ucdb_file)\n", + "inF = gpd.read_file(fua_file)\n", + "inD = inD.to_crs(inF.crs)" + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
eFUA_IDUC_numUC_IDseFUA_nameCommutingCntry_ISOCntry_nameFUA_areaUC_areaFUA_p_2015UC_p_2015Com_p_2015geometry
01.01.05959Herat1.0AFGAfghanistan139.060.0888811.175807844574.56433144236.611476MULTIPOLYGON (((5529000.000 4155000.000, 55310...
1192.01.05964Guzarah1.0AFGAfghanistan32.016.0169489.573231160204.2608649285.312366MULTIPOLYGON (((5541000.000 4139000.000, 55420...
2354.01.05968Shindand0.0AFGAfghanistan12.012.0147553.403870147553.4038700.000000MULTIPOLYGON (((5573000.000 4030000.000, 55740...
3505.01.05970Qala i Naw0.0AFGAfghanistan3.03.079809.72265679809.7226560.000000MULTIPOLYGON (((5592000.000 4221000.000, 55900...
4648.01.05973Farah1.0AFGAfghanistan32.016.0131508.797060122843.4603278665.336733MULTIPOLYGON (((5607000.000 3923000.000, 56080...
\n", + "
" + ], + "text/plain": [ + " eFUA_ID UC_num UC_IDs eFUA_name Commuting Cntry_ISO Cntry_name \\\n", + "0 1.0 1.0 5959 Herat 1.0 AFG Afghanistan \n", + "1 192.0 1.0 5964 Guzarah 1.0 AFG Afghanistan \n", + "2 354.0 1.0 5968 Shindand 0.0 AFG Afghanistan \n", + "3 505.0 1.0 5970 Qala i Naw 0.0 AFG Afghanistan \n", + "4 648.0 1.0 5973 Farah 1.0 AFG Afghanistan \n", + "\n", + " FUA_area UC_area FUA_p_2015 UC_p_2015 Com_p_2015 \\\n", + "0 139.0 60.0 888811.175807 844574.564331 44236.611476 \n", + "1 32.0 16.0 169489.573231 160204.260864 9285.312366 \n", + "2 12.0 12.0 147553.403870 147553.403870 0.000000 \n", + "3 3.0 3.0 79809.722656 79809.722656 0.000000 \n", + "4 32.0 16.0 131508.797060 122843.460327 8665.336733 \n", + "\n", + " geometry \n", + "0 MULTIPOLYGON (((5529000.000 4155000.000, 55310... \n", + "1 MULTIPOLYGON (((5541000.000 4139000.000, 55420... \n", + "2 MULTIPOLYGON (((5573000.000 4030000.000, 55740... \n", + "3 MULTIPOLYGON (((5592000.000 4221000.000, 55900... \n", + "4 MULTIPOLYGON (((5607000.000 3923000.000, 56080... " + ] + }, + "execution_count": 85, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "inF.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 113, + "metadata": {}, + "outputs": [], + "source": [ + "fua_peripheries = inF.copy()\n", + "for idx, row in inF.iterrows():\n", + " # grab the related UCDBs\n", + " ucdb_ids = row['UC_IDs'].split(\";\")\n", + " ucdb_ids = [int(x) for x in ucdb_ids]\n", + " sel_cores = inD.loc[inD['ID_HDC_G0'].isin(ucdb_ids)]\n", + " periphery_geom = row['geometry'].difference(sel_cores.unary_union)\n", + " fua_peripheries.loc[idx, 'geometry'] = periphery_geom\n", + "\n", + "fua_peripheries.to_file(os.path.join(out_folder, 'FUA_peripheries.gpkg'), driver='GPKG')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "urban_test", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/Implementations/URB_LKA_CCDR_Support/LKA_Urban_Comp.ipynb b/notebooks/Implementations/URB_LKA_CCDR_Support/LKA_Urban_Comp.ipynb new file mode 100644 index 0000000..32dfcaf --- /dev/null +++ b/notebooks/Implementations/URB_LKA_CCDR_Support/LKA_Urban_Comp.ipynb @@ -0,0 +1,152 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "9fd5e3dd", + "metadata": {}, + "source": [ + "# Urbanization comparison\n", + "\n", + "Compare the multiple urban extents in the project\n", + "\n", + "1. Official administrative boundaries provided by the project team. \n", + " a. Need to get metadata on these boundaries from Swati \n", + "2. Official administrative boundaries with buffers \n", + " a. Why these specific buffer distances? \n", + "3. DoU boundaries \n", + " a. Which population source \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9c6da02f", + "metadata": {}, + "outputs": [], + "source": [ + "import sys, os\n", + "import rasterio\n", + "\n", + "import pandas as pd\n", + "import geopandas as gpd\n", + "\n", + "import GOSTurban.UrbanRaster as urban\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df1bf2ed", + "metadata": {}, + "outputs": [], + "source": [ + "base_folder = \"C:/WBG/Work/LKA_URB_CCDR/Data\"\n", + "urban_folder = os.path.join(base_folder, \"Urban_extents\")\n", + "population_folder = os.path.join(base_folder, \"Population\")\n", + "\n", + "worldpop_file = os.path.join(population_folder, \"lka_ppp_2020_UNadj.tif\")\n", + "combo_extents = os.path.join(urban_folder, \"combo_extents.shp\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b135cd57", + "metadata": {}, + "outputs": [], + "source": [ + "# Combine all the urban extents into a single file\n", + "if not os.path.exists(combo_extents):\n", + " urban_extents = [x for x in os.listdir(os.path.join(urban_folder, \"Final_AoI\")) if x.endswith(\".shp\")]\n", + " all_extents = []\n", + " for urban_extent in urban_extents:\n", + " urban_extent_gdf = gpd.read_file(os.path.join(urban_folder, \"Final_AoI\", urban_extent))\n", + " type = urban_extent.split(\"_\")[1].replace(\".shp\", \"\").lower()\n", + " if type == \"colombo\":\n", + " type = \"city\"\n", + " if \"buffer\" in urban_extent:\n", + " type = \"20km\"\n", + " urban_extent_gdf[\"Type\"] = type\n", + " all_extents.append(urban_extent_gdf)\n", + "\n", + " all_extents_gdf = gpd.GeoDataFrame(pd.concat(all_extents, ignore_index=True))\n", + " all_extents_gdf.to_file(combo_extents)\n", + "else:\n", + " all_extents_gdf = gpd.read_file(combo_extents)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "97add9be", + "metadata": {}, + "outputs": [], + "source": [ + "# extract urban extents from population grids\n", + "for pop_layer, name in [[worldpop_file, \"WorldPop\"]]:\n", + " urb_out_file = os.path.join(urban_folder, \"DOU_urb_extent_%s.shp\" % name)\n", + " hd_out_file = os.path.join(urban_folder, \"DOU_hd_extent_%s.shp\" % name)\n", + " if not os.path.exists(hd_out_file):\n", + " curPop = rasterio.open(pop_layer)\n", + " urban_calculator = urban.urbanGriddedPop(curPop)\n", + " urban_extents = urban_calculator.calculateUrban(\n", + " densVal=3, totalPopThresh=5000, smooth=False, queen=False, verbose=True\n", + " )\n", + " urban_extents['Type'] = name\n", + " urban_extents.to_file(urb_out_file)\n", + "\n", + " hd_urban_extents = urban_calculator.calculateUrban(\n", + " densVal=15,\n", + " totalPopThresh=50000,\n", + " smooth=True,\n", + " queen=True, # high density extents use queen's case contiguity, and\n", + " verbose=True,\n", + " ) # High density extents have hole smoothing applied.\n", + " hd_urban_extents['Type'] = name\n", + " hd_urban_extents.to_file(hd_out_file)\n", + "\n", + "\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34d495aa", + "metadata": {}, + "outputs": [], + "source": [ + "dir(GOSTurban)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "50115a0d", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/Implementations/URB_LKA_CCDR_Support/Untitled.ipynb b/notebooks/Implementations/URB_LKA_CCDR_Support/Untitled.ipynb deleted file mode 100644 index 0901847..0000000 --- a/notebooks/Implementations/URB_LKA_CCDR_Support/Untitled.ipynb +++ /dev/null @@ -1,209 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "9fd5e3dd", - "metadata": {}, - "source": [ - "# Urbanization comparison\n", - "\n", - "Compare the multiple urban extents in the project\n", - "\n", - "1. Official administrative boundaries provided by the project team. \n", - " a. Need to get metadata on these boundaries from Swati \n", - "2. Official administrative boundaries with buffers \n", - " a. Why these specific buffer distances? \n", - "3. DoU boundaries \n", - " a. Which population source \n" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "id": "9c6da02f", - "metadata": {}, - "outputs": [], - "source": [ - "import sys, os\n", - "import rasterio\n", - "\n", - "import pandas as pd\n", - "import geopandas as gpd\n", - "\n", - "import GOSTurban.UrbanRaster as urban\n" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "df1bf2ed", - "metadata": {}, - "outputs": [], - "source": [ - "base_folder = \"C:/WBG/Work/LKA_URB_CCDR/Data\"\n", - "urban_folder = os.path.join(base_folder, \"Urban_extents\")\n", - "population_folder = os.path.join(base_folder, \"Population\")\n", - "\n", - "worldpop_file = os.path.join(population_folder, \"lka_ppp_2020_UNadj.tif\")\n", - "combo_extents = os.path.join(urban_folder, \"combo_extents.shp\")" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "b135cd57", - "metadata": {}, - "outputs": [], - "source": [ - "# Combine all the urban extents into a single file\n", - "if not os.path.exists(combo_extents):\n", - " urban_extents = [x for x in os.listdir(os.path.join(urban_folder, \"Final_AoI\")) if x.endswith(\".shp\")]\n", - " all_extents = []\n", - " for urban_extent in urban_extents:\n", - " urban_extent_gdf = gpd.read_file(os.path.join(urban_folder, \"Final_AoI\", urban_extent))\n", - " type = urban_extent.split(\"_\")[1].replace(\".shp\", \"\").lower()\n", - " if type == \"colombo\":\n", - " type = \"city\"\n", - " if \"buffer\" in urban_extent:\n", - " type = \"20km\"\n", - " urban_extent_gdf[\"Type\"] = type\n", - " all_extents.append(urban_extent_gdf)\n", - "\n", - " all_extents_gdf = gpd.GeoDataFrame(pd.concat(all_extents, ignore_index=True))\n", - " all_extents_gdf.to_file(combo_extents)\n", - "else:\n", - " all_extents_gdf = gpd.read_file(combo_extents)" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "97add9be", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "13:19:42\t: Read in urban data\n", - "13:19:42\t: Creating Shape 0\n", - "13:19:45\t: Read in urban data\n", - "13:19:45\t: Creating Shape 0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "c:\\wbg\\Anaconda3\\envs\\urban_test\\Lib\\site-packages\\GOSTurban\\UrbanRaster.py:404: UserWarning: Geometry is in a geographic CRS. Results from 'buffer' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", - "\n", - " xxGeom[\"geometry\"] = xxGeom.buffer((popRaster.res[0] / 2))\n" - ] - }, - { - "ename": "ValueError", - "evalue": "graph should have two dimensions", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[29], line 14\u001b[0m\n\u001b[0;32m 11\u001b[0m urban_extents[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mType\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m name\n\u001b[0;32m 12\u001b[0m urban_extents\u001b[38;5;241m.\u001b[39mto_file(urb_out_file)\n\u001b[1;32m---> 14\u001b[0m hd_urban_extents \u001b[38;5;241m=\u001b[39m urban_calculator\u001b[38;5;241m.\u001b[39mcalculateUrban(\n\u001b[0;32m 15\u001b[0m densVal\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1500\u001b[39m,\n\u001b[0;32m 16\u001b[0m totalPopThresh\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m50000\u001b[39m,\n\u001b[0;32m 17\u001b[0m smooth\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[0;32m 18\u001b[0m queen\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[38;5;66;03m# high density extents use queen's case contiguity, and\u001b[39;00m\n\u001b[0;32m 19\u001b[0m verbose\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[0;32m 20\u001b[0m ) \u001b[38;5;66;03m# High density extents have hole smoothing applied.\u001b[39;00m\n\u001b[0;32m 21\u001b[0m hd_urban_extents[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mType\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m name\n\u001b[0;32m 22\u001b[0m hd_urban_extents\u001b[38;5;241m.\u001b[39mto_file(hd_out_file)\n", - "File \u001b[1;32mc:\\wbg\\Anaconda3\\envs\\urban_test\\Lib\\site-packages\\GOSTurban\\UrbanRaster.py:407\u001b[0m, in \u001b[0;36murbanGriddedPop.calculateUrban\u001b[1;34m(self, densVal, totalPopThresh, smooth, verbose, queen, raster, raster_pop, print_message)\u001b[0m\n\u001b[0;32m 405\u001b[0m s \u001b[38;5;241m=\u001b[39m xxGeom[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgeometry\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[0;32m 406\u001b[0m overlap_matrix \u001b[38;5;241m=\u001b[39m s\u001b[38;5;241m.\u001b[39mapply(\u001b[38;5;28;01mlambda\u001b[39;00m x: s\u001b[38;5;241m.\u001b[39mintersects(x))\u001b[38;5;241m.\u001b[39mvalues\u001b[38;5;241m.\u001b[39mastype(\u001b[38;5;28mint\u001b[39m)\n\u001b[1;32m--> 407\u001b[0m n, ids \u001b[38;5;241m=\u001b[39m connected_components(overlap_matrix)\n\u001b[0;32m 408\u001b[0m xxGeom[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgroup\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m ids\n\u001b[0;32m 409\u001b[0m xxGeom \u001b[38;5;241m=\u001b[39m xxGeom\u001b[38;5;241m.\u001b[39mdissolve(by\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgroup\u001b[39m\u001b[38;5;124m\"\u001b[39m, aggfunc\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msum\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[1;32m_traversal.pyx:97\u001b[0m, in \u001b[0;36mscipy.sparse.csgraph._traversal.connected_components\u001b[1;34m()\u001b[0m\n", - "File \u001b[1;32mc:\\wbg\\Anaconda3\\envs\\urban_test\\Lib\\site-packages\\scipy\\sparse\\csgraph\\_validation.py:46\u001b[0m, in \u001b[0;36mvalidate_graph\u001b[1;34m(csgraph, directed, dtype, csr_output, dense_output, copy_if_dense, copy_if_sparse, null_value_in, null_value_out, infinity_null, nan_null)\u001b[0m\n\u001b[0;32m 44\u001b[0m csgraph[mask] \u001b[38;5;241m=\u001b[39m null_value_out\n\u001b[0;32m 45\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m---> 46\u001b[0m csgraph \u001b[38;5;241m=\u001b[39m csgraph_from_dense(csgraph, null_value\u001b[38;5;241m=\u001b[39mnull_value_in,\n\u001b[0;32m 47\u001b[0m infinity_null\u001b[38;5;241m=\u001b[39minfinity_null,\n\u001b[0;32m 48\u001b[0m nan_null\u001b[38;5;241m=\u001b[39mnan_null)\n\u001b[0;32m 50\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m csgraph\u001b[38;5;241m.\u001b[39mndim \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m2\u001b[39m:\n\u001b[0;32m 51\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcompressed-sparse graph must be 2-D\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[1;32m_tools.pyx:215\u001b[0m, in \u001b[0;36mscipy.sparse.csgraph._tools.csgraph_from_dense\u001b[1;34m()\u001b[0m\n", - "File \u001b[1;32m_tools.pyx:140\u001b[0m, in \u001b[0;36mscipy.sparse.csgraph._tools.csgraph_masked_from_dense\u001b[1;34m()\u001b[0m\n", - "\u001b[1;31mValueError\u001b[0m: graph should have two dimensions" - ] - } - ], - "source": [ - "# extract urban extents from population grids\n", - "for pop_layer, name in [[worldpop_file, \"WorldPop\"]]:\n", - " urb_out_file = os.path.join(urban_folder, \"DOU_urb_extent_%s.shp\" % name)\n", - " hd_out_file = os.path.join(urban_folder, \"DOU_hd_extent_%s.shp\" % name)\n", - " if not os.path.exists(hd_out_file):\n", - " curPop = rasterio.open(pop_layer)\n", - " urban_calculator = urban.urbanGriddedPop(curPop)\n", - " urban_extents = urban_calculator.calculateUrban(\n", - " densVal=3, totalPopThresh=5000, smooth=False, queen=False, verbose=True\n", - " )\n", - " urban_extents['Type'] = name\n", - " urban_extents.to_file(urb_out_file)\n", - "\n", - " hd_urban_extents = urban_calculator.calculateUrban(\n", - " densVal=15,\n", - " totalPopThresh=50000,\n", - " smooth=True,\n", - " queen=True, # high density extents use queen's case contiguity, and\n", - " verbose=True,\n", - " ) # High density extents have hole smoothing applied.\n", - " hd_urban_extents['Type'] = name\n", - " hd_urban_extents.to_file(hd_out_file)\n", - "\n", - "\n", - " \n" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "34d495aa", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['__builtins__',\n", - " '__cached__',\n", - " '__doc__',\n", - " '__file__',\n", - " '__loader__',\n", - " '__name__',\n", - " '__package__',\n", - " '__path__',\n", - " '__spec__',\n", - " '__version__',\n", - " '_version']" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dir(GOSTurban)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "50115a0d", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "urban_test", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From 3577d7e1e2a347f27e4ab99116fc73fd9feb0bbb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 25 May 2024 11:13:51 +0000 Subject: [PATCH 3/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/_toc.yml | 1 - docs/conf.py | 2 +- .../MENA_Benchmarking/NTL_zonal_stats.ipynb | 33 ++++----- .../URB_LKA_CCDR_Support/LKA_Urban_Comp.ipynb | 23 +++--- .../URB_LKA_CCDR_Support/README.md | 2 +- .../Replications/URB_NovelUrbanization.py | 30 ++++---- notebooks/Tutorials/LEI_Example.ipynb | 72 ++++++++++++------- .../Tutorials/UrbanAreas_tutorials.ipynb | 8 +-- requirements.txt | 2 - src/GOSTurban/LEI.py | 2 +- 10 files changed, 102 insertions(+), 73 deletions(-) diff --git a/docs/_toc.yml b/docs/_toc.yml index 8000614..209f5d8 100755 --- a/docs/_toc.yml +++ b/docs/_toc.yml @@ -15,4 +15,3 @@ parts: chapters: - file: notebooks/Tutorials/UrbanAreas_tutorials.ipynb - file: notebooks/Tutorials/LEI_Example.ipynb - diff --git a/docs/conf.py b/docs/conf.py index 08d061d..5867762 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,7 +24,7 @@ "sphinx_jupyterbook_latex", "sphinx.ext.napoleon", "sphinxcontrib.apidoc", - #"nbsphinx" + # "nbsphinx" ] external_toc_exclude_missing = True external_toc_path = "_toc.yml" diff --git a/notebooks/Implementations/MENA_Benchmarking/NTL_zonal_stats.ipynb b/notebooks/Implementations/MENA_Benchmarking/NTL_zonal_stats.ipynb index 0426341..0b0a63c 100644 --- a/notebooks/Implementations/MENA_Benchmarking/NTL_zonal_stats.ipynb +++ b/notebooks/Implementations/MENA_Benchmarking/NTL_zonal_stats.ipynb @@ -22,19 +22,13 @@ "metadata": {}, "outputs": [], "source": [ - "import sys, os\n", - "import rasterio\n", + "import sys\n", + "import os\n", "\n", - "import pandas as pd\n", "import geopandas as gpd\n", - "import numpy as np\n", "\n", - "from shapely.geometry import Point\n", "\n", - "sys.path.append(\"C:/WBG/Work/Code/GOSTrocks/src\")\n", - "\n", - "import GOSTrocks.rasterMisc as rMisc\n", - "from GOSTrocks.misc import tPrint" + "sys.path.append(\"C:/WBG/Work/Code/GOSTrocks/src\")" ] }, { @@ -44,8 +38,15 @@ "outputs": [], "source": [ "data_folder = \"C:/WBG/Work/data\"\n", - "ucdb_file = os.path.join(data_folder, \"URBAN\",\"GHS_STAT_UCDB2015MT_GLOBE_R2019A\", \"GHS_STAT_UCDB2015MT_GLOBE_R2019A_V1_2.gpkg\")\n", - "fua_file = os.path.join(data_folder, \"URBAN\", \"GHS_FUA_UCDB2015_GLOBE_R2019A_54009_1K_V1_0.gpkg\")\n", + "ucdb_file = os.path.join(\n", + " data_folder,\n", + " \"URBAN\",\n", + " \"GHS_STAT_UCDB2015MT_GLOBE_R2019A\",\n", + " \"GHS_STAT_UCDB2015MT_GLOBE_R2019A_V1_2.gpkg\",\n", + ")\n", + "fua_file = os.path.join(\n", + " data_folder, \"URBAN\", \"GHS_FUA_UCDB2015_GLOBE_R2019A_54009_1K_V1_0.gpkg\"\n", + ")\n", "\n", "out_folder = \"C:/WBG/Work/MENA_Urban/\"\n", "urban_res_folder = os.path.join(out_folder, \"urban_data\")\n", @@ -236,13 +237,13 @@ "fua_peripheries = inF.copy()\n", "for idx, row in inF.iterrows():\n", " # grab the related UCDBs\n", - " ucdb_ids = row['UC_IDs'].split(\";\")\n", + " ucdb_ids = row[\"UC_IDs\"].split(\";\")\n", " ucdb_ids = [int(x) for x in ucdb_ids]\n", - " sel_cores = inD.loc[inD['ID_HDC_G0'].isin(ucdb_ids)]\n", - " periphery_geom = row['geometry'].difference(sel_cores.unary_union)\n", - " fua_peripheries.loc[idx, 'geometry'] = periphery_geom\n", + " sel_cores = inD.loc[inD[\"ID_HDC_G0\"].isin(ucdb_ids)]\n", + " periphery_geom = row[\"geometry\"].difference(sel_cores.unary_union)\n", + " fua_peripheries.loc[idx, \"geometry\"] = periphery_geom\n", "\n", - "fua_peripheries.to_file(os.path.join(out_folder, 'FUA_peripheries.gpkg'), driver='GPKG')" + "fua_peripheries.to_file(os.path.join(out_folder, \"FUA_peripheries.gpkg\"), driver=\"GPKG\")" ] } ], diff --git a/notebooks/Implementations/URB_LKA_CCDR_Support/LKA_Urban_Comp.ipynb b/notebooks/Implementations/URB_LKA_CCDR_Support/LKA_Urban_Comp.ipynb index 32dfcaf..41fd046 100644 --- a/notebooks/Implementations/URB_LKA_CCDR_Support/LKA_Urban_Comp.ipynb +++ b/notebooks/Implementations/URB_LKA_CCDR_Support/LKA_Urban_Comp.ipynb @@ -24,13 +24,13 @@ "metadata": {}, "outputs": [], "source": [ - "import sys, os\n", + "import os\n", "import rasterio\n", "\n", "import pandas as pd\n", "import geopandas as gpd\n", "\n", - "import GOSTurban.UrbanRaster as urban\n" + "import GOSTurban.UrbanRaster as urban" ] }, { @@ -57,10 +57,16 @@ "source": [ "# Combine all the urban extents into a single file\n", "if not os.path.exists(combo_extents):\n", - " urban_extents = [x for x in os.listdir(os.path.join(urban_folder, \"Final_AoI\")) if x.endswith(\".shp\")]\n", + " urban_extents = [\n", + " x\n", + " for x in os.listdir(os.path.join(urban_folder, \"Final_AoI\"))\n", + " if x.endswith(\".shp\")\n", + " ]\n", " all_extents = []\n", " for urban_extent in urban_extents:\n", - " urban_extent_gdf = gpd.read_file(os.path.join(urban_folder, \"Final_AoI\", urban_extent))\n", + " urban_extent_gdf = gpd.read_file(\n", + " os.path.join(urban_folder, \"Final_AoI\", urban_extent)\n", + " )\n", " type = urban_extent.split(\"_\")[1].replace(\".shp\", \"\").lower()\n", " if type == \"colombo\":\n", " type = \"city\"\n", @@ -92,7 +98,7 @@ " urban_extents = urban_calculator.calculateUrban(\n", " densVal=3, totalPopThresh=5000, smooth=False, queen=False, verbose=True\n", " )\n", - " urban_extents['Type'] = name\n", + " urban_extents[\"Type\"] = name\n", " urban_extents.to_file(urb_out_file)\n", "\n", " hd_urban_extents = urban_calculator.calculateUrban(\n", @@ -102,11 +108,8 @@ " queen=True, # high density extents use queen's case contiguity, and\n", " verbose=True,\n", " ) # High density extents have hole smoothing applied.\n", - " hd_urban_extents['Type'] = name\n", - " hd_urban_extents.to_file(hd_out_file)\n", - "\n", - "\n", - " \n" + " hd_urban_extents[\"Type\"] = name\n", + " hd_urban_extents.to_file(hd_out_file)" ] }, { diff --git a/notebooks/Implementations/URB_LKA_CCDR_Support/README.md b/notebooks/Implementations/URB_LKA_CCDR_Support/README.md index a8b0bef..3d58bb8 100644 --- a/notebooks/Implementations/URB_LKA_CCDR_Support/README.md +++ b/notebooks/Implementations/URB_LKA_CCDR_Support/README.md @@ -8,4 +8,4 @@ The Sri Lanka CCDR team is looking for support in urban deep dives on 9 provinci | Urban Poverty | 3 | | Analyze high-resolution satellite imagery to identify informality in residential settlements | | Urban economics | 2 | | Summarize monthly nighttime lights | | Status of infrastructure | 2 | | TBD | -| GHG emissions | 1 | | Summarize Brian B's GHG emissions data | \ No newline at end of file +| GHG emissions | 1 | | Summarize Brian B's GHG emissions data | diff --git a/notebooks/Replications/URB_NovelUrbanization.py b/notebooks/Replications/URB_NovelUrbanization.py index a31150b..4897bf6 100644 --- a/notebooks/Replications/URB_NovelUrbanization.py +++ b/notebooks/Replications/URB_NovelUrbanization.py @@ -1,39 +1,43 @@ -import sys, os, shutil, requests +import sys +import os +import shutil +import requests import rasterio -import pandas as pd -import geopandas as gpd -import numpy as np import GOSTurban.UrbanRaster as urban + def download_pop_file(url, filename): # Open the url r = requests.get(url) # Set decode_content value to True, otherwise the downloaded image file's size will be zero. r.raw.decode_content = True # Open a local file with wb ( write binary ) permission. - with open(filename,'wb') as f: + with open(filename, "wb") as f: shutil.copyfileobj(r.raw, f) + def main(iso3, out_folder): # download the population data - wp_url = f'https://data.worldpop.org/GIS/Population/Global_2000_2020_1km/2020/{iso3.upper()}/{iso3.lower()}_ppp_2020_1km_Aggregated.tif' - print (wp_url) + wp_url = f"https://data.worldpop.org/GIS/Population/Global_2000_2020_1km/2020/{iso3.upper()}/{iso3.lower()}_ppp_2020_1km_Aggregated.tif" + print(wp_url) if not os.path.exists(out_folder): os.makedirs(out_folder) - out_file = os.path.join(out_folder, f'{iso3}_ppp_2020_1km_Aggregated.tif') + out_file = os.path.join(out_folder, f"{iso3}_ppp_2020_1km_Aggregated.tif") out_urban = os.path.join(out_folder, "urban_extents.geojson") out_hd_urban = os.path.join(out_folder, "hd_urban_extents.geojson") - + try: if not os.path.exists(out_file): download_pop_file(wp_url, out_file) except: print(f"Could not download national population data for {iso3} from {wp_url}") - print("If you can manually download to the defined out_folder, the script will run") + print( + "If you can manually download to the defined out_folder, the script will run" + ) if os.path.exists(out_file): - inR = rasterio.open(out_file) + inR = rasterio.open(out_file) urban_calculator = urban.urbanGriddedPop(inR) urban_extents = urban_calculator.calculateUrban( densVal=300, totalPopThresh=5000, smooth=False, queen=False @@ -43,14 +47,14 @@ def main(iso3, out_folder): densVal=1500, totalPopThresh=50000, smooth=True, - queen=True, # high density extents use queen's case contiguity, and are smoothed + queen=True, # high density extents use queen's case contiguity, and are smoothed ) urban_extents.to_file(out_urban, driver="GeoJSON") hd_urban_extents.to_file(out_hd_urban, driver="GeoJSON") + if __name__ == "__main__": iso3 = sys.argv[1] out_folder = sys.argv[2] main(iso3, out_folder) - diff --git a/notebooks/Tutorials/LEI_Example.ipynb b/notebooks/Tutorials/LEI_Example.ipynb index 2a78b22..b9f0bea 100644 --- a/notebooks/Tutorials/LEI_Example.ipynb +++ b/notebooks/Tutorials/LEI_Example.ipynb @@ -24,20 +24,16 @@ "outputs": [], "source": [ "import os\n", - "import sys\n", - "import importlib\n", "import rasterio\n", "import rasterio.features\n", "\n", "import geopandas as gpd\n", "import pandas as pd\n", - "import numpy as np\n", "\n", "import GOSTrocks.rasterMisc as rMisc\n", "import GOSTrocks.ghslMisc as ghslMisc\n", "import GOSTurban.LEI as lei\n", "import GOSTrocks.mapMisc as mapMisc\n", - "import GOSTrocks.rasterMisc as rMisc\n", "\n", "%load_ext autoreload\n", "%autoreload 2" @@ -71,18 +67,22 @@ " temp_folder = \"C:/Temp\"\n", " # clip from global GHSL file\n", " ghsl_folder = \"J:/Data/GLOBAL/GHSL/built\"\n", - " ghsl_files = [os.path.join(ghsl_folder, x) for x in os.listdir(ghsl_folder) if x.endswith(\".tif\")] \n", + " ghsl_files = [\n", + " os.path.join(ghsl_folder, x)\n", + " for x in os.listdir(ghsl_folder)\n", + " if x.endswith(\".tif\")\n", + " ]\n", " inA = gpd.read_file(aoi_file)\n", - " \n", + "\n", " temp_ghsl_files = []\n", " for ghsl_file in ghsl_files:\n", " temp_file = os.path.join(temp_folder, os.path.basename(ghsl_file))\n", " temp_ghsl_files.append(temp_file)\n", " if not os.path.exists(temp_file):\n", " rMisc.clipRaster(rasterio.open(ghsl_file), inA, temp_file)\n", - " \n", + "\n", " ghsl_res, ghsl_profile = ghslMisc.combine_ghsl_annual(temp_ghsl_files)\n", - " with rasterio.open(input_ghsl, 'w', **ghsl_profile) as outR:\n", + " with rasterio.open(input_ghsl, \"w\", **ghsl_profile) as outR:\n", " outR.write_band(1, ghsl_res)" ] }, @@ -116,7 +116,7 @@ "source": [ "ghsl_r = rasterio.open(input_ghsl)\n", "ghsl_d = ghsl_r.read()\n", - "ghsl_d[ghsl_d == ghsl_r.meta['nodata']] = 0\n", + "ghsl_d[ghsl_d == ghsl_r.meta[\"nodata\"]] = 0\n", "\n", "thresh = list(range(1975, 2031, 5))\n", "with rMisc.create_rasterio_inmemory(ghsl_r.profile, ghsl_d) as temp_ghsl:\n", @@ -218,8 +218,14 @@ ], "source": [ "# This calculates the change from 1990 and 2000\n", - "lei_raw = lei.calculate_LEI(input_ghsl, old_list=list(range(1975,1991,5)), new_list=list(range(1995,2001,5)))\n", - "lei_90_00 = gpd.GeoDataFrame(pd.DataFrame(lei_raw, columns=[\"geometry\", \"old\", \"total\"]), geometry='geometry', crs=ghsl_r.crs)\n", + "lei_raw = lei.calculate_LEI(\n", + " input_ghsl, old_list=list(range(1975, 1991, 5)), new_list=list(range(1995, 2001, 5))\n", + ")\n", + "lei_90_00 = gpd.GeoDataFrame(\n", + " pd.DataFrame(lei_raw, columns=[\"geometry\", \"old\", \"total\"]),\n", + " geometry=\"geometry\",\n", + " crs=ghsl_r.crs,\n", + ")\n", "lei_90_00[\"LEI\"] = lei_90_00[\"old\"] / lei_90_00[\"total\"]\n", "\n", "lei_90_00.head()" @@ -260,10 +266,12 @@ } ], "source": [ - "#Map LEI results\n", - "leap_val=0.30\n", - "exp_val=0.70\n", - "lei_90_00['area'] = lei_90_00['geometry'].apply(lambda x: x.area)\n", + "# Map LEI results\n", + "leap_val = 0.30\n", + "exp_val = 0.70\n", + "lei_90_00[\"area\"] = lei_90_00[\"geometry\"].apply(lambda x: x.area)\n", + "\n", + "\n", "def calculate_LEI(val, leap_val, exp_val):\n", " if val <= leap_val:\n", " return 3\n", @@ -271,8 +279,14 @@ " return 2\n", " else:\n", " return 1\n", - "lei_90_00[\"class\"] = lei_90_00[\"LEI\"].apply(lambda x: calculate_LEI(x, leap_val, exp_val))\n", - "mapMisc.static_map_vector(lei_90_00, \"class\", edgecolor='match', colormap=\"Dark2\")#, basemap=ctx.providers.CartoDB.Voyager)" + "\n", + "\n", + "lei_90_00[\"class\"] = lei_90_00[\"LEI\"].apply(\n", + " lambda x: calculate_LEI(x, leap_val, exp_val)\n", + ")\n", + "mapMisc.static_map_vector(\n", + " lei_90_00, \"class\", edgecolor=\"match\", colormap=\"Dark2\"\n", + ") # , basemap=ctx.providers.CartoDB.Voyager)" ] }, { @@ -395,7 +409,9 @@ ], "source": [ "# This calculates the change from 2000 and 2014\n", - "lei_raw = lei.calculate_LEI(input_ghsl, old_list=list(range(1975,2011,5)), new_list=list(range(2015,2030,5)))\n", + "lei_raw = lei.calculate_LEI(\n", + " input_ghsl, old_list=list(range(1975, 2011, 5)), new_list=list(range(2015, 2030, 5))\n", + ")\n", "lei_00_14 = pd.DataFrame(lei_raw, columns=[\"geometry\", \"old\", \"total\"])\n", "lei_00_14[\"LEI\"] = lei_00_14[\"old\"] / lei_00_14[\"total\"]\n", "lei_00_14.head()" @@ -436,10 +452,12 @@ } ], "source": [ - "#Map LEI results\n", - "leap_val=0.30\n", - "exp_val=0.70\n", - "lei_90_00['area'] = lei_90_00['geometry'].apply(lambda x: x.area)\n", + "# Map LEI results\n", + "leap_val = 0.30\n", + "exp_val = 0.70\n", + "lei_90_00[\"area\"] = lei_90_00[\"geometry\"].apply(lambda x: x.area)\n", + "\n", + "\n", "def calculate_LEI(val, leap_val, exp_val):\n", " if val <= leap_val:\n", " return 3\n", @@ -447,8 +465,14 @@ " return 2\n", " else:\n", " return 1\n", - "lei_90_00[\"class\"] = lei_90_00[\"LEI\"].apply(lambda x: calculate_LEI(x, leap_val, exp_val))\n", - "mapMisc.static_map_vector(lei_90_00, \"class\", edgecolor='match', colormap=\"Dark2\")#, basemap=ctx.providers.CartoDB.Voyager)" + "\n", + "\n", + "lei_90_00[\"class\"] = lei_90_00[\"LEI\"].apply(\n", + " lambda x: calculate_LEI(x, leap_val, exp_val)\n", + ")\n", + "mapMisc.static_map_vector(\n", + " lei_90_00, \"class\", edgecolor=\"match\", colormap=\"Dark2\"\n", + ") # , basemap=ctx.providers.CartoDB.Voyager)" ] }, { diff --git a/notebooks/Tutorials/UrbanAreas_tutorials.ipynb b/notebooks/Tutorials/UrbanAreas_tutorials.ipynb index 1d20c25..8ba47e3 100644 --- a/notebooks/Tutorials/UrbanAreas_tutorials.ipynb +++ b/notebooks/Tutorials/UrbanAreas_tutorials.ipynb @@ -100,7 +100,7 @@ } ], "source": [ - "mapMisc.static_map_raster(inR, thresh=[1,5,50,100,300,1000,3000])" + "mapMisc.static_map_raster(inR, thresh=[1, 5, 50, 100, 300, 1000, 3000])" ] }, { @@ -223,7 +223,7 @@ "urban_extents = urban_calculator.calculateUrban(\n", " densVal=300, totalPopThresh=5000, smooth=False, queen=False, verbose=True\n", ")\n", - "urban_extents['Type'] = 1\n", + "urban_extents[\"Type\"] = 1\n", "urban_extents.head()" ] }, @@ -406,7 +406,7 @@ " queen=True, # high density extents use queen's case contiguity, and\n", " verbose=True,\n", ") # High density extents have hole smoothing applied.\n", - "hd_urban_extents['Type'] = 2\n", + "hd_urban_extents[\"Type\"] = 2\n", "hd_urban_extents.head()" ] }, @@ -491,7 +491,7 @@ ], "source": [ "combo_extents = pd.concat([urban_extents, hd_urban_extents])\n", - "mapMisc.static_map_vector(combo_extents, \"Type\", colormap='magma')" + "mapMisc.static_map_vector(combo_extents, \"Type\", colormap=\"magma\")" ] }, { diff --git a/requirements.txt b/requirements.txt index abbc506..f1a28c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,5 +10,3 @@ elevation geojson hatch git - - diff --git a/src/GOSTurban/LEI.py b/src/GOSTurban/LEI.py index 5ee1cf0..5293a34 100755 --- a/src/GOSTurban/LEI.py +++ b/src/GOSTurban/LEI.py @@ -195,4 +195,4 @@ def calculate_LEI(val, leap_val, exp_val): res["class"] = res["LEI"].apply(lambda x: calculate_LEI(x, leap_val, exp_val)) xx = res.groupby("class") - return xx['area'].sum() + return xx["area"].sum()