From 0b330d9a5a13a674418ded8a5ecb1adeaa935063 Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Wed, 4 Oct 2023 18:42:12 -0400 Subject: [PATCH 1/6] Use pixetl with rasterize improvements --- batch/pixetl.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batch/pixetl.dockerfile b/batch/pixetl.dockerfile index be38d976d..bded9aaa2 100644 --- a/batch/pixetl.dockerfile +++ b/batch/pixetl.dockerfile @@ -1,4 +1,4 @@ -FROM globalforestwatch/pixetl:v1.7.4 +FROM globalforestwatch/pixetl:robustify_vector_tiles # Copy scripts From 375e6cdaea785e519be498341709efc64cff28ef Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Wed, 4 Oct 2023 19:23:37 -0400 Subject: [PATCH 2/6] Fix some bugs --- batch/pixetl.dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/batch/pixetl.dockerfile b/batch/pixetl.dockerfile index bded9aaa2..3c12c8dc3 100644 --- a/batch/pixetl.dockerfile +++ b/batch/pixetl.dockerfile @@ -1,5 +1,4 @@ -FROM globalforestwatch/pixetl:robustify_vector_tiles - +FROM globalforestwatch/pixetl:robustify_vector_tiles_1 # Copy scripts COPY ./batch/scripts/ /opt/scripts/ @@ -19,4 +18,4 @@ WORKDIR /tmp ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 -ENTRYPOINT ["/opt/scripts/report_status.sh"] +ENTRYPOINT ["/opt/scripts/report_status.sh"] \ No newline at end of file From 3e3a0938cf7a142f321dfdc73526e26451a0d964 Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Wed, 11 Oct 2023 15:20:26 -0400 Subject: [PATCH 3/6] Tweak vector rasterization test --- batch/pixetl.dockerfile | 2 +- tests/conftest.py | 1 + tests/routes/datasets/test_assets.py | 21 ++++++++++----------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/batch/pixetl.dockerfile b/batch/pixetl.dockerfile index 3c12c8dc3..b91b45b5e 100644 --- a/batch/pixetl.dockerfile +++ b/batch/pixetl.dockerfile @@ -1,4 +1,4 @@ -FROM globalforestwatch/pixetl:robustify_vector_tiles_1 +FROM globalforestwatch/pixetl:robustify_vector_tiles # Copy scripts COPY ./batch/scripts/ /opt/scripts/ diff --git a/tests/conftest.py b/tests/conftest.py index cfb7326de..1fbda0fb4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -229,6 +229,7 @@ def client(): except Exception as ex: print(f"Exception deleting asset {asset['asset_id']}: {ex}") try: + # FIXME: Mock-out cache invalidation function _ = client.delete(f"/dataset/{ds_id}/{version}") except Exception as ex: print(f"Exception deleting version {version}: {ex}") diff --git a/tests/routes/datasets/test_assets.py b/tests/routes/datasets/test_assets.py index b8d6e1744..eb45f43c8 100644 --- a/tests/routes/datasets/test_assets.py +++ b/tests/routes/datasets/test_assets.py @@ -5,6 +5,7 @@ import httpx import pytest from botocore.exceptions import ClientError +from httpx import AsyncClient from app.application import ContextEngine from app.crud import tasks @@ -255,21 +256,22 @@ async def test_auxiliary_raster_asset(async_client, httpd, logs): @pytest.mark.asyncio -async def test_auxiliary_vector_asset(async_client, batch_client, httpd): +async def test_rasterize_vector_asset(async_client: AsyncClient, batch_client, httpd): """""" _, logs = batch_client # Add a dataset, version, and default asset dataset = "test_vector" version = "v1.1.1" + grid = "10/40000" pixetl_output_files = [ - f"{dataset}/{version}/raster/epsg-4326/90/27008/gfw_fid/gdal-geotiff/extent.geojson", - f"{dataset}/{version}/raster/epsg-4326/90/27008/gfw_fid/geotiff/extent.geojson", - f"{dataset}/{version}/raster/epsg-4326/90/27008/gfw_fid/gdal-geotiff/tiles.geojson", - f"{dataset}/{version}/raster/epsg-4326/90/27008/gfw_fid/geotiff/tiles.geojson", - f"{dataset}/{version}/raster/epsg-4326/90/27008/gfw_fid/gdal-geotiff/90N_000E.tif", - f"{dataset}/{version}/raster/epsg-4326/90/27008/gfw_fid/geotiff/90N_000E.tif", + f"{dataset}/{version}/raster/epsg-4326/{grid}/gfw_fid/gdal-geotiff/extent.geojson", + f"{dataset}/{version}/raster/epsg-4326/{grid}/gfw_fid/geotiff/extent.geojson", + f"{dataset}/{version}/raster/epsg-4326/{grid}/gfw_fid/gdal-geotiff/tiles.geojson", + f"{dataset}/{version}/raster/epsg-4326/{grid}/gfw_fid/geotiff/tiles.geojson", + f"{dataset}/{version}/raster/epsg-4326/{grid}/gfw_fid/gdal-geotiff/60N_010E.tif", + f"{dataset}/{version}/raster/epsg-4326/{grid}/gfw_fid/geotiff/60N_010E.tif", ] for key in pixetl_output_files: @@ -294,15 +296,12 @@ async def test_auxiliary_vector_asset(async_client, batch_client, httpd): # vector asset asset_payload = { "asset_type": "Raster tile set", - "asset_uri": "http://www.osnews.com", "is_managed": True, "creation_options": { "data_type": FAKE_INT_DATA_PARAMS["dtype"], "pixel_meaning": "gfw_fid", - "grid": "90/27008", + "grid": grid, "resampling": "nearest", - "overwrite": True, - "subset": "90N_000E", }, } From d0f0e6541f76348b5ca4ae25c686dcf261fe33c7 Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Mon, 16 Oct 2023 14:02:00 -0400 Subject: [PATCH 4/6] Bump pixetl WIP version --- batch/pixetl.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batch/pixetl.dockerfile b/batch/pixetl.dockerfile index b91b45b5e..3c12c8dc3 100644 --- a/batch/pixetl.dockerfile +++ b/batch/pixetl.dockerfile @@ -1,4 +1,4 @@ -FROM globalforestwatch/pixetl:robustify_vector_tiles +FROM globalforestwatch/pixetl:robustify_vector_tiles_1 # Copy scripts COPY ./batch/scripts/ /opt/scripts/ From efe570433ca68d9bdf78c4444d6949104d8a4522 Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Mon, 23 Oct 2023 17:49:23 -0400 Subject: [PATCH 5/6] Un-restrict parallelism to see what breaks --- batch/pixetl.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batch/pixetl.dockerfile b/batch/pixetl.dockerfile index 3c12c8dc3..a4405d55b 100644 --- a/batch/pixetl.dockerfile +++ b/batch/pixetl.dockerfile @@ -1,4 +1,4 @@ -FROM globalforestwatch/pixetl:robustify_vector_tiles_1 +FROM globalforestwatch/pixetl:robustify_vector_tiles_unlim_procs # Copy scripts COPY ./batch/scripts/ /opt/scripts/ From 234c01e8b2c00ed0f9e4713f791fa6916468d611 Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Mon, 20 Nov 2023 15:51:41 -0500 Subject: [PATCH 6/6] Use final pixetl version --- batch/pixetl.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batch/pixetl.dockerfile b/batch/pixetl.dockerfile index a4405d55b..f89b2db60 100644 --- a/batch/pixetl.dockerfile +++ b/batch/pixetl.dockerfile @@ -1,4 +1,4 @@ -FROM globalforestwatch/pixetl:robustify_vector_tiles_unlim_procs +FROM globalforestwatch/pixetl:v1.7.5 # Copy scripts COPY ./batch/scripts/ /opt/scripts/