Merge pull request #444 from wri/develop

Develop -> master merge of package upgrades, improved API key error

.dockerignore

@@ -3,18 +3,29 @@
 *__pycache__*
 *.idea*
 
-#MyPy
+# MyPy
 .mypy_cache/*
 
 # Docker Files
 docker-compose.dev.yml
+docker-compose.prod.yml
 docker-compose.test.yml
 
+# GIT
+.git
+
 # Ignore Files
 .gitignore
 
-# Terraform
+# Mac stuff
+*.DS_Store
+
+# Test stuff
+tests/cobertura.xml
+tests_v2/cobertura.xml
+
+# Terraform stuff
 *terraform*
 
 # Virtual Environments
-.venv*
+.venv*

.gitignore

@@ -11,13 +11,17 @@
 
 # Environment Files
 /.env
+.python-version
+
+# Mac stuff
+*.DS_Store
 
 # Test stuff
 tests/cobertura.xml
 tests_v2/cobertura.xml
 
-# TF stuff
+# Terraform stuff
 terraform/*
 
-# Virtualenvs
-.venv*
+# Virtual Environments
+.venv*

app/routes/authentication/authentication.py

@@ -83,6 +83,16 @@ async def create_api_key(
             detail="Domain name did not match the request origin or referrer.",
         )
 
+    # Give a good error code/message if user is specifying an alias that exists for
+    # another one of his API keys.
+    prev_keys: List[ORMApiKey] = await api_keys.get_api_keys_from_user(user_id=user_id)
+    for key in prev_keys:
+        if key.alias == api_key_data.alias:
+            raise HTTPException(
+                status_code=409,
+                detail="Key with specified alias already exists; use a different alias"
+            )
+
     row: ORMApiKey = await api_keys.create_api_key(user_id=user_id, **input_data)
 
     is_internal = api_key_is_internal(

batch/pixetl.dockerfile

@@ -1,5 +1,4 @@
-FROM globalforestwatch/pixetl:v1.7.4
-
+FROM globalforestwatch/pixetl:v1.7.5
 
 # Copy scripts
 COPY ./batch/scripts/ /opt/scripts/
@@ -19,4 +18,4 @@ WORKDIR /tmp
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 
-ENTRYPOINT ["/opt/scripts/report_status.sh"]
+ENTRYPOINT ["/opt/scripts/report_status.sh"]

tests/conftest.py

@@ -229,6 +229,7 @@ def client():
                         except Exception as ex:
                             print(f"Exception deleting asset {asset['asset_id']}: {ex}")
                     try:
+                        # FIXME: Mock-out cache invalidation function
                         _ = client.delete(f"/dataset/{ds_id}/{version}")
                     except Exception as ex:
                         print(f"Exception deleting version {version}: {ex}")

tests/routes/datasets/test_assets.py

@@ -5,6 +5,7 @@
 import httpx
 import pytest
 from botocore.exceptions import ClientError
+from httpx import AsyncClient
 
 from app.application import ContextEngine
 from app.crud import tasks
@@ -255,21 +256,22 @@ async def test_auxiliary_raster_asset(async_client, httpd, logs):
 
 
 @pytest.mark.asyncio
-async def test_auxiliary_vector_asset(async_client, batch_client, httpd):
+async def test_rasterize_vector_asset(async_client: AsyncClient, batch_client, httpd):
     """"""
     _, logs = batch_client
 
     # Add a dataset, version, and default asset
     dataset = "test_vector"
     version = "v1.1.1"
+    grid = "10/40000"
 
     pixetl_output_files = [
-        f"{dataset}/{version}/raster/epsg-4326/90/27008/gfw_fid/gdal-geotiff/extent.geojson",
-        f"{dataset}/{version}/raster/epsg-4326/90/27008/gfw_fid/geotiff/extent.geojson",
-        f"{dataset}/{version}/raster/epsg-4326/90/27008/gfw_fid/gdal-geotiff/tiles.geojson",
-        f"{dataset}/{version}/raster/epsg-4326/90/27008/gfw_fid/geotiff/tiles.geojson",
-        f"{dataset}/{version}/raster/epsg-4326/90/27008/gfw_fid/gdal-geotiff/90N_000E.tif",
-        f"{dataset}/{version}/raster/epsg-4326/90/27008/gfw_fid/geotiff/90N_000E.tif",
+        f"{dataset}/{version}/raster/epsg-4326/{grid}/gfw_fid/gdal-geotiff/extent.geojson",
+        f"{dataset}/{version}/raster/epsg-4326/{grid}/gfw_fid/geotiff/extent.geojson",
+        f"{dataset}/{version}/raster/epsg-4326/{grid}/gfw_fid/gdal-geotiff/tiles.geojson",
+        f"{dataset}/{version}/raster/epsg-4326/{grid}/gfw_fid/geotiff/tiles.geojson",
+        f"{dataset}/{version}/raster/epsg-4326/{grid}/gfw_fid/gdal-geotiff/60N_010E.tif",
+        f"{dataset}/{version}/raster/epsg-4326/{grid}/gfw_fid/geotiff/60N_010E.tif",
     ]
 
     for key in pixetl_output_files:
@@ -294,15 +296,12 @@ async def test_auxiliary_vector_asset(async_client, batch_client, httpd):
     # vector asset
     asset_payload = {
         "asset_type": "Raster tile set",
-        "asset_uri": "http://www.osnews.com",
         "is_managed": True,
         "creation_options": {
             "data_type": FAKE_INT_DATA_PARAMS["dtype"],
             "pixel_meaning": "gfw_fid",
-            "grid": "90/27008",
+            "grid": grid,
             "resampling": "nearest",
-            "overwrite": True,
-            "subset": "90N_000E",
         },
     }
 

tests_v2/unit/app/crud/test_api_keys.py

@@ -5,6 +5,7 @@
 import asyncpg
 import boto3
 import pytest
+import pytest_asyncio
 from moto import mock_apigateway
 
 from app.application import ContextEngine
@@ -31,7 +32,7 @@
 )
 
 
-@pytest.fixture(autouse=True)
+@pytest_asyncio.fixture(autouse=True)
 @pytest.mark.asyncio
 async def delete_api_keys():
     yield

tests_v2/unit/app/routes/datasets/test_query.py

@@ -1,5 +1,6 @@
 from typing import Tuple
 from unittest.mock import Mock
+from urllib.parse import parse_qsl, urlparse
 
 import pytest
 from _pytest.monkeypatch import MonkeyPatch
@@ -235,11 +236,10 @@ async def test_redirect_get_query(
         params=params,
         follow_redirects=False,
     )
-
     assert response.status_code == 308
     assert (
-        response.headers["location"]
-        == f"/dataset/{dataset_name}/{version_name}/query/json?{response.request.url.query.decode('utf-8')}"
+        parse_qsl(urlparse(response.headers["location"]).query, strict_parsing=True)
+        == parse_qsl(urlparse(f"/dataset/{dataset_name}/{version_name}/query/json?{response.request.url.query.decode('utf-8')}").query, strict_parsing=True)
     )
 
 

tests_v2/unit/app/routes/datasets/test_version.py

@@ -2,12 +2,12 @@
 from _pytest.monkeypatch import MonkeyPatch
 from httpx import AsyncClient
 
-from tests_v2.fixtures.metadata.version import VERSION_METADATA
 from app.routes.datasets import versions
 from app.tasks import batch
 from tests_v2.fixtures.creation_options.versions import VECTOR_SOURCE_CREATION_OPTIONS
+from tests_v2.fixtures.metadata.version import VERSION_METADATA
 from tests_v2.unit.app.routes.utils import assert_jsend
-from tests_v2.utils import BatchJobMock, void_coroutine
+from tests_v2.utils import BatchJobMock, void_function
 
 
 @pytest.mark.asyncio
@@ -69,6 +69,9 @@ async def test_delete_version_metadata(
         f"/dataset/{dataset_name}/{version_name}/metadata",
     )
     assert resp.status_code == 404
+
+
+@pytest.mark.asyncio
 async def test_create_version_bare_minimum(
     async_client: AsyncClient, generic_dataset, monkeypatch: MonkeyPatch
 ):
@@ -78,7 +81,7 @@ async def test_create_version_bare_minimum(
     # patch all functions which reach out to external services
     batch_job_mock = BatchJobMock()
     monkeypatch.setattr(batch, "submit_batch_job", batch_job_mock.submit_batch_job)
-    monkeypatch.setattr(versions, "_verify_source_file_access", void_coroutine)
+    monkeypatch.setattr(versions, "_verify_source_file_access", void_function)
 
     payload = {"creation_options": VECTOR_SOURCE_CREATION_OPTIONS}
 
@@ -99,7 +102,7 @@ async def test_append_version_bare_minimum(
     # patch all functions which reach out to external services
     batch_job_mock = BatchJobMock()
     monkeypatch.setattr(batch, "submit_batch_job", batch_job_mock.submit_batch_job)
-    monkeypatch.setattr(versions, "_verify_source_file_access", void_coroutine)
+    monkeypatch.setattr(versions, "_verify_source_file_access", void_function)
 
     payload = {"source_uri": ["s3://some_bucket/test.shp.zip"]}