Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: titiler-pgstac v1 upgrade #398

Merged
merged 22 commits into from
Jul 25, 2024
Merged

feat: titiler-pgstac v1 upgrade #398

Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
213fcec
initial changes for titiler pgstac v1 upgrade
smohiudd Jul 18, 2024
6361c8d
format
smohiudd Jul 18, 2024
12ec468
add add_search_list_route
smohiudd Jul 18, 2024
47ffa3d
update titiler.core
smohiudd Jul 19, 2024
9342294
fix jinja2 template
vincentsarago Jul 19, 2024
cdbaa28
fix list
vincentsarago Jul 19, 2024
fcf3fed
fix dependency
vincentsarago Jul 19, 2024
a5ecbcc
fix tests
vincentsarago Jul 19, 2024
cdd8986
change route prefix
smohiudd Jul 22, 2024
ddb3a5f
re-add colormap dependency import
smohiudd Jul 22, 2024
d455440
format
smohiudd Jul 22, 2024
d4578f1
update tests
smohiudd Jul 22, 2024
139182e
format
smohiudd Jul 22, 2024
2db7ac6
add collection endpoint
smohiudd Jul 24, 2024
89f2d0f
change items endpoint
smohiudd Jul 24, 2024
6fecd6c
update alt endpoint
smohiudd Jul 24, 2024
5ba94f3
add endpoint tags
smohiudd Jul 24, 2024
dd96a7f
fix tests
smohiudd Jul 24, 2024
7a6848e
format
smohiudd Jul 24, 2024
ab06ebb
update tags
smohiudd Jul 24, 2024
4652db8
format, tag change
smohiudd Jul 24, 2024
4e10923
not include alt schema
smohiudd Jul 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 25 additions & 21 deletions .github/workflows/tests/test_raster.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,29 +19,31 @@ def test_raster_api():
def test_mosaic_api():
"""test mosaic."""
query = {"collections": ["noaa-emergency-response"], "filter-lang": "cql-json"}
resp = httpx.post(f"{raster_endpoint}/mosaic/register", json=query)
resp = httpx.post(f"{raster_endpoint}/searches/register", json=query)
assert resp.headers["content-type"] == "application/json"
assert resp.status_code == 200
assert resp.json()["searchid"]
assert resp.json()["id"]
assert resp.json()["links"]

searchid = resp.json()["searchid"]
searchid = resp.json()["id"]

resp = httpx.get(f"{raster_endpoint}/mosaic/{searchid}/-85.6358,36.1624/assets")
resp = httpx.get(f"{raster_endpoint}/searches/{searchid}/-85.6358,36.1624/assets")
assert resp.status_code == 200
assert len(resp.json()) == 1
assert list(resp.json()[0]) == ["id", "bbox", "assets", "collection"]
assert resp.json()[0]["id"] == "20200307aC0853900w361030"

resp = httpx.get(f"{raster_endpoint}/mosaic/{searchid}/tiles/15/8589/12849/assets")
resp = httpx.get(
f"{raster_endpoint}/searches/{searchid}/tiles/15/8589/12849/assets"
)
assert resp.status_code == 200
assert len(resp.json()) == 1
assert list(resp.json()[0]) == ["id", "bbox", "assets", "collection"]
assert resp.json()[0]["id"] == "20200307aC0853900w361030"

z, x, y = 15, 8589, 12849
resp = httpx.get(
f"{raster_endpoint}/mosaic/{searchid}/tiles/{z}/{x}/{y}",
f"{raster_endpoint}/searches/{searchid}/tiles/{z}/{x}/{y}",
params={"assets": "cog"},
headers={"Accept-Encoding": "br, gzip"},
timeout=10.0,
Expand Down Expand Up @@ -105,11 +107,11 @@ def test_mosaic_search():
},
]
for search in searches:
resp = httpx.post(f"{raster_endpoint}/mosaic/register", json=search)
resp = httpx.post(f"{raster_endpoint}/searches/register", json=search)
assert resp.status_code == 200
assert resp.json()["searchid"]
assert resp.json()["id"]

resp = httpx.get(f"{raster_endpoint}/mosaic/list")
resp = httpx.get(f"{raster_endpoint}/searches/list")
assert resp.headers["content-type"] == "application/json"
assert resp.status_code == 200
assert (
Expand All @@ -118,16 +120,18 @@ def test_mosaic_search():
assert resp.json()["context"]["returned"] == 10 # default limit is 10

# Make sure all mosaics returned have
for mosaic in resp.json()["searches"]:
assert mosaic["search"]["metadata"]["type"] == "mosaic"
for search in resp.json()["searches"]:
assert search["search"]["metadata"]["type"] == "mosaic"

links = resp.json()["links"]
assert len(links) == 2
assert links[0]["rel"] == "self"
assert links[1]["rel"] == "next"
assert links[1]["href"] == f"{raster_endpoint}/mosaic/list?limit=10&offset=10"
assert links[1]["href"] == f"{raster_endpoint}/searches/list?limit=10&offset=10"

resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"limit": 1, "offset": 1})
resp = httpx.get(
f"{raster_endpoint}/searches/list", params={"limit": 1, "offset": 1}
)
assert resp.status_code == 200
assert resp.json()["context"]["matched"] > 10
assert resp.json()["context"]["limit"] == 1
Expand All @@ -136,33 +140,33 @@ def test_mosaic_search():
links = resp.json()["links"]
assert len(links) == 3
assert links[0]["rel"] == "self"
assert links[0]["href"] == f"{raster_endpoint}/mosaic/list?limit=1&offset=1"
assert links[0]["href"] == f"{raster_endpoint}/searches/list?limit=1&offset=1"
assert links[1]["rel"] == "next"
assert links[1]["href"] == f"{raster_endpoint}/mosaic/list?limit=1&offset=2"
assert links[1]["href"] == f"{raster_endpoint}/searches/list?limit=1&offset=2"
assert links[2]["rel"] == "prev"
assert links[2]["href"] == f"{raster_endpoint}/mosaic/list?limit=1&offset=0"
assert links[2]["href"] == f"{raster_endpoint}/searches/list?limit=1&offset=0"

# Filter on mosaic metadata
resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"owner": "vincent"})
resp = httpx.get(f"{raster_endpoint}/searches/list", params={"owner": "vincent"})
assert resp.status_code == 200
assert resp.json()["context"]["matched"] == 7
assert resp.json()["context"]["limit"] == 10
assert resp.json()["context"]["returned"] == 7

# sortBy
resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "lastused"})
resp = httpx.get(f"{raster_endpoint}/searches/list", params={"sortby": "lastused"})
assert resp.status_code == 200

resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "usecount"})
resp = httpx.get(f"{raster_endpoint}/searches/list", params={"sortby": "usecount"})
assert resp.status_code == 200

resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "-owner"})
resp = httpx.get(f"{raster_endpoint}/searches/list", params={"sortby": "-owner"})
assert resp.status_code == 200
assert (
"owner" not in resp.json()["searches"][0]["search"]["metadata"]
) # some mosaic don't have owners

resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "owner"})
resp = httpx.get(f"{raster_endpoint}/searches/list", params={"sortby": "owner"})
assert resp.status_code == 200
assert "owner" in resp.json()["searches"][0]["search"]["metadata"]

Expand Down
8 changes: 4 additions & 4 deletions raster_api/runtime/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@
inst_reqs = [
"boto3",
"rio-tiler==6.5.0",
"titiler.pgstac==0.8.3",
"titiler.core>=0.15.5,<0.16",
"titiler.mosaic>=0.15.5,<0.16",
"titiler.extensions[cogeo]>=0.15.5,<0.16",
"titiler.pgstac==1.3.0",
"titiler.core>=0.18.5,<0.19",
"titiler.mosaic>=0.18.5,<0.19",
"titiler.extensions[cogeo]>=0.18.5,<0.19",
"starlette-cramjam>=0.3,<0.4",
"aws_xray_sdk>=2.6.0,<3",
"aws-lambda-powertools>=1.18.0",
Expand Down
45 changes: 37 additions & 8 deletions raster_api/runtime/src/app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,25 @@
from starlette.requests import Request
from starlette_cramjam.middleware import CompressionMiddleware
from titiler.core.errors import DEFAULT_STATUS_CODES, add_exception_handlers
from titiler.core.factory import MultiBaseTilerFactory, TilerFactory, TMSFactory
from titiler.core.factory import (
ColorMapFactory,
MultiBaseTilerFactory,
TilerFactory,
TMSFactory,
)
from titiler.core.middleware import CacheControlMiddleware
from titiler.core.resources.enums import OptionalHeader
from titiler.core.resources.responses import JSONResponse
from titiler.extensions import cogValidateExtension, cogViewerExtension
from titiler.mosaic.errors import MOSAIC_STATUS_CODES
from titiler.pgstac.db import close_db_connection, connect_to_db
from titiler.pgstac.factory import MosaicTilerFactory
from titiler.pgstac.dependencies import SearchIdParams
from titiler.pgstac.extensions import searchInfoExtension
from titiler.pgstac.factory import (
MosaicTilerFactory,
add_search_list_route,
add_search_register_route,
)
from titiler.pgstac.reader import PgSTACReader

logging.getLogger("botocore.credentials").disabled = True
Expand Down Expand Up @@ -67,24 +78,36 @@ async def lifespan(app: FastAPI):
# /mosaic - PgSTAC Mosaic titiler endpoint
###############################################################################
mosaic = MosaicTilerFactory(
router_prefix="/mosaic",
router_prefix="/searches/{search_id}",
path_dependency=SearchIdParams,
optional_headers=optional_headers,
environment_dependency=settings.get_gdal_config,
process_dependency=PostProcessParams,
Copy link
Contributor Author

@smohiudd smohiudd Jul 22, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we still need process_dependency if we have tile_dependencies below?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@smohiudd the process dependency is different from the tile_dependency. I see that we have custom post_process method, so if they are still used we should keep the process_dependency

router=APIRouter(route_class=LoggerRouteHandler),
# add /list (default to False)
add_mosaic_list=settings.enable_mosaic_search,
# add /statistics [POST] (default to False)
add_statistics=True,
# add /map viewer (default to False)
add_viewer=False,
# add /bbox [GET] and /feature [POST] (default to False)
add_part=True,
colormap_dependency=ColorMapParams,
extensions=[
searchInfoExtension(),
],
)
app.include_router(mosaic.router, prefix="/searches/{search_id}", tags=["Mosaic"])

add_search_register_route(
Copy link
Contributor

@vincentsarago vincentsarago Jul 24, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add

if settings.enable_mosaic_search:
    add_search_register_route(

app,
prefix="/searches",
# any dependency we want to validate
# when creating the tilejson/map links
tile_dependencies=[mosaic.process_dependency, mosaic.colormap_dependency],
tags=["Mosaic"],
)
app.include_router(mosaic.router, prefix="/mosaic", tags=["Mosaic"])
# TODO
# prefix will be replaced by `/mosaics/{search_id}` in titiler-pgstac 1.0.0
# add /list endpoint
add_search_list_route(app, prefix="/searches", tags=["Mosaic"])
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this the equivalent of the list registered mosaics endpoint that we were setting in the .env as VEDA_RASTER_ENABLE_MOSAIC_SEARCH and then

  1. Inserting in the lambda env in infrastructure construct

  2. And using the runtime config to conditionally enable the list endpoint which could potentially be expensive because it is not paginated (or it was not yet paginated at some point)?

If so, do we want it always on now?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess the tags should be Searches ?



###############################################################################
# /stac - Custom STAC titiler endpoint
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

from titiler.pgstac.dependencies import ItemIdParams

stac = MultiBaseTilerFactory(
    reader=PgSTACReader,
    path_dependency=ItemIdParams,
    optional_headers=optional_headers,
    router_prefix="/stac",
    environment_dependency=settings.get_gdal_config,
    router=APIRouter(route_class=LoggerRouteHandler),
    extensions=[
        stacViewerExtension(),
    ],
    colormap_dependency=ColorMapParams,
)
app.include_router(stac.router, tags=["Items"], prefix="/collections/{collection_id}/items/{item_id}")

###############################################################################
# /alt - Custom STAC titiler endpoint for alternate asset locations
###############################################################################
stac_alt = MultiBaseTilerFactory(
    reader=PgSTACReaderAlt,
    path_dependency=ItemIdParams,
    optional_headers=optional_headers,
    router_prefix="/alt/collections/{collection_id}/items/{item_id}",
    environment_dependency=settings.get_gdal_config,
    router=APIRouter(route_class=LoggerRouteHandler),
    extensions=[
        stacViewerExtension(),
    ],
    colormap_dependency=ColorMapParams,
)
app.include_router(stac_alt.router, tags=["Items"], prefix="/alt/collections/{collection_id}/items/{item_id}")

Expand Down Expand Up @@ -137,6 +160,12 @@ async def lifespan(app: FastAPI):

app.include_router(cog.router, tags=["Cloud Optimized GeoTIFF"], prefix="/cog")

###############################################################################
# Colormaps endpoints
###############################################################################
cmaps = ColorMapFactory()
app.include_router(cmaps.router, tags=["ColorMaps"])


@app.get("/healthz", description="Health Check", tags=["Health Check"])
def ping():
Expand Down
9 changes: 5 additions & 4 deletions raster_api/runtime/src/extensions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,10 @@
from titiler.core.factory import BaseTilerFactory, FactoryExtension

DEFAULT_TEMPLATES = Jinja2Templates(
directory="",
loader=jinja2.ChoiceLoader([jinja2.PackageLoader(__package__, "templates")]),
) # type:ignore
env=jinja2.Environment(
loader=jinja2.ChoiceLoader([jinja2.PackageLoader(__package__, "templates")])
)
)


@dataclass
Expand All @@ -32,9 +33,9 @@ def stac_viewer(
):
"""STAC Viewer."""
return self.templates.TemplateResponse(
request,
name="stac-viewer.html",
context={
"request": request,
"endpoint": request.url.path.replace("/viewer", ""),
"collection": item.collection_id,
"item": item.id,
Expand Down