diff --git a/Dockerfile b/Dockerfile index 9cdae58..ebdf942 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,9 +4,9 @@ COPY app requirements.txt /app WORKDIR /app RUN pip install -r requirements.txt -ENV OVERPASS_URL=https://overpass.kumi.systems/api/interpreter/ +ENV BACKEND_URL=https://overpass.kumi.systems/api/interpreter/ # Leave these variables undefined; to use sentry, provide them in a .env file with docker compose or on the command line. ENV SENTRY_DSN ENV SENTRY_TSR EXPOSE 8080 -CMD python main.py --overpass-url $OVERPASS_URL --sentry-dsn $SENTRY_DSN --sentry-tsr $SENTRY_TSR \ No newline at end of file +CMD python main.py --backend-url $BACKEND_URL --sentry-dsn $SENTRY_DSN --sentry-tsr $SENTRY_TSR \ No newline at end of file diff --git a/README.md b/README.md index 00f1cb6..39068ef 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ It serves map data by sending queries to a public or privately-hosted [Overpass] You can also run the original Soundscape server code as provided by Microsoft. Unlike Overscape, the Microsoft version involves loading and hosting of bulk OpenStreetMap data in a PostGIS database. See the [docker-compose file ](https://github.com/openscape-community/openscape/blob/master/svcs/data/docker-compose.yml) for details on spinning up the necessary services. +Overscape also supports using a PostGIS server as a backend -- simply pass the argument `--backend-url postgres://user:password@host:port/db` when launching the server. + ## Running tests ``` pip install -r requirements_test.txt diff --git a/app/main.py b/app/main.py index 7122b42..01199c9 100644 --- a/app/main.py +++ b/app/main.py @@ -7,8 +7,8 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( - "--overpass-url", - help="URL of Overpass API server", + "--backend-url", + help="URL of Overpass API or PostGIS server", default="https://overpass.kumi.systems/api/interpreter/", # default="http://overpass-api.de/api/interpreter", ) @@ -21,7 +21,7 @@ "--cache-days", type=int, help="Number of days after which cached items should be refreshed", - default=7, + default=30, ) parser.add_argument( "--cache-dir", @@ -58,7 +58,7 @@ logging.basicConfig(level=args.log_level) run_server( - args.overpass_url, + args.backend_url, args.user_agent, args.cache_dir, args.cache_days, diff --git a/app/postgis.py b/app/postgis.py new file mode 100644 index 0000000..1dc8e43 --- /dev/null +++ b/app/postgis.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +import json + +import aiopg +import psycopg2 +from psycopg2.extras import NamedTupleCursor +import sentry_sdk + +from overpass import ZOOM_DEFAULT + + +tile_query = """ + SELECT * from soundscape_tile(%(zoom)s, %(tile_x)s, %(tile_y)s) +""" + + +class PostgisClient: + """A drop-in replacement for OverpassClient that uses a PostGIS server. + The server is assumed to already be populated, including having the + soundscape_tile function installed. + """ + def __init__(self, server, user_agent, cache_dir, cache_days, cache_size): + # all the other args are only used by the OverpassClient + self.server = server + + @sentry_sdk.trace + async def query(self, x, y): + async with aiopg.connect(self.server) as conn: + async with conn.cursor(cursor_factory=NamedTupleCursor) as cursor: + response = await self._gentile_async(cursor, x, y) + return response + + # based on https://github.com/microsoft/soundscape/blob/main/svcs/data/gentiles.py + async def _gentile_async(self, cursor, x, y, zoom=ZOOM_DEFAULT): + try: + await cursor.execute(tile_query, {'zoom': int(zoom), 'tile_x': x, 'tile_y': y}) + value = await cursor.fetchall() + return { + 'type': 'FeatureCollection', + 'features': list(map(lambda x: x._asdict(), value)) + } + except psycopg2.Error as e: + print(e) + raise diff --git a/app/server.py b/app/server.py index 5d9caee..05abc95 100644 --- a/app/server.py +++ b/app/server.py @@ -1,17 +1,27 @@ #!/usr/bin/env python3 import json +from urllib.parse import urlparse + from aiohttp import web -from overpass import ZOOM_DEFAULT, OverpassClient import sentry_sdk from sentry_sdk.integrations.aiohttp import AioHttpIntegration +from overpass import ZOOM_DEFAULT, OverpassClient +from postgis import PostgisClient + import logging logger=logging.getLogger(__name__) +# workaround for aiohttp on WIndows (https://stackoverflow.com/a/69195609) +import sys, asyncio +if sys.version_info >= (3, 8) and sys.platform.lower().startswith("win"): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + + # based on https://github.com/microsoft/soundscape/blob/main/svcs/data/gentiles.py @sentry_sdk.trace -async def gentile_async(zoom, x, y, overpass_client): - response = await overpass_client.query(x, y) +async def gentile_async(zoom, x, y, backend_client): + response = await backend_client.query(x, y) if response is None: return response return json.dumps(response, sort_keys=True) @@ -26,7 +36,7 @@ async def tile_handler(request): x = int(request.match_info["x"]) y = int(request.match_info["y"]) try: - tile_data = await gentile_async(zoom, x, y, request.app["overpass_client"]) + tile_data = await gentile_async(zoom, x, y, request.app["backend_client"]) if tile_data == None: raise web.HTTPServiceUnavailable() else: @@ -35,10 +45,23 @@ async def tile_handler(request): logger.error(f"request: {request.rel_url}") +def backend_client(backend_url, user_agent, cache_dir, cache_days, cache_size): + """Determine which backend to use based on URL format.""" + url_parts = urlparse(backend_url) + if url_parts.scheme in ('http', 'https'): + return OverpassClient( + backend_url, user_agent, cache_dir, cache_days, cache_size + ) + elif url_parts.scheme in ('postgis', 'postgres'): + return PostgisClient( + backend_url, user_agent, cache_dir, cache_days, cache_size + ) + else: + raise ValueError("Unrecognized protocol %r" % url_parts.scheme) def run_server( - overpass_url, + backend_url, user_agent, cache_dir, cache_days, @@ -55,12 +78,11 @@ def run_server( ], ) sentry_sdk.set_tag( - "overpass_url", overpass_url + "backend_url", backend_url ) # Tag all requests for the lifecycle of the app with the overpass URL used app = web.Application() - app["overpass_client"] = OverpassClient( - overpass_url, user_agent, cache_dir, cache_days, cache_size - ) + app["backend_client"] =backend_client( + backend_url, user_agent, cache_dir, cache_days, cache_size) app.add_routes( [ web.get(r"/tiles/{zoom:\d+}/{x:\d+}/{y:\d+}.json", tile_handler), diff --git a/app/tests.py b/app/tests.py index 0035493..9a729a2 100644 --- a/app/tests.py +++ b/app/tests.py @@ -10,6 +10,8 @@ from cache import CompressedJSONCache from overpass import OverpassClient, OverpassResponse +from postgis import PostgisClient +from server import backend_client class TestCompressedJSONCache: @@ -221,3 +223,27 @@ async def test_intersections(self, x, y, overpass_client): ) ) ) + + +class TestPostgisClient: + @pytest.mark.parametrize( + "url,expected_type", + [ + ["https://overpass.kumi.systems/api/interpreter/", OverpassClient], + ["postgres://username:password@example.com:5432/osm/", PostgisClient], + ["ftp://example.com/", ValueError], + ], + ) + def test_url_recognition(self, url, expected_type): + """We should get an OverpassClient, PostgisClient, or ValueError based on the URL.""" + try: + return_value = backend_client( + url, + "Overscape/0.1", + cache_dir=Path("_test_cache"), + cache_days=7, + cache_size=1e5, + ) + except Exception as exc: + return_value = exc + assert type(return_value) is expected_type diff --git a/docker-compose.yml b/docker-compose.yml index 51d9896..7964e84 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,7 @@ services: build: dockerfile: Dockerfile environment: - - OVERPASS_URL=http://overpass/api/interpreter + - BACKEND_URL=http://overpass/api/interpreter - SENTRY_DSN=$SENTRY_DSN - SENTRY_TSR=$SENTRY_TSR ports: diff --git a/requirements.txt b/requirements.txt index bcbb31e..4e3653f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ aiohttp==3.8.4 +aiopg==1.4.0 osm2geojson==0.2.3 shapely==2.0.1 sentry-sdk==1.25.1