From 75677e50c21f32508fe0717d4cede80c6e280ee8 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Tue, 21 Nov 2023 13:20:28 +1000 Subject: [PATCH] Add support for (some) XYZ tile layers Take care to catch unsuported {q} token in layer uris --- felt/core/api_client.py | 34 +++++++++++++++++++ felt/core/layer_exporter.py | 54 +++++++++++++++++++++++++++++- felt/core/map_uploader.py | 52 ++++++++++++++++++++--------- felt/test/test_layer_exporter.py | 57 ++++++++++++++++++++++++++++++++ 4 files changed, 180 insertions(+), 17 deletions(-) diff --git a/felt/core/api_client.py b/felt/core/api_client.py index 9241bad..7500424 100644 --- a/felt/core/api_client.py +++ b/felt/core/api_client.py @@ -55,6 +55,7 @@ class FeltApiClient: CREATE_MAP_ENDPOINT = '/maps' CREATE_LAYER_ENDPOINT = '/maps/{}/layers' FINISH_LAYER_ENDPOINT = '/maps/{}/layers/{}/finish_upload' + URL_IMPORT_ENDPOINT = '/maps/{}/layers/url_import' USAGE_ENDPOINT = '/internal/reports' RECENT_MAPS_ENDPOINT = '/maps/recent' @@ -187,6 +188,39 @@ def create_map(self, # pylint: enable=unused-argument + def url_import_to_map(self, + map_id: str, + name: str, + layer_url: str, + blocking: bool = False, + feedback: Optional[QgsFeedback] = None) \ + -> Union[QNetworkReply, QgsNetworkReplyContent]: + """ + Prepares a layer upload + """ + request = self._build_request( + self.URL_IMPORT_ENDPOINT.format(map_id), + {'Content-Type': 'application/json'} + ) + + request_params = { + 'name': name, + 'layer_url': layer_url + } + + json_data = json.dumps(request_params) + if blocking: + return QgsNetworkAccessManager.instance().blockingPost( + request, + json_data.encode(), + feedback=feedback + ) + + return QgsNetworkAccessManager.instance().post( + request, + json_data.encode() + ) + def prepare_layer_upload(self, map_id: str, name: str, diff --git a/felt/core/layer_exporter.py b/felt/core/layer_exporter.py index 34758f4..c59e737 100644 --- a/felt/core/layer_exporter.py +++ b/felt/core/layer_exporter.py @@ -13,6 +13,7 @@ # This will get replaced with a git SHA1 when you do a git archive __revision__ = '$Format:%H$' +import json import tempfile import uuid import zipfile @@ -20,7 +21,8 @@ from pathlib import Path from typing import ( Optional, - Tuple + Tuple, + Dict ) from qgis.PyQt.QtCore import ( @@ -31,6 +33,7 @@ QDomDocument ) from qgis.core import ( + QgsDataSourceUri, QgsFeedback, QgsMapLayer, QgsVectorLayer, @@ -54,6 +57,7 @@ QgsReadWriteContext ) +from .api_client import API_CLIENT from .enums import ( LayerExportResult, LayerSupport @@ -61,6 +65,7 @@ from .exceptions import LayerPackagingException from .layer_style import LayerStyle from .logger import Logger +from .map import Map @dataclass @@ -112,6 +117,19 @@ def can_export_layer(layer: QgsMapLayer) \ ): return LayerSupport.Supported, '' + if layer.providerType() == 'wms': + ds = QgsDataSourceUri() + ds.setEncodedUri(layer.source()) + if ds.param('type') == 'xyz': + url = ds.param('url') + if '{q}' in url: + return ( + LayerSupport.NotImplementedProvider, + '{q} token in XYZ tile layers not supported' + ) + + return LayerSupport.Supported, '' + return ( LayerSupport.NotImplementedProvider, '{} raster layers are not yet supported'.format( @@ -125,6 +143,40 @@ def can_export_layer(layer: QgsMapLayer) \ ) ) + @staticmethod + def layer_import_url(layer: QgsMapLayer) -> Optional[str]: + """ + Returns the layer URL if the URL import method should be used to add + a layer + """ + if isinstance(layer, QgsRasterLayer): + if layer.providerType() == 'wms': + ds = QgsDataSourceUri() + ds.setEncodedUri(layer.source()) + if ds.param('type') == 'xyz': + url = ds.param('url') + if '{q}' not in url: + return url + + return None + + @staticmethod + def import_from_url(layer: QgsMapLayer, target_map: Map, + feedback: Optional[QgsFeedback] = None) -> Dict: + """ + Imports a layer from URI to the given map + """ + layer_url = LayerExporter.layer_import_url(layer) + + reply = API_CLIENT.url_import_to_map( + map_id=target_map.id, + name=layer.name(), + layer_url=layer_url, + blocking=True, + feedback=feedback + ) + return json.loads(reply.content().data().decode()) + @staticmethod def representative_layer_style(layer: QgsVectorLayer) -> LayerStyle: """ diff --git a/felt/core/map_uploader.py b/felt/core/map_uploader.py index 5e169e6..69ed9d1 100644 --- a/felt/core/map_uploader.py +++ b/felt/core/map_uploader.py @@ -308,27 +308,47 @@ def run(self): if self.isCanceled(): return False - self.status_changed.emit( - self.tr('Exporting {}').format(layer.name()) - ) - try: - result = exporter.export_layer_for_felt( + if LayerExporter.layer_import_url(layer): + result = LayerExporter.import_from_url( layer, - multi_step_feedback - ) - except LayerPackagingException as e: + self.associated_map, + multi_step_feedback) + + if 'errors' in result: + self.error_string = self.tr( + 'Error occurred while exporting layer {}: {}').format( + layer.name(), + result['errors'][0]['detail'] + ) + self.status_changed.emit(self.error_string) + + return False + layer.moveToThread(None) - self.error_string = self.tr( - 'Error occurred while exporting layer {}: {}').format( - layer.name(), - e + else: + + self.status_changed.emit( + self.tr('Exporting {}').format(layer.name()) ) - self.status_changed.emit(self.error_string) + try: + result = exporter.export_layer_for_felt( + layer, + multi_step_feedback + ) + except LayerPackagingException as e: + layer.moveToThread(None) + self.error_string = self.tr( + 'Error occurred while exporting layer {}: {}').format( + layer.name(), + e + ) + self.status_changed.emit(self.error_string) - return False + return False + + layer.moveToThread(None) + to_upload[layer] = result - layer.moveToThread(None) - to_upload[layer] = result multi_step_feedback.step_finished() if self.isCanceled(): diff --git a/felt/test/test_layer_exporter.py b/felt/test/test_layer_exporter.py index 59e1b76..61a125b 100644 --- a/felt/test/test_layer_exporter.py +++ b/felt/test/test_layer_exporter.py @@ -64,9 +64,66 @@ def test_can_export_layer(self): '&zmax=19&zmin=0', 'test', 'wms') support, reason = LayerExporter.can_export_layer(layer) + self.assertEqual(support, LayerSupport.Supported) + + layer = QgsRasterLayer( + 'http-header:referer=&type=xyz&url=http://ecn.t3.tiles.' + 'virtualearth.net/tiles/a%7Bq%7D.jpeg?g%3D1&zmax=18&zmin=0', + 'test', 'wms') + support, reason = LayerExporter.can_export_layer(layer) + self.assertEqual(support, LayerSupport.NotImplementedProvider) + self.assertEqual(reason, '{q} token in XYZ tile layers not supported') + + layer = QgsRasterLayer( + 'crs=EPSG:4326&dpiMode=7&format=image/png&layers=' + 'wofs_summary_clear&styles&' + 'tilePixelRatio=0&url=https://ows.dea.ga.gov.au/', + 'test', 'wms') + support, reason = LayerExporter.can_export_layer(layer) self.assertEqual(support, LayerSupport.NotImplementedProvider) self.assertEqual(reason, 'wms raster layers are not yet supported') + def test_use_url_import_method(self): + """ + Test determining if layers should use the URL import method + """ + file = str(TEST_DATA_PATH / 'points.gpkg') + layer = QgsVectorLayer(file, 'test') + self.assertTrue(layer.isValid()) + self.assertFalse( + LayerExporter.layer_import_url(layer)) + + file = str(TEST_DATA_PATH / 'dem.tif') + layer = QgsRasterLayer(file, 'test') + self.assertTrue(layer.isValid()) + self.assertFalse( + LayerExporter.layer_import_url(layer)) + + layer = QgsRasterLayer( + 'crs=EPSG:3857&format&type=xyz&url=' + 'https://tile.openstreetmap.org/%7Bz%7D/%7Bx%7D/%7By%7D.png' + '&zmax=19&zmin=0', + 'test', 'wms') + self.assertEqual( + LayerExporter.layer_import_url(layer), + 'https://tile.openstreetmap.org/{z}/{x}/{y}.png' + ) + + layer = QgsRasterLayer( + 'http-header:referer=&type=xyz&url=http://ecn.t3.tiles.' + 'virtualearth.net/tiles/a%7Bq%7D.jpeg?g%3D1&zmax=18&zmin=0', + 'test', 'wms') + self.assertFalse( + LayerExporter.layer_import_url(layer)) + + layer = QgsRasterLayer( + 'crs=EPSG:4326&dpiMode=7&format=image/png&layers=' + 'wofs_summary_clear&styles&tilePixelRatio=0' + '&url=https://ows.dea.ga.gov.au/', + 'test', 'wms') + self.assertFalse( + LayerExporter.layer_import_url(layer)) + def test_file_name(self): """ Test building temporary file names