From b4f8095e375dc2a3768902bf4fcbc2fcc9b544b3 Mon Sep 17 00:00:00 2001 From: Nyall Dawson Date: Fri, 3 May 2024 12:21:36 +1000 Subject: [PATCH] Use v2 API for uploading layers --- felt/core/api_client.py | 43 ++++++++++++++++++++++---- felt/core/constants.py | 1 + felt/core/map_uploader.py | 50 +++++++------------------------ felt/core/s3_upload_parameters.py | 3 +- felt/test/test_api_client.py | 3 +- 5 files changed, 52 insertions(+), 48 deletions(-) diff --git a/felt/core/api_client.py b/felt/core/api_client.py index 6d96813..79e5771 100644 --- a/felt/core/api_client.py +++ b/felt/core/api_client.py @@ -42,7 +42,8 @@ from .s3_upload_parameters import S3UploadParameters from .enums import UsageType from .constants import ( - FELT_API_URL + FELT_API_URL, + FELT_APIV2_URL ) PLUGIN_VERSION = "0.7.0" @@ -61,6 +62,7 @@ class FeltApiClient: URL_IMPORT_ENDPOINT = '/maps/{}/layers/url_import' USAGE_ENDPOINT = '/internal/reports' RECENT_MAPS_ENDPOINT = '/maps/recent' + UPLOAD_V2_ENDPOINT = '/maps/{}/upload' def __init__(self): # default headers to add to all requests @@ -84,11 +86,14 @@ def set_token(self, token: Optional[str]): pass @staticmethod - def build_url(endpoint: str) -> QUrl: + def build_url(endpoint: str, version: int = 1) -> QUrl: """ Returns the full url of the specified endpoint """ - return QUrl(FELT_API_URL + endpoint) + if version == 1: + return QUrl(FELT_API_URL + endpoint) + elif version == 2: + return QUrl(FELT_APIV2_URL + endpoint) @staticmethod def _to_url_query(parameters: Dict[str, object]) -> QUrlQuery: @@ -104,12 +109,13 @@ def _to_url_query(parameters: Dict[str, object]) -> QUrlQuery: query.addQueryItem(name, str(value)) return query - def _build_request(self, endpoint: str, headers=None, params=None) \ + def _build_request(self, endpoint: str, headers=None, params=None, + version: int = 1) \ -> QNetworkRequest: """ Builds a network request """ - url = self.build_url(endpoint) + url = self.build_url(endpoint, version) if params: url.setQuery(FeltApiClient._to_url_query(params)) @@ -281,6 +287,33 @@ def prepare_layer_upload(self, json_data.encode() ) + def prepare_layer_upload_v2(self, + map_id: str, + name: str, + feedback: Optional[QgsFeedback] = None) \ + -> Union[QNetworkReply, QgsNetworkReplyContent]: + """ + Prepares a layer upload, using v2 api + """ + request = self._build_request( + self.UPLOAD_V2_ENDPOINT.format(map_id), + {'Content-Type': 'application/json'}, + version=2 + ) + + request_params = { + 'name': name + } + + json_data = json.dumps(request_params) + reply = QgsNetworkAccessManager.instance().blockingPost( + request, + json_data.encode(), + feedback=feedback + ) + + return reply + def create_upload_file_request(self, filename: str, content: bytes, diff --git a/felt/core/constants.py b/felt/core/constants.py index 0b6268c..39a23f1 100644 --- a/felt/core/constants.py +++ b/felt/core/constants.py @@ -15,6 +15,7 @@ FELT_API_BASE = "https://felt.com" FELT_API_URL = f'{FELT_API_BASE}/api/v1' +FELT_APIV2_URL = f'{FELT_API_BASE}/api/v2' SIGNUP_URL = f'{FELT_API_BASE}/signup' TOS_URL = f'{FELT_API_BASE}/terms' PRIVACY_POLICY_URL = f'{FELT_API_BASE}/privacy' diff --git a/felt/core/map_uploader.py b/felt/core/map_uploader.py index 199d147..2abf7fa 100644 --- a/felt/core/map_uploader.py +++ b/felt/core/map_uploader.py @@ -13,6 +13,7 @@ # This will get replaced with a git SHA1 when you do a git archive __revision__ = '$Format:%H$' +import json import re from collections import defaultdict from pathlib import Path @@ -381,14 +382,12 @@ def run(self): ) while True: - reply = API_CLIENT.prepare_layer_upload( + reply = API_CLIENT.prepare_layer_upload_v2( map_id=self.associated_map.id, name=layer.name(), - file_names=[Path(details.filename).name], - style=details.style, - blocking=True, feedback=self.feedback ) + if reply.attribute( QNetworkRequest.HttpStatusCodeAttribute) == 429: self.status_changed.emit( @@ -413,9 +412,14 @@ def run(self): if self.isCanceled(): return False + upload_details = json.loads(reply.content().data().decode()) upload_params = S3UploadParameters.from_json( - reply.content().data().decode() - ) + upload_details) + + # unused in api v2? + # file_names = [Path(details.filename).name], + # style = details.style, + if not upload_params.url: self.error_string = self.tr('Could not prepare layer upload') message = "Error retrieving upload parameters: {}".format( @@ -470,40 +474,6 @@ def _upload_progress(sent, total): if self.isCanceled(): return False - self.status_changed.emit( - self.tr('Finalizing {}').format(layer.name()) - ) - - while True: - reply = API_CLIENT.finalize_layer_upload( - self.associated_map.id, - upload_params.layer_id, - Path(details.filename).name, - blocking=True, - feedback=self.feedback - ) - if reply.attribute( - QNetworkRequest.HttpStatusCodeAttribute) == 429: - self.status_changed.emit( - self.tr('Rate throttled -- waiting') - ) - QThread.sleep(5) - continue - - if reply.error() != QNetworkReply.NoError: - self.error_string = reply.errorString() - Logger.instance().log_error_json( - { - 'type': Logger.MAP_EXPORT, - 'error': - 'Error finalizing layer upload: {}'.format( - self.error_string) - } - ) - return False - - break - multi_step_feedback.step_finished() return True diff --git a/felt/core/s3_upload_parameters.py b/felt/core/s3_upload_parameters.py index 73bcffe..243d448 100644 --- a/felt/core/s3_upload_parameters.py +++ b/felt/core/s3_upload_parameters.py @@ -56,11 +56,10 @@ def to_form_fields(self) -> Dict: } @staticmethod - def from_json(jsons: str) -> 'S3UploadParameters': + def from_json(res: str) -> 'S3UploadParameters': """ Creates upload parameters from a JSON string """ - res = json.loads(jsons) return S3UploadParameters( type=res.get('data', {}).get('type'), aws_access_key_id=res.get('data', {}).get('attributes', {}).get( diff --git a/felt/test/test_api_client.py b/felt/test/test_api_client.py index 1eef631..d47cd2c 100644 --- a/felt/test/test_api_client.py +++ b/felt/test/test_api_client.py @@ -15,6 +15,7 @@ __revision__ = '$Format:%H$' import unittest +import json from pathlib import Path from qgis.PyQt.QtCore import ( @@ -214,7 +215,7 @@ def test_create_layer(self): QNetworkReply.NoError) json_params = reply.readAll().data().decode() - params = S3UploadParameters.from_json(json_params) + params = S3UploadParameters.from_json(json.loads(json_params)) self.assertEqual(params.type, 'presigned_upload') self.assertTrue(params.aws_access_key_id, 'presigned_upload') # self.assertTrue(params.acl)