Merge pull request #2340 from Turbo87/weglide-upload-task

Add `upload_to_weglide` worker task

Pipfile

@@ -34,6 +34,7 @@ requests-oauthlib = ">=0.6.2,<1.2.0"  # see https://github.com/lepture/flask-oau
 sentry-sdk = {extras = ["flask"],version = "==0.12.3"}
 mapproxy = "*"
 gunicorn = "*"
+requests = "==2.22.0"
 
 [dev-packages]
 fabric = "==1.14.0"

migrations/versions/58325345d375_add_weglide_fields_to_igcfile.py

@@ -0,0 +1,17 @@
+# revision identifiers, used by Alembic.
+revision = "58325345d375"
+down_revision = "70a6f5e6f0e1"
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects.postgresql import JSONB
+
+
+def upgrade():
+    op.add_column("igc_files", sa.Column("weglide_data", JSONB(), nullable=True))
+    op.add_column("igc_files", sa.Column("weglide_status", sa.Integer(), nullable=True))
+
+
+def downgrade():
+    op.drop_column("igc_files", "weglide_status")
+    op.drop_column("igc_files", "weglide_data")

skylines/api/views/upload.py

@@ -290,6 +290,18 @@ def index_post():
         # flush data to make sure we don't get duplicate files from ZIP files
         db.session.flush()
 
+        # Queue WeGlide upload if requested
+        weglide_user_id = data.get("weglideUserId")
+        weglide_birthday = data.get("weglideBirthday")
+        if weglide_user_id and weglide_birthday:
+            tasks.upload_to_weglide.delay(
+                igc_file.id, weglide_user_id, weglide_birthday.isoformat()
+            )
+
+            # Update `weglide_status` in the database
+            igc_file.weglide_status = 1
+            db.session.flush()
+
         # Store data in cache for image creation
         cache_key = hashlib.sha1(
             (to_unicode(flight.id) + u"_" + to_unicode(current_user.id)).encode("utf-8")

skylines/model/igcfile.py

@@ -5,6 +5,7 @@
 
 from sqlalchemy.sql.expression import and_
 from sqlalchemy.types import Integer, DateTime, String, Unicode, Date
+from sqlalchemy.dialects.postgresql import JSONB
 
 from skylines.database import db
 from skylines.lib import files
@@ -33,6 +34,15 @@ class IGCFile(db.Model):
 
     date_utc = db.Column(Date, nullable=False)
 
+    # WeGlide API response HTTP status code,
+    # or None (no upload requested)
+    # or 1 (upload in progress),
+    # or 2 (generic error),
+    weglide_status = db.Column(Integer)
+
+    # WeGlide API response JSON payload, or `null`
+    weglide_data = db.Column(JSONB)
+
     def __repr__(self):
         return unicode_to_str(
             "<IGCFile: id=%d filename='%s'>" % (self.id, self.filename)

skylines/schemas/schemas.py

@@ -263,6 +263,8 @@ class FlightUploadSchema(Schema):
         allow_none=True,
         validate=validate.Length(max=255),
     )
+    weglideBirthday = fields.Date()
+    weglideUserId = fields.Integer()
 
     @pre_load
     def pre_load(self, in_data, **kwargs):
@@ -271,6 +273,10 @@ def pre_load(self, in_data, **kwargs):
             del data["pilotId"]
         if data.get("pilotName") == "":
             del data["pilotName"]
+        if data.get("weglideBirthday") == "":
+            del data["weglideBirthday"]
+        if data.get("weglideUserId") == "":
+            del data["weglideUserId"]
 
         if not data.get("pilotId") and not data.get("pilotName"):
             raise ValidationError("Either pilotName or pilotId must be set")

skylines/weglide.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+
+import logging
+import requests
+import sentry_sdk
+
+from skylines.database import db
+from skylines.lib.files import open_file
+from skylines.model import IGCFile
+
+log = logging.getLogger(__name__)
+
+
+def upload(igc_file_id, weglide_user_id, weglide_birthday):
+    with sentry_sdk.push_scope() as scope:
+        scope.set_context(
+            "WeGlide Upload",
+            {
+                "igc_file_id": igc_file_id,
+                "weglide_user_id": weglide_user_id,
+            },
+        )
+
+        log.info(
+            "Uploading IGC file %s to WeGlide for user %s…",
+            igc_file_id,
+            weglide_user_id,
+        )
+
+        igc_file = db.session.query(IGCFile).get(igc_file_id)
+        if not igc_file:
+            # missing IGCFile in the database is logged and sent to Sentry
+            log.warn(
+                "IGC file (%s) upload failed because it can not be found in the database",
+                igc_file_id,
+            )
+            sentry_sdk.capture_message(
+                "IGC file can not be found in the database", "warn"
+            )
+            return False
+
+        try:
+            with open_file(igc_file.filename) as f:
+                data = {"user_id": weglide_user_id, "date_of_birth": weglide_birthday}
+                files = {"file": (igc_file.filename, f)}
+
+                response = requests.post(
+                    "https://api.weglide.org/v1/igcfile",
+                    data=data,
+                    files=files,
+                    timeout=30,
+                )
+
+                igc_file.weglide_status = response.status_code
+                try:
+                    igc_file.weglide_data = response.json()
+                except ValueError:
+                    # `igc_file.weglide_data` is already `None` so we don't have to do anything
+                    pass
+
+                db.session.commit()
+
+                if 200 <= response.status_code < 300:
+                    log.info("%s: WeGlide IGC file upload successful", igc_file)
+                else:
+                    log.warn(
+                        "%s: WeGlide IGC file upload failed (HTTP %d)",
+                        igc_file,
+                        response.status_code,
+                    )
+
+                return True
+
+        except requests.exceptions.RequestException:
+            # network errors are an expected failure so we only log them as warnings and move on... 🤷‍
+            log.warn("%s: WeGlide IGC file upload failed", igc_file, exc_info=True)
+
+        except Exception:
+            # unexpected errors are logged and sent to Sentry
+            log.error("%s: WeGlide IGC file upload failed", igc_file, exc_info=True)
+            sentry_sdk.capture_exception()
+
+        igc_file.weglide_status = 2
+        db.session.commit()

skylines/worker/tasks.py

@@ -56,3 +56,10 @@ def find_meetings(flight_id):
             )
 
     db.session.commit()
+
+
+@celery.task
+def upload_to_weglide(igc_file_id, weglide_user_id, weglide_birthday):
+    from skylines import weglide
+
+    weglide.upload(igc_file_id, weglide_user_id, weglide_birthday)

tests/api/views/flights/upload_test.py

@@ -1,8 +1,10 @@
+from mock import patch
 from pytest_voluptuous import S, Partial
 from voluptuous.validators import ExactSequence, Datetime, Match, IsTrue
 from werkzeug.datastructures import MultiDict
 
 from skylines.lib.compat import text_type
+from skylines.worker import tasks
 
 from tests.api import auth_for
 from tests.data import users, igcs
@@ -259,3 +261,57 @@ def test_missing_pilot_fields(db_session, client):
         u"error": u"validation-failed",
         u"fields": {u"_schema": [u"Either pilotName or pilotId must be set"]},
     }
+
+
+def test_upload_with_weglide(db_session, client):
+    john = users.john()
+    db_session.add(john)
+    db_session.commit()
+
+    data = dict(
+        pilotId=john.id,
+        weglideUserId="123",
+        weglideBirthday="2020-01-07",
+        files=(igcs.simple_path,),
+    )
+
+    with patch.object(tasks.upload_to_weglide, "delay", return_value=None) as mock:
+        res = client.post("/flights/upload", headers=auth_for(john), data=data)
+
+    mock.assert_called_once()
+    assert len(mock.call_args.args) == 3
+    assert mock.call_args.args[1] == 123
+    assert mock.call_args.args[2] == "2020-01-07"
+
+    assert res.status_code == 200
+    assert res.json == S(
+        {
+            u"club_members": list,
+            u"aircraft_models": list,
+            u"results": ExactSequence(
+                [
+                    {
+                        u"status": 0,
+                        u"cacheKey": IsTrue(),
+                        u"flight": Partial(
+                            {
+                                u"club": None,
+                                u"copilot": None,
+                                u"copilotName": None,
+                                u"distance": 7872,
+                                u"igcFile": dict,
+                                u"pilotName": None,
+                                u"pilot": {
+                                    u"id": john.id,
+                                    u"name": john.name,
+                                },
+                            }
+                        ),
+                        u"name": Match(r".*simple.igc"),
+                        u"trace": dict,
+                        u"airspaces": [],
+                    }
+                ]
+            ),
+        }
+    )

tests/schemas/schemas/test_flight_upload.py

@@ -1,3 +1,4 @@
+from datetime import date
 import pytest
 
 from skylines.schemas import FlightUploadSchema
@@ -54,3 +55,59 @@ def test_pilot_id_and_name(schema):
     data, errors = schema.load({"pilotId": 123, "pilotName": "JD"})
     assert data == {"pilot_id": 123}
     assert errors == {}
+
+
+def test_with_weglide(schema):
+    data, errors = schema.load(
+        {"pilotId": 123, "weglideUserId": 123, "weglideBirthday": "2020-01-01"}
+    )
+    assert data == {
+        "pilot_id": 123,
+        "weglideUserId": 123,
+        "weglideBirthday": date(2020, 1, 1),
+    }
+    assert errors == {}
+
+
+def test_weglide_with_broken_id(schema):
+    data, errors = schema.load(
+        {"pilotId": 123, "weglideUserId": "foo", "weglideBirthday": "2020-01-01"}
+    )
+    assert data == {"pilot_id": 123, "weglideBirthday": date(2020, 1, 1)}
+    assert errors == {"weglideUserId": [u"Not a valid integer."]}
+
+
+def test_weglide_with_missing_id(schema):
+    data, errors = schema.load({"pilotId": 123, "weglideBirthday": "2020-01-01"})
+    assert data == {"pilot_id": 123, "weglideBirthday": date(2020, 1, 1)}
+    assert errors == {}
+
+
+def test_weglide_with_empty_id(schema):
+    data, errors = schema.load(
+        {"pilotId": 123, "weglideUserId": "", "weglideBirthday": "2020-01-01"}
+    )
+    assert data == {"pilot_id": 123, "weglideBirthday": date(2020, 1, 1)}
+    assert errors == {}
+
+
+def test_weglide_with_broken_birthday(schema):
+    data, errors = schema.load(
+        {"pilotId": 123, "weglideUserId": 123, "weglideBirthday": "2abc-01-01"}
+    )
+    assert data == {"pilot_id": 123, "weglideUserId": 123}
+    assert errors == {"weglideBirthday": [u"Not a valid date."]}
+
+
+def test_weglide_with_missing_birthday(schema):
+    data, errors = schema.load({"pilotId": 123, "weglideUserId": 123})
+    assert data == {"pilot_id": 123, "weglideUserId": 123}
+    assert errors == {}
+
+
+def test_weglide_with_empty_birthday(schema):
+    data, errors = schema.load(
+        {"pilotId": 123, "weglideUserId": 123, "weglideBirthday": ""}
+    )
+    assert data == {"pilot_id": 123, "weglideUserId": 123}
+    assert errors == {}