From b7a876e86e915ab8fcf29fdb37d6b36883e61b3a Mon Sep 17 00:00:00 2001 From: Jonathan Rios Date: Mon, 29 Apr 2024 12:21:03 +0200 Subject: [PATCH] LITE-30091 Invalid rql filter when fetching reports for upload creation --- connect_bi_reporter/uploads/services.py | 2 +- docker-compose.yml | 2 - poetry.lock | 10 ++--- pyproject.toml | 2 +- tests/uploads/test_tasks.py | 60 +++++++++++++++++++------ 5 files changed, 54 insertions(+), 22 deletions(-) diff --git a/connect_bi_reporter/uploads/services.py b/connect_bi_reporter/uploads/services.py index 2c2154d..115c32a 100644 --- a/connect_bi_reporter/uploads/services.py +++ b/connect_bi_reporter/uploads/services.py @@ -118,13 +118,13 @@ def disable_feeds(db, feeds, logger): def create_uploads(db, client, logger, feeds): feeds_to_disable = [] uploads = [] - rql = R().status.eq('succeeded') feed_ids = [f.id for f in feeds] existing_reports_ids = util.flatten_iterator(db.query(Upload.report_id).filter( Upload.feed_id.in_(feed_ids), )) for feed in feeds: report_file = None + rql = R().status.eq('succeeded') rql &= R().account.id.eq(feed.account_id) & R().schedule.id.eq(feed.schedule_id) reason, mark_as_disabled = _get_report_schedule_reason(client, feed.schedule_id) if mark_as_disabled: diff --git a/docker-compose.yml b/docker-compose.yml index 111a95f..982b220 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,8 +36,6 @@ services: DATABASE_URL: postgresql+psycopg2://postgres:1q2w3e@db/bi_reporter depends_on: - db - links: - - "db_ram:db" connect_bi_reporter_test: container_name: connect_bi_reporter_test diff --git a/poetry.lock b/poetry.lock index 0f62a79..e419de0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -362,13 +362,13 @@ urllib3 = "<2" [[package]] name = "connect-extension-utils" -version = "1.1.0" +version = "1.2.0" description = "This repository extend EaaS's out-of-box functionality providing a set of common utilities that can be reused between different extensions." optional = false -python-versions = ">=3.8,<4" +python-versions = "<4,>=3.8" files = [ - {file = "connect_extension_utils-1.1.0-py3-none-any.whl", hash = "sha256:ce9a5ac0975fa2cedf407aeeb65c92b69af45cd60d58e64e8f9fd9feeba4d62e"}, - {file = "connect_extension_utils-1.1.0.tar.gz", hash = "sha256:b64d97d40bb91bcb84ecd7ec515d78660883eb6dd9caecdee4c1c9ed590a8cdd"}, + {file = "connect_extension_utils-1.2.0-py3-none-any.whl", hash = "sha256:c43bde436cd5ceec928109778fcc7137aae5e220884930bcbfc0d4d598ae8421"}, + {file = "connect_extension_utils-1.2.0.tar.gz", hash = "sha256:c7d26cc641762fc1b8fbaad346509f094d47aae9dd7298828a1c638f9dbdde07"}, ] [package.dependencies] @@ -1762,4 +1762,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.8,<4" -content-hash = "c37dd4ddf285159cc69fcd45cb344d6fbe5a0252c232998a9ea7dc9e0e4a5092" +content-hash = "7cca1535bf22df5adf76eb2110bccb9da8cf00b4e7319bfbbbe1d912a84a9afd" diff --git a/pyproject.toml b/pyproject.toml index e57907d..178073d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ sqlalchemy = "^1.3.12" psycopg2-binary = "^2.9.6" pyjwt = "^2.8.0" azure-storage-blob = "^12.19.0" -connect-extension-utils = "^1.0.0" +connect-extension-utils = "^1.2.0" [tool.poetry.dev-dependencies] pytest = ">=6.1.2,<8" diff --git a/tests/uploads/test_tasks.py b/tests/uploads/test_tasks.py index c3eeafd..2ae78cb 100644 --- a/tests/uploads/test_tasks.py +++ b/tests/uploads/test_tasks.py @@ -1,7 +1,9 @@ import re +from unittest.mock import call import pytest from connect.client import ClientError +from connect.client.rql import R from connect.eaas.core.inject.models import Context from sqlalchemy.exc import DBAPIError @@ -213,7 +215,10 @@ def test_create_upload_schedule_task( 'account_id': installation['owner']['id'], }, } - report_file = {'id': 'RP-262-019-481', 'renderer': 'csv'} + report_file = [ + {'id': 'RP-262-019-481', 'renderer': 'csv'}, + {'id': 'RP-262-019-482', 'renderer': 'csv'}, + ] ext = ConnectBiReporterEventsApplication( connect_client, logger, @@ -232,37 +237,66 @@ def test_create_upload_schedule_task( 'connect_bi_reporter.uploads.services.get_report_schedule', return_value=report_schedule, ) - mocker.patch( + p_get_reporting_report = mocker.patch( 'connect_bi_reporter.uploads.services.get_reporting_report', - return_value=report_file, + side_effect=report_file, ) mocker.patch( 'connect_bi_reporter.scheduler.create_schedule_task', return_value=eaas_schedule_task, ) - feed = feed_factory( + feed1 = feed_factory( + schedule_id=report_schedule['id'], + account_id=installation['owner']['id'], + status=feed_factory._meta.model.STATUSES.enabled, + ) + feed2 = feed_factory( schedule_id=report_schedule['id'], account_id=installation['owner']['id'], status=feed_factory._meta.model.STATUSES.enabled, ) result = ext.create_uploads(schedule) - upload = dbsession.query(upload_factory._meta.model).first() - assert result.status == 'success' - assert upload.report_id == report_file['id'] - assert upload.status == upload_factory._meta.model.STATUSES.pending - assert upload.feed_id == feed.id + uploads = dbsession.query(upload_factory._meta.model).all() + p_get_reporting_report.assert_has_calls( + [ + call( + connect_client, ( + R().status.eq('succeeded') & R().account.id.eq(feed1.account_id) + & R().schedule.id.eq(feed1.schedule_id) + ), + ), + call( + connect_client, ( + R().status.eq('succeeded') & R().account.id.eq(feed2.account_id) + & R().schedule.id.eq(feed2.schedule_id) + ), + ), + ], + ) + for idx, zipped in enumerate(zip(uploads, [feed1, feed2])): + upload, feed = zipped + assert result.status == 'success' + assert upload.report_id == report_file[idx]['id'] + assert upload.status == upload_factory._meta.model.STATUSES.pending + assert upload.feed_id == feed.id + assert logger.method_calls[0].args[0] == ( - f'New Uploads were created: `Upload={upload.id}' - f' for Feed={feed.id}`.' + f'New Uploads were created: `Upload={uploads[0].id} for Feed={feed1.id}, ' + f'Upload={uploads[1].id} for Feed={feed2.id}`.' ) assert logger.method_calls[1].args[0] == ( f'Periodic Schedule Task created: `{eaas_schedule_task["id"]}`.' ) assert logger.method_calls[2].args[0] == ( f'New Scheduled Task `{eaas_schedule_task["id"]}`' - f' created for Upload `{upload.id}`: ' - f'Will process Report File `{report_file["id"]}`' + f' created for Upload `{uploads[0].id}`: ' + f'Will process Report File `{report_file[0]["id"]}`' + ) + assert logger.method_calls[4].args[0] == ( + f'New Scheduled Task `{eaas_schedule_task["id"]}`' + f' created for Upload `{uploads[1].id}`: ' + f'Will process Report File `{report_file[1]["id"]}`' )