From fc674f64e753083cf3db1650b47ba4e81804392d Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Thu, 7 Dec 2023 14:53:22 -0700 Subject: [PATCH 1/7] eibkolibri: Remove timeout option from tasks options The tasks in neither Kolibri 0.15 nor 0.16 marshal the `timeout` option. We don't actually need this, though, as the default timeout of 60 seconds is fine. If the server hasn't responded in 60 seconds, there's probably something wrong. The long timeout was added back in 32ed0f2 when we were using Fastly as a Kolibri CDN and I believed it wouldn't respond for a long time in our unusual configuration. https://phabricator.endlessm.com/T34697 --- lib/eibkolibri.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/eibkolibri.py b/lib/eibkolibri.py index 1d49854b..ac0bb515 100755 --- a/lib/eibkolibri.py +++ b/lib/eibkolibri.py @@ -106,7 +106,6 @@ def import_content(session, base_url, channel_id): # Fetch all nodes so that the channel is fully mirrored. 'renderable_only': False, 'fail_on_error': True, - 'timeout': 300, } logger.info(f'Importing channel {channel_id} content') with session.post(url, json=data) as resp: @@ -143,7 +142,6 @@ def update_channel(session, base_url, channel_id): # Fetch all nodes so that the channel is fully mirrored. 'renderable_only': False, 'fail_on_error': True, - 'timeout': 300, } logger.info(f'Updating channel {channel_id} content') with session.post(url, json=data) as resp: From 62d454859b5d6cf4db6f650a391819078ceebf0a Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Fri, 8 Dec 2023 13:16:24 -0700 Subject: [PATCH 2/7] eibkolibri: Fix percentage issue if first status is COMPLETED `last_marker` will still be `None` if the first received status is `COMPLETED`. https://phabricator.endlessm.com/T34697 --- lib/eibkolibri.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/eibkolibri.py b/lib/eibkolibri.py index ac0bb515..12a98335 100755 --- a/lib/eibkolibri.py +++ b/lib/eibkolibri.py @@ -53,7 +53,7 @@ def wait_for_job(session, base_url, job_id): elif status == 'CANCELED': raise Exception(f'Job {job_id} cancelled') elif status == 'COMPLETED': - if last_marker < 100: + if last_marker is None or last_marker < 100: logger.info('Progress: 100%') break From 2e850762d9d0040f4c54d6a772a9f0a009b488f8 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Fri, 8 Dec 2023 14:49:18 -0700 Subject: [PATCH 3/7] eibkolibri: Return boolean from seed_remote_channels This will be used during testing to ensure seeding was not skipped. https://phabricator.endlessm.com/T34697 --- lib/eibkolibri.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/eibkolibri.py b/lib/eibkolibri.py index 12a98335..443b8b6e 100755 --- a/lib/eibkolibri.py +++ b/lib/eibkolibri.py @@ -155,25 +155,30 @@ def update_channel(session, base_url, channel_id): def seed_remote_channels(channel_ids): - """Import channels and content on remote Kolibri server""" + """Import channels and content on remote Kolibri server + + Seeding is skipped if a custom content server is not used or there + are no credentials for the server. Returns True when channels were + seeded and False otherwise. + """ config = eib.get_config() base_url = config.get('kolibri', 'central_content_base_url', fallback=None) if not base_url: logger.info('Not using custom Kolibri content server') - return + return False netrc_path = os.path.join(eib.SYSCONFDIR, 'netrc') if not os.path.exists(netrc_path): logger.info(f'No credentials in {netrc_path}') - return + return False netrc_creds = netrc(netrc_path) host = urlparse(base_url).netloc creds = netrc_creds.authenticators(host) if not creds: logger.info(f'No credentials for {host} in {netrc_path}') - return + return False username, _, password = creds # Start a requests session with the credentials. @@ -194,3 +199,5 @@ def seed_remote_channels(channel_ids): else: import_channel(session, base_url, channel) import_content(session, base_url, channel) + + return True From d647a68f230c5bf47f8ca47b2e6ac4055810b94f Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Thu, 7 Dec 2023 16:54:17 -0700 Subject: [PATCH 4/7] eibkolibri: Refactor remote API requests into class This is a bit nicer from an encapsulation standpoint, but it will primarily be used to handle different server versions more cleanly. https://phabricator.endlessm.com/T34697 --- lib/eibkolibri.py | 292 ++++++++++++++++++++++++---------------------- 1 file changed, 154 insertions(+), 138 deletions(-) diff --git a/lib/eibkolibri.py b/lib/eibkolibri.py index 443b8b6e..e8648e89 100755 --- a/lib/eibkolibri.py +++ b/lib/eibkolibri.py @@ -27,131 +27,161 @@ logger = logging.getLogger(__name__) -def get_job_status(session, base_url, job_id): - """Get remote Kolibri job status""" - url = urljoin(base_url, f'api/tasks/tasks/{job_id}/') - with session.get(url) as resp: - resp.raise_for_status() - return resp.json() - - -def wait_for_job(session, base_url, job_id): - """Wait for remote Kolibri job to complete""" - logger.debug(f'Waiting for job {job_id} to complete') - last_marker = None - while True: - data = get_job_status(session, base_url, job_id) - - # See the kolibri.core.tasks.job.State class for potential states - # https://github.com/learningequality/kolibri/blob/develop/kolibri/core/tasks/job.py#L17 - status = data['status'] - if status == 'FAILED': - logger.error( - f'Job {job_id} failed: {data["exception"]}\n{data["traceback"]}' - ) - raise Exception(f'Job {job_id} failed') - elif status == 'CANCELED': - raise Exception(f'Job {job_id} cancelled') - elif status == 'COMPLETED': - if last_marker is None or last_marker < 100: - logger.info('Progress: 100%') - break - - pct = int(data['percentage'] * 100) - marker = pct - pct % 5 - if last_marker is None or marker > last_marker: - logger.info(f'Progress: {pct}%') - last_marker = marker - - # Wait a bit before checking the status again. - sleep(0.5) - - -def channel_exists(session, base_url, channel_id): - """Check if channel exists on remote Kolibri server""" - url = urljoin(base_url, f'api/content/channel/{channel_id}/') - logger.debug(f'Checking if channel {channel_id} exists') - with session.get(url) as resp: - try: - resp.raise_for_status() - except requests.exceptions.HTTPError: - if resp.status_code == 404: - return False - logger.error('Failed to check channel existence: %s', resp.json()) - raise +class RemoteKolibri: + """Kolibri remote instance""" + def __init__(self, base_url, username, password): + self.base_url = base_url + + # Start a requests session with the credentials. + self.session = requests.Session() + self.session.auth = (username, password) + self.session.headers.update({ + 'Content-Type': 'application/json', + }) + + def import_channel(self, channel_id): + """Import channel and content on remote Kolibri server""" + # Import channel metadata. + url = urljoin( + self.base_url, + 'api/tasks/tasks/startremotechannelimport/', + ) + data = {'channel_id': channel_id} + logger.info(f'Importing channel {channel_id} metadata') + with self.session.post(url, json=data) as resp: + try: + resp.raise_for_status() + except requests.exceptions.HTTPError: + logger.error('Failed to import channel: %s', resp.json()) + raise + job = resp.json() + self._wait_for_job(job['id']) + + # Import channel content. + url = urljoin( + self.base_url, + 'api/tasks/tasks/startremotecontentimport/', + ) + data = { + 'channel_id': channel_id, + # Fetch all nodes so that the channel is fully mirrored. + 'renderable_only': False, + 'fail_on_error': True, + } + logger.info(f'Importing channel {channel_id} content') + with self.session.post(url, json=data) as resp: + try: + resp.raise_for_status() + except requests.exceptions.HTTPError: + logger.error('Failed to import content: %s', resp.json()) + raise + job = resp.json() + self._wait_for_job(job['id']) + + def update_channel(self, channel_id): + """Update channel and content on remote Kolibri server""" + # Generate channel diff stats. + url = urljoin(self.base_url, 'api/tasks/tasks/channeldiffstats/') + data = {'channel_id': channel_id, 'method': 'network'} + logger.info(f'Generating channel {channel_id} diff') + with self.session.post(url, json=data) as resp: + try: + resp.raise_for_status() + except requests.exceptions.HTTPError: + logger.error( + 'Failed to generate channel diff: %s', + resp.json(), + ) + raise + job = resp.json() + self._wait_for_job(job['id']) + + # Update channel metadata and content. + url = urljoin(self.base_url, 'api/tasks/tasks/startchannelupdate/') + data = { + 'channel_id': channel_id, + 'sourcetype': 'remote', + # Fetch all nodes so that the channel is fully mirrored. + 'renderable_only': False, + 'fail_on_error': True, + } + logger.info(f'Updating channel {channel_id} content') + with self.session.post(url, json=data) as resp: + try: + resp.raise_for_status() + except requests.exceptions.HTTPError: + logger.error('Failed to update channel: %s', resp.json()) + raise + job = resp.json() + self._wait_for_job(job['id']) + + def seed_channel(self, channel_id): + """Import or update channel and content on remote Kolibri server + + If the channel exists, it will be updated since Kolibri won't + import new content nodes otherwise. + """ + if self._channel_exists(channel_id): + self.update_channel(channel_id) else: - return True - + self.import_channel(channel_id) -def import_channel(session, base_url, channel_id): - """Import channel on remote Kolibri server""" - url = urljoin(base_url, 'api/tasks/tasks/startremotechannelimport/') - data = {'channel_id': channel_id} - logger.info(f'Importing channel {channel_id} metadata') - with session.post(url, json=data) as resp: - try: - resp.raise_for_status() - except requests.exceptions.HTTPError: - logger.error('Failed to import channel: %s', resp.json()) - raise - job = resp.json() - wait_for_job(session, base_url, job['id']) - - -def import_content(session, base_url, channel_id): - """Import channel content on remote Kolibri server""" - url = urljoin(base_url, 'api/tasks/tasks/startremotecontentimport/') - data = { - 'channel_id': channel_id, - # Fetch all nodes so that the channel is fully mirrored. - 'renderable_only': False, - 'fail_on_error': True, - } - logger.info(f'Importing channel {channel_id} content') - with session.post(url, json=data) as resp: - try: - resp.raise_for_status() - except requests.exceptions.HTTPError: - logger.error('Failed to import content: %s', resp.json()) - raise - job = resp.json() - wait_for_job(session, base_url, job['id']) - - -def diff_channel(session, base_url, channel_id): - """Generate channel diff on remote Kolibri server""" - url = urljoin(base_url, 'api/tasks/tasks/channeldiffstats/') - data = {'channel_id': channel_id, 'method': 'network'} - logger.info(f'Generating channel {channel_id} diff') - with session.post(url, json=data) as resp: - try: + def _get_job_status(self, job_id): + """Get remote Kolibri job status""" + url = urljoin(self.base_url, f'api/tasks/tasks/{job_id}/') + with self.session.get(url) as resp: resp.raise_for_status() - except requests.exceptions.HTTPError: - logger.error('Failed to generate channel diff: %s', resp.json()) - raise - job = resp.json() - wait_for_job(session, base_url, job['id']) - - -def update_channel(session, base_url, channel_id): - """Update channel on remote Kolibri server""" - url = urljoin(base_url, 'api/tasks/tasks/startchannelupdate/') - data = { - 'channel_id': channel_id, - 'sourcetype': 'remote', - # Fetch all nodes so that the channel is fully mirrored. - 'renderable_only': False, - 'fail_on_error': True, - } - logger.info(f'Updating channel {channel_id} content') - with session.post(url, json=data) as resp: - try: - resp.raise_for_status() - except requests.exceptions.HTTPError: - logger.error('Failed to update channel: %s', resp.json()) - raise - job = resp.json() - wait_for_job(session, base_url, job['id']) + return resp.json() + + def _wait_for_job(self, job_id): + """Wait for remote Kolibri job to complete""" + logger.debug(f'Waiting for job {job_id} to complete') + last_marker = None + while True: + data = self._get_job_status(job_id) + + # See the kolibri.core.tasks.job.State class for potential states + # https://github.com/learningequality/kolibri/blob/develop/kolibri/core/tasks/job.py#L17 + status = data['status'] + if status == 'FAILED': + logger.error( + f'Job {job_id} failed: ' + f'{data["exception"]}\n{data["traceback"]}' + ) + raise Exception(f'Job {job_id} failed') + elif status == 'CANCELED': + raise Exception(f'Job {job_id} cancelled') + elif status == 'COMPLETED': + if last_marker is None or last_marker < 100: + logger.info('Progress: 100%') + break + + pct = int(data['percentage'] * 100) + marker = pct - pct % 5 + if last_marker is None or marker > last_marker: + logger.info(f'Progress: {pct}%') + last_marker = marker + + # Wait a bit before checking the status again. + sleep(0.5) + + def _channel_exists(self, channel_id): + """Check if channel exists on remote Kolibri server""" + url = urljoin(self.base_url, f'api/content/channel/{channel_id}/') + logger.debug(f'Checking if channel {channel_id} exists') + with self.session.get(url) as resp: + try: + resp.raise_for_status() + except requests.exceptions.HTTPError: + if resp.status_code == 404: + return False + logger.error( + 'Failed to check channel existence: %s', + resp.json(), + ) + raise + else: + return True def seed_remote_channels(channel_ids): @@ -181,23 +211,9 @@ def seed_remote_channels(channel_ids): return False username, _, password = creds - # Start a requests session with the credentials. - session = requests.Session() - session.auth = (username, password) - session.headers.update({ - 'Content-Type': 'application/json', - }) - + remote = RemoteKolibri(base_url, username, password) for channel in channel_ids: logger.info(f'Seeding channel {channel} on {host}') - - # If the channel exists, update it since Kolibri won't import - # new content nodes otherwise. - if channel_exists(session, base_url, channel): - diff_channel(session, base_url, channel) - update_channel(session, base_url, channel) - else: - import_channel(session, base_url, channel) - import_content(session, base_url, channel) + remote.seed_channel(channel) return True From d8f262800cabee19396c850b8f5b063d8caccaef Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Thu, 7 Dec 2023 21:55:15 -0700 Subject: [PATCH 5/7] eibkolibri: Support Kolibri 0.16 server The tasks API in Kolibri 0.16 is different than 0.15, so some version detection and version specific methods are needed to support it. One advantage of 0.16 is that there's a single `remoteimport` API that combines importing channel metadata and content as well as providing an `update` argument. https://phabricator.endlessm.com/T34697 --- lib/eibkolibri.py | 87 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/lib/eibkolibri.py b/lib/eibkolibri.py index e8648e89..e9a2f440 100755 --- a/lib/eibkolibri.py +++ b/lib/eibkolibri.py @@ -17,6 +17,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import eib +import enum import logging from netrc import netrc import os @@ -29,6 +30,12 @@ class RemoteKolibri: """Kolibri remote instance""" + + class Series(enum.Enum): + """Supported Kolibri server series""" + KOLIBRI_0_15 = enum.auto() + KOLIBRI_0_16 = enum.auto() + def __init__(self, base_url, username, password): self.base_url = base_url @@ -39,8 +46,19 @@ def __init__(self, base_url, username, password): 'Content-Type': 'application/json', }) + self.series = self._get_server_series() + def import_channel(self, channel_id): """Import channel and content on remote Kolibri server""" + if self.series == self.Series.KOLIBRI_0_15: + return self._import_channel_0_15(channel_id) + elif self.series == self.Series.KOLIBRI_0_16: + return self._import_channel_0_16(channel_id) + + raise AssertionError('Unsupported server series') + + def _import_channel_0_15(self, channel_id): + """Import channel and content on remote Kolibri 0.15 server""" # Import channel metadata. url = urljoin( self.base_url, @@ -78,8 +96,39 @@ def import_channel(self, channel_id): job = resp.json() self._wait_for_job(job['id']) + def _import_channel_0_16(self, channel_id, update=False): + """Import channel and content on remote Kolibri 0.16 server""" + url = urljoin(self.base_url, 'api/tasks/tasks/') + data = { + 'type': 'kolibri.core.content.tasks.remoteimport', + 'channel_id': channel_id, + 'channel_name': 'unknown', + 'update': update, + # Fetch all nodes so that the channel is fully mirrored. + 'renderable_only': False, + 'fail_on_error': True, + } + logger.info(f'Importing channel {channel_id}') + with self.session.post(url, json=data) as resp: + try: + resp.raise_for_status() + except requests.exceptions.HTTPError: + logger.error('Failed to import channel: %s', resp.json()) + raise + job = resp.json() + self._wait_for_job(job['id']) + def update_channel(self, channel_id): """Update channel and content on remote Kolibri server""" + if self.series == self.Series.KOLIBRI_0_15: + return self._update_channel_0_15(channel_id) + elif self.series == self.Series.KOLIBRI_0_16: + return self._update_channel_0_16(channel_id) + + raise AssertionError('Unsupported server series') + + def _update_channel_0_15(self, channel_id): + """Update channel and content on remote Kolibri 0.15 server""" # Generate channel diff stats. url = urljoin(self.base_url, 'api/tasks/tasks/channeldiffstats/') data = {'channel_id': channel_id, 'method': 'network'} @@ -115,6 +164,28 @@ def update_channel(self, channel_id): job = resp.json() self._wait_for_job(job['id']) + def _update_channel_0_16(self, channel_id): + """Update channel and content on remote Kolibri 0.15 server""" + # Generate channel diff stats. + url = urljoin(self.base_url, 'api/tasks/tasks/') + data = { + 'type': 'kolibri.core.content.tasks.remotechanneldiffstats', + 'channel_id': channel_id, + 'channel_name': 'unknown', + } + logger.info(f'Generating channel {channel_id} diff') + with self.session.post(url, json=data) as resp: + try: + resp.raise_for_status() + except requests.exceptions.HTTPError: + logger.error('Failed to generate channel diff: %s', resp.json()) + raise + job = resp.json() + self._wait_for_job(job['id']) + + # Update channel metadata and content. + self._import_channel_0_16(channel_id, update=True) + def seed_channel(self, channel_id): """Import or update channel and content on remote Kolibri server @@ -126,6 +197,22 @@ def seed_channel(self, channel_id): else: self.import_channel(channel_id) + def _get_server_series(self): + """Determine the server Kolibri series""" + url = urljoin(self.base_url, 'api/public/info/') + with self.session.get(url) as resp: + resp.raise_for_status() + info = resp.json() + + kolibri_version = info.get('kolibri_version', '') + logger.debug(f'Server Kolibri version: "{kolibri_version}"') + if kolibri_version.startswith('0.15.'): + return self.Series.KOLIBRI_0_15 + elif kolibri_version.startswith('0.16.'): + return self.Series.KOLIBRI_0_16 + + raise Exception(f'Unsupported remote Kolibri version "{kolibri_version}"') + def _get_job_status(self, job_id): """Get remote Kolibri job status""" url = urljoin(self.base_url, f'api/tasks/tasks/{job_id}/') From 542b0d51c1d4dcd7c3e22014f2eb220edd6533c6 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Fri, 8 Dec 2023 10:05:31 -0700 Subject: [PATCH 6/7] tests: Put python dependencies in requirements file Really this is just `pytest` right now (with `flake8` included for lack of a better location), but this provides a place to collect more test dependencies that's better than updating a bunch of adhoc instructions. https://phabricator.endlessm.com/T34697 --- .github/workflows/tests.yml | 6 +++--- README.md | 5 +++-- requirements-test.txt | 3 +++ 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 requirements-test.txt diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index be5f6439..dbd5eabf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,11 +32,11 @@ jobs: ostree \ python3-gi \ python3-pip - - name: Python dependencies - run: | - python3 -m pip install flake8 pytest - name: Checkout uses: actions/checkout@v2 + - name: Python dependencies + run: | + python3 -m pip install -r requirements-test.txt - name: Lint run: | python3 -m flake8 diff --git a/README.md b/README.md index 8238d41a..2cc4f885 100644 --- a/README.md +++ b/README.md @@ -394,8 +394,9 @@ Testing ======= Some parts of the image builder can be tested with [pytest][pytest-url]. -After installing pytest, run `pytest` (or `pytest-3` if `pytest` is for -python 2) from the root of the checkout. +Install the testing dependencies with `pip3 install -r +requirements-test.txt`. After installing pytest, run `pytest` (or +`pytest-3` if `pytest` is for python 2) from the root of the checkout. Various options can be passed to `pytest` to control how the tests are run. See the pytest [usage][pytest-usage] documentation for details. diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 00000000..4b5e87ba --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,3 @@ +# Python requirements for testing +flake8 +pytest From 60c71925a999c6be0ce26aaebffa050c8cd07b15 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Fri, 8 Dec 2023 13:15:24 -0700 Subject: [PATCH 7/7] tests: Add tests for eibkolibri Test the eibkolibri module to ensure it works with both Kolibri 0.15 and 0.16 servers. Server responses are mocked using `requests_mock` rather than trying to spin up either a real or fake Kolibri server. https://phabricator.endlessm.com/T34697 --- requirements-test.txt | 2 + tests/eib/test_eibkolibri.py | 360 +++++++++++++++++++++++++++++++++++ 2 files changed, 362 insertions(+) create mode 100644 tests/eib/test_eibkolibri.py diff --git a/requirements-test.txt b/requirements-test.txt index 4b5e87ba..e668b556 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,5 @@ # Python requirements for testing flake8 pytest +requests +requests-mock diff --git a/tests/eib/test_eibkolibri.py b/tests/eib/test_eibkolibri.py new file mode 100644 index 00000000..f2da63da --- /dev/null +++ b/tests/eib/test_eibkolibri.py @@ -0,0 +1,360 @@ +# Tests for eibkolibri module +# +# Copyright © 2023 Endless OS Foundation LLC +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import logging +import os +import pytest +from urllib.parse import urljoin, urlparse + +import eib +import eibkolibri + +logger = logging.getLogger(__name__) + +SERVER_URL = 'https://kolibri.example.com' +CHANNEL_ID = 'b43aae9d37294548ae75674cd23ddf4a' +JOB_ID = '1d67e97a98be4eb597b5cdb93e998989' + + +class MockKolibriServer: + def __init__(self, requests_mocker, kolibri_version='0.15.12'): + self.mocker = requests_mocker + self.version = kolibri_version + self.setup_responses(self.version) + + def setup_responses(self, kolibri_version): + self.set_version_response(kolibri_version) + self.set_channel_response() + self.set_task_responses(kolibri_version) + self.set_job_status_response() + + def set_version_response(self, kolibri_version): + self.version = kolibri_version + self.mocker.get( + urljoin(SERVER_URL, 'api/public/info/'), + json={ + "application": "kolibri", + "kolibri_version": kolibri_version, + "instance_id": "c9c0307035a3450fb315ed1ebb2fc215", + "device_name": "test", + "operating_system": "Linux" + }, + ) + + def set_channel_response(self, channel_id=CHANNEL_ID, exists=False): + if exists: + code = 200 + data = { + "author": "", + "description": "Test channel", + "tagline": None, + "id": channel_id, + "last_updated": "2023-12-08T10:45:20.401110-07:00", + "name": "Test", + "root": channel_id, + "thumbnail": None, + "version": 2, + "public": True, + "num_coach_contents": None, + "available": True, + "lang_code": "mul", + "lang_name": "Multiple languages", + } + else: + code = 404 + data = [ + { + "id": "NOT_FOUND", + "metadata": { + "view": "Channel Metadata Instance", + }, + }, + ] + + self.mocker.get( + urljoin(SERVER_URL, f'api/content/channel/{channel_id}/'), + status_code=code, + json=data, + ) + + def set_task_responses( + self, + kolibri_version, + job_id=JOB_ID, + channel_id=CHANNEL_ID, + ): + if kolibri_version.startswith('0.15.'): + return self._set_task_responses_0_15( + kolibri_version, + job_id, + channel_id, + ) + elif kolibri_version.startswith('0.16.'): + return self._set_task_responses_0_16( + kolibri_version, + job_id, + channel_id, + ) + + logger.warning( + f'Cannot create task responses for Kolibri {kolibri_version}' + ) + + def _set_task_responses_0_15( + self, + kolibri_version, + job_id, + channel_id, + ): + for task_name in ( + 'channeldiffstats', + 'startchannelupdate', + 'startremotechannelimport', + 'startremotecontentimport', + ): + job_type = task_name.replace('start', '', 1).upper() + data = { + "status": "QUEUED", + "exception": "None", + "traceback": "None", + "percentage": 0, + "id": job_id, + "cancellable": True, + "clearable": False, + "baseurl": "https://studio.learningequality.org", + "type": job_type, + "started_by": "156680771d8d5a1c9628393c5ca73c8e", + "channel_id": channel_id, + } + + self.mocker.post( + urljoin(SERVER_URL, f'api/tasks/tasks/{task_name}/'), + json=data, + ) + + def _set_task_responses_0_16( + self, + kolibri_version, + job_id, + channel_id, + ): + # All tasks use the same URL, so the response needs to be dynamic + # based on the type in the request data. + def task_data_callback(request, context): + # In the real Request, json is a property, but it's a funcion in + # requests_mock's fake Request. Handle both. + if callable(request.json): + req_data = request.json() + else: + req_data = request.json + + job_type = req_data.get('type', '') + data = { + "status": "QUEUED", + "type": job_type, + "exception": None, + "traceback": "", + "percentage": 0, + "id": job_id, + "cancellable": True, + "clearable": False, + "facility_id": None, + "args": [channel_id], + "kwargs": { + "baseurl": "https://studio.learningequality.org/", + "peer_id": None, + }, + "extra_metadata": { + "channel_name": "unknown", + "channel_id": channel_id, + "peer_id": None, + "started_by": "156680771d8d5a1c9628393c5ca73c8e", + "started_by_username": "admin" + }, + "scheduled_datetime": "2023-12-08T18:40:57.472971+00:00", + "repeat": 0, + "repeat_interval": 0, + "retry_interval": None, + } + + if job_type == 'kolibri.core.content.tasks.remoteimport': + data['kwargs'].update({ + "node_ids": None, + "exclude_node_ids": None, + "update": False, + "renderable_only": False, + "fail_on_error": True, + "all_thumbnails": False, + }) + + return data + + self.mocker.post( + urljoin(SERVER_URL, 'api/tasks/tasks/'), + json=task_data_callback, + ) + + def set_job_status_response(self, job_id=JOB_ID, channel_id=CHANNEL_ID): + # The responses are a bit different between 0.15 and 0.16, but the code + # only looks at that status and percentage field, which are the same. + data = { + "status": "COMPLETED", + "exception": "None", + "traceback": "None", + "percentage": 0, + "id": job_id, + "cancellable": True, + "clearable": True, + "baseurl": "https://studio.learningequality.org", + # We're reusing the same job ID for all tasks, so we don't know the + # type. Fortunately, the eibkolibri code doesn't look at it. + "type": "SOMETASK", + "started_by": "156680771d8d5a1c9628393c5ca73c8e", + "channel_id": channel_id, + } + self.mocker.get( + urljoin(SERVER_URL, f'api/tasks/tasks/{job_id}/'), + json=data, + ) + + def update_tasks_run(self): + if self.version.startswith('0.15.'): + def is_update_task_request(request): + if request.method != 'POST': + return False + return request.url.endswith('/channeldiffstats/') + elif self.version.startswith('0.16.'): + def is_update_task_request(request): + if request.method != 'POST': + return False + if callable(request.json): + req_data = request.json() + else: + req_data = request.json + return req_data.get('type') == ( + 'kolibri.core.content.tasks.remotechanneldiffstats' + ) + else: + logger.warning( + f'Cannot parse task responses for Kolibri {self.version}' + ) + return False + + return any([ + is_update_task_request(req) for req in self.mocker.request_history + ]) + + +@pytest.mark.parametrize( + ['version', 'series'], + [ + ('0.15.12', eibkolibri.RemoteKolibri.Series.KOLIBRI_0_15), + ('0.16.0b9', eibkolibri.RemoteKolibri.Series.KOLIBRI_0_16), + ('0.16.5', eibkolibri.RemoteKolibri.Series.KOLIBRI_0_16), + ('0.14.9', None), + ('0.17.0a1', None), + ('1', None), + ('', None), + ], +) +def test_version(version, series, requests_mock): + """Test server version matching""" + server = MockKolibriServer(requests_mock) + + server.set_version_response(version) + if series is None: + # Unsupported or invalid versions + with pytest.raises( + Exception, + match=r'Unsupported remote Kolibri version', + ): + eibkolibri.RemoteKolibri(SERVER_URL, 'admin', 'admin') + else: + # Supported versions + remote = eibkolibri.RemoteKolibri(SERVER_URL, 'admin', 'admin') + assert remote.series == series + + +@pytest.mark.parametrize('version', ['0.15.12', '0.16.0']) +def test_import_channel(version, requests_mock): + """Test importing channel""" + server = MockKolibriServer(requests_mock, version) + server.set_channel_response(exists=False) + remote = eibkolibri.RemoteKolibri(SERVER_URL, 'admin', 'admin') + remote.import_channel(CHANNEL_ID) + assert not server.update_tasks_run() + + +@pytest.mark.parametrize('version', ['0.15.12', '0.16.0']) +def test_update_channel(version, requests_mock): + """Test updating channel""" + server = MockKolibriServer(requests_mock, version) + server.set_channel_response(exists=True) + remote = eibkolibri.RemoteKolibri(SERVER_URL, 'admin', 'admin') + remote.update_channel(CHANNEL_ID) + assert server.update_tasks_run() + + +@pytest.mark.parametrize('version', ['0.15.12', '0.16.0']) +def test_seed_channel(version, requests_mock): + """Test seeding channel""" + server = MockKolibriServer(requests_mock, version) + + # Import channel + server.set_channel_response(exists=False) + remote = eibkolibri.RemoteKolibri(SERVER_URL, 'admin', 'admin') + remote.seed_channel(CHANNEL_ID) + assert server.update_tasks_run() is False + + # Update channel + requests_mock.reset_mock() + server.set_channel_response(exists=True) + remote.seed_channel(CHANNEL_ID) + assert server.update_tasks_run() is True + + +@pytest.mark.parametrize('version', ['0.15.12', '0.16.0']) +def test_seed_remote_channels( + version, + config, + tmp_builder_paths, + monkeypatch, + requests_mock, +): + # Set the mock server URL in the configuration. + config.add_section('kolibri') + config.set('kolibri', 'central_content_base_url', SERVER_URL) + monkeypatch.setattr(eib, 'get_config', lambda: config) + + # Write credentials to the netrc file. + netrc_path = os.path.join(tmp_builder_paths['SYSCONFDIR'], 'netrc') + server_host = urlparse(SERVER_URL).netloc + with open(netrc_path, 'w') as f: + f.write(f'machine {server_host} login admin password admin\n') + + # Import channel + server = MockKolibriServer(requests_mock, version) + server.set_channel_response(exists=False) + assert eibkolibri.seed_remote_channels([CHANNEL_ID]) is True + assert server.update_tasks_run() is False + + # Update channel + requests_mock.reset_mock() + server.set_channel_response(exists=True) + assert eibkolibri.seed_remote_channels([CHANNEL_ID]) is True + assert server.update_tasks_run() is True