Preload Endless Key content

We want to preload the whole Endless Key library in Endless OS images.
We also need to ensure the content is available in our Kolibri cache
server, if we are pulling the content from it.

#117

config/defaults.ini

@@ -134,6 +134,9 @@ hooks_add =
   50-gnome-software-cache
   50-eos-dir-mode-700
   50-xkb-layout.chroot
+  51-ek-content-list
+  52-ek-content-cache
+  53-ek-content-preload
   60-dconf-prepare
   60-flatpak-autoinstall-counters.chroot
   60-kolibri-content

hooks/image/51-ek-content-list

@@ -0,0 +1,11 @@
+# Populate the Endless Key home directory
+
+if [[ ! "${EIB_FLATPAK_REMOTE_FLATHUB_APPS}" =~ .*"org.endlessos.Key".* ]]; then
+  exit 0
+fi
+
+rm -f "${EIB_TMPDIR}"/ek-channels
+collections="${OSTREE_VAR}"/lib/flatpak/app/org.endlessos.Key/current/active/files/share/endless-key/collections/*.json
+for collection in ${collections}; do
+  jq -r '.channels[].id' "${collection}" >> "${EIB_TMPDIR}"/ek-channels
+done

hooks/image/52-ek-content-cache

@@ -0,0 +1,125 @@
+#!/usr/bin/python3
+
+# Instruct the Kolibri content server to import Endless Key channels needed for
+# this build.
+
+import eib
+import logging
+from netrc import netrc
+import os
+import requests
+from urllib.parse import urljoin, urlparse
+
+logger = logging.getLogger(os.path.basename(__file__))
+
+
+def get_job_status(session, base_url, job_id):
+    url = urljoin(base_url, f'api/tasks/tasks/{job_id}/')
+    with session.get(url) as resp:
+        resp.raise_for_status()
+        return resp.json()
+
+
+def wait_for_job(session, base_url, job_id):
+    logger.debug(f'Waiting for job {job_id} to complete')
+    last_marker = None
+    while True:
+        data = get_job_status(session, base_url, job_id)
+
+        # See the kolibri.core.tasks.job.State class for potential states
+        # https://github.com/learningequality/kolibri/blob/develop/kolibri/core/tasks/job.py#L17
+        status = data['status']
+        if status == 'FAILED':
+            logger.error(
+              f'Job {job_id} failed: {data["exception"]}\n{data["traceback"]}'
+            )
+            raise Exception(f'Job {job_id} failed')
+        elif status == 'CANCELED':
+            raise Exception(f'Job {job_id} cancelled')
+        elif status == 'COMPLETED':
+            if last_marker < 100:
+                logger.info('Progress: 100%')
+            break
+
+        pct = int(data['percentage'] * 100)
+        marker = pct - pct % 5
+        if last_marker is None or marker > last_marker:
+            logger.info(f'Progress: {pct}%')
+            last_marker = marker
+
+
+def import_channel(session, base_url, channel_id):
+    url = urljoin(base_url, 'api/tasks/tasks/startremotechannelimport/')
+    data = {'channel_id': channel_id}
+    logger.info(f'Importing channel {channel_id} metadata')
+    with session.post(url, json=data) as resp:
+        try:
+            resp.raise_for_status()
+        except requests.exceptions.HTTPError:
+            logger.error('Failed to import channel: %s', resp.json())
+            raise
+        job = resp.json()
+    wait_for_job(session, base_url, job['id'])
+
+
+def import_content(session, base_url, channel_id):
+    url = urljoin(base_url, 'api/tasks/tasks/startremotecontentimport/')
+    data = {
+        'channel_id': channel_id,
+        'fail_on_error': True,
+        'timeout': 300,
+    }
+    logger.info(f'Importing channel {channel_id} content')
+    with session.post(url, json=data) as resp:
+        try:
+            resp.raise_for_status()
+        except requests.exceptions.HTTPError:
+            logger.error('Failed to import content: %s', resp.json())
+            raise
+        job = resp.json()
+    wait_for_job(session, base_url, job['id'])
+
+
+def main():
+    eib.setup_logging()
+    config = eib.get_config()
+
+    base_url = config.get('kolibri', 'central_content_base_url', fallback=None)
+    if not base_url:
+        logger.info('Not using custom Kolibri content server')
+        return
+
+    netrc_path = os.path.join(eib.SYSCONFDIR, 'netrc')
+    if not os.path.exists(netrc_path):
+        logger.info(f'No credentials in {netrc_path}')
+        return
+
+    channels_path = os.path.join(os.environ['EIB_TMPDIR'], 'ek-channels')
+    if not os.path.exists(channels_path):
+        logger.info(f'No channel list in {channels_path}')
+        return
+
+    netrc_creds = netrc(netrc_path)
+    host = urlparse(base_url).netloc
+    creds = netrc_creds.authenticators(host)
+    if not creds:
+        logger.info(f'No credentials for {host}')
+        return
+    username, _, password = creds
+
+    # Start a requests session with the credentials.
+    session = requests.Session()
+    session.auth = (username, password)
+    session.headers.update({
+        'Content-Type': 'application/json',
+    })
+
+    with open(channels_path) as channels_file:
+        for line in channels_file:
+            channel = line.strip()
+            import_channel(session, base_url, channel)
+            import_content(session, base_url, channel)
+
+
+if __name__ == '__main__':
+    main()

hooks/image/53-ek-content-preload

@@ -0,0 +1,34 @@
+# Populate the Endless Key home directory
+
+if [ ! -f "${EIB_TMPDIR}"/ek-channels ]; then
+  exit 0
+fi
+
+# Use a separate content URL if configured.
+if [ -n "${EIB_KOLIBRI_CENTRAL_CONTENT_BASE_URL}" ]; then
+  KOLIBRI_CENTRAL_CONTENT_BASE_URL="${EIB_KOLIBRI_CENTRAL_CONTENT_BASE_URL}"
+  export KOLIBRI_CENTRAL_CONTENT_BASE_URL
+fi
+
+# Do not create symlinks for static files inside the image builder.
+export KOLIBRI_STATIC_USE_SYMLINKS=0
+
+venv_dir="${EIB_TMPDIR}/kolibri-content-venv"
+python3 -m venv ${venv_dir}
+source ${venv_dir}/bin/activate
+pip install kolibri==${EIB_KOLIBRI_APP_VERSION}
+
+export KOLIBRI_HOME="${OSTREE_VAR}"/lib/endless-key/data
+mkdir -p "${KOLIBRI_HOME}"
+
+channels=$(sort -u "${EIB_TMPDIR}"/ek-channels)
+for channel in $channels; do
+  kolibri manage importchannel network "${channel}"
+  EIB_RETRY_ATTEMPTS=2 EIB_RETRY_INTERVAL=30 eib_retry \
+    kolibri manage importcontent --fail-on-error network "${channel}"
+done
+
+# Empty the user database, and ensure that each instance of this image has a
+# unique Facility ID.
+# <https://kolibri.readthedocs.io/en/latest/install/provision.html#prepare-the-kolibri-folder-for-copying>
+(echo yes; echo yes) | kolibri manage deprovision