From be5072f0decf9721388bfe5f737c23b9e28dbd7f Mon Sep 17 00:00:00 2001
From: Dan Nicholson <dbn@endlessos.org>
Date: Thu, 9 Nov 2023 10:58:47 -0700
Subject: [PATCH 1/2] hooks: Request Kolibri content server fetch all nodes

By default, Kolibri filters the content nodes to download by the types
of content the instance can render. For the content server we want to
fully mirror the channel and download all nodes regardless of whether
the content can be rendered or not.

To do that, the API request is made with the `renderable_only` option
set to `False`. For better or worse, Kolibri 0.15 ignores unknown
options, so this will only do anything useful if the content server
handles the `renderable_only` option[1].

1. https://github.com/learningequality/kolibri/pull/11513

Helps: https://github.com/endlessm/endless-key-content-private/issues/103
---
 .editorconfig                    | 2 +-
 hooks/content/50-kolibri-content | 2 ++
 hooks/image/52-ek-content-cache  | 2 ++
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/.editorconfig b/.editorconfig
index ea5a6474..5fcf8bcf 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -14,6 +14,6 @@ indent_style = space
 indent_size = 2
 
 # Python
-[{*.py,eos-image-builder,run-build,helpers/{assemble-manifest,fetch-remote-collection-id,generate-ovf-files,kill-chroot-procs,kolibri-pick-content-from-channel,mutable-path,packages-manifest},hooks/{content/{50-flatpak,50-kolibri-content},image/{50-flatpak.chroot,62-kolibri-options,70-flatpak-manifest,70-ostree-manifest}}}]
+[{*.py,eos-image-builder,run-build,helpers/{assemble-manifest,fetch-remote-collection-id,generate-ovf-files,kill-chroot-procs,kolibri-pick-content-from-channel,mutable-path,packages-manifest},hooks/{content/{50-flatpak,50-kolibri-content},image/{50-flatpak.chroot,52-ek-content-cache,62-kolibri-options,70-flatpak-manifest,70-ostree-manifest}}}]
 indent_size = 4
 max_line_length = 88
diff --git a/hooks/content/50-kolibri-content b/hooks/content/50-kolibri-content
index 6efd84b0..9df3aea5 100755
--- a/hooks/content/50-kolibri-content
+++ b/hooks/content/50-kolibri-content
@@ -68,6 +68,8 @@ def import_content(session, base_url, channel_id, node_ids=None,
     url = urljoin(base_url, 'api/tasks/tasks/startremotecontentimport/')
     data = {
         'channel_id': channel_id,
+        # Fetch all nodes so that the channel is fully mirrored.
+        'renderable_only': False,
         'fail_on_error': True,
         'timeout': 300,
     }
diff --git a/hooks/image/52-ek-content-cache b/hooks/image/52-ek-content-cache
index 48452380..533617bf 100755
--- a/hooks/image/52-ek-content-cache
+++ b/hooks/image/52-ek-content-cache
@@ -66,6 +66,8 @@ def import_content(session, base_url, channel_id):
     url = urljoin(base_url, 'api/tasks/tasks/startremotecontentimport/')
     data = {
         'channel_id': channel_id,
+        # Fetch all nodes so that the channel is fully mirrored.
+        'renderable_only': False,
         'fail_on_error': True,
         'timeout': 300,
     }

From 8fec870b790b25112e89e56917296dec3fe0fc2e Mon Sep 17 00:00:00 2001
From: Dan Nicholson <dbn@endlessos.org>
Date: Wed, 8 Nov 2023 16:36:20 -0700
Subject: [PATCH 2/2] image: Ensure all available Kolibri content downloaded

By default, Kolibri only imports content nodes that the current instance
has a renderer for. Since we're operating on a homedir that may be
missing plugins that are available at runtime (such as the zim plugin),
it may skip nodes that we ultimately want at runtime. Use the
`--include-unrenderable-content` option so that no nodes are filtered
out.

Helps: https://github.com/endlessm/endless-key-content-private/issues/103
---
 hooks/image/53-ek-content-preload | 4 +++-
 hooks/image/60-kolibri-content    | 5 +++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/hooks/image/53-ek-content-preload b/hooks/image/53-ek-content-preload
index beab1d43..9c5652f4 100644
--- a/hooks/image/53-ek-content-preload
+++ b/hooks/image/53-ek-content-preload
@@ -29,7 +29,9 @@ channels=$(sort -u "${EIB_TMPDIR}"/ek-channels)
 for channel in $channels; do
   kolibri manage --skip-update importchannel network "${channel}"
   EIB_RETRY_ATTEMPTS=2 EIB_RETRY_INTERVAL=30 eib_retry \
-    kolibri manage --skip-update importcontent --fail-on-error network "${channel}"
+    kolibri manage --skip-update \
+    importcontent --include-unrenderable-content --fail-on-error \
+    network "${channel}"
 done
 
 # Empty the user database, and ensure that each instance of this image has a
diff --git a/hooks/image/60-kolibri-content b/hooks/image/60-kolibri-content
index b3f69b9e..983bd1ae 100644
--- a/hooks/image/60-kolibri-content
+++ b/hooks/image/60-kolibri-content
@@ -10,6 +10,11 @@ import_kolibri_channel()
   local channel_include_node_ids_var="EIB_KOLIBRI_${channel_id^^}_INCLUDE_NODE_IDS"
   local channel_exclude_node_ids_var="EIB_KOLIBRI_${channel_id^^}_EXCLUDE_NODE_IDS"
   local importcontent_opts=(
+    # By default, importcontent skips content nodes that it doesn't
+    # think are renderable. Since we don't know what renderers will be
+    # available at runtime, request everything.
+    --include-unrenderable-content
+
     # Normally importcontent ignores download errors. Make it fail so we
     # can be sure we've fully provisioned channels.
     --fail-on-error