Various doc build improvements

.github/workflows/build_one.yml

@@ -77,7 +77,7 @@ jobs:
     - uses: actions/upload-artifact@v3
       with:
         name: ${{ inputs.project }}
-        path: doc/${{ inputs.project }}/
+        path: doc/gallery/${{ inputs.project }}/
         retention-days: 3
     - name: clean project folder
       run: doit clean build_list_existing_files:${{ inputs.project }}
@@ -97,7 +97,7 @@ jobs:
         git config user.name "travis"
 
         # Move doc to move to a tmp directory
-        mv ./doc/$DIR ./tmp
+        mv ./doc/gallery/$DIR ./tmp
 
 
         # Checkout tmp dev branch
@@ -112,13 +112,13 @@ jobs:
             git switch --orphan $BRANCHNAME
         fi
 
-        mkdir -p doc
+        mkdir -p doc/gallery
         git diff
-        if [ -d  ./doc/$DIR ]; then rm -rf ./doc/$DIR; fi
-        mkdir ./doc/$DIR
-        mv ./tmp/* ./doc/$DIR
+        if [ -d  ./doc/gallery/$DIR ]; then rm -rf ./doc/gallery/$DIR; fi
+        mkdir ./doc/gallery/$DIR
+        mv ./tmp/* ./doc/gallery/$DIR
         rmdir ./tmp
-        git add ./doc/$DIR
+        git add ./doc/gallery/$DIR
         git commit -m "adding $DIR"
         git push --force "https://pyviz-developers:${{ secrets.GITHUB_TOKEN }}@github.com/holoviz-topics/examples.git" HEAD:$BRANCHNAME
         git checkout local_branch_qpeori

.github/workflows/docs.yml

@@ -163,14 +163,14 @@ jobs:
           echo "Parse projects to remove them"
           items=$(echo $CHANGEDPROJECTS | jq -c -r '.[]')
           for item in ${items[@]}; do
-              echo "Removing doc/$item..."
-              rm -rf doc/$item/
-              echo "Removed doc/$item"
+              echo "Removing doc/gallery/$item..."
+              rm -rf doc/gallery/$item/
+              echo "Removed doc/gallery/$item"
           done
-          echo "Pull evaluated docs from the deb branch"
-          git checkout $DEVBRANCH -- doc/
+          echo "Pull evaluated docs from the dev branch"
+          git checkout $DEVBRANCH -- doc/gallery/
           git diff
-          git add './doc/'
+          git add './doc/gallery/'
           git commit -m "Add $CHANGEDPROJECTS"
           git log -n 10 --oneline
           echo "Push changes to evaluated"
@@ -192,10 +192,10 @@ jobs:
           echo "Parse projects to remove them"
           items=$(echo $REMOVEDPROJECTS | jq -c -r '.[]')
           for item in ${items[@]}; do
-              echo "Removing doc/$item..."
-              rm -rf doc/$item/
-              echo "Removed doc/$item"
-              git add './doc/$item'
+              echo "Removing doc/gallery/$item..."
+              rm -rf doc/gallery/$item/
+              echo "Removed doc/gallery/$item"
+              git add './doc/gallery/$item'
           done
           git commit -m "Remove $REMOVEDPROJECTS"
           git log -n 10 --oneline
@@ -210,9 +210,9 @@ jobs:
         # Work from a temporary branch
         git checkout -b deploy--temp-asdfghjkl
         git fetch https://github.com/${GITHUB_REPOSITORY}.git evaluated:refs/remotes/evaluated
-        # Checkout only the /doc folder than contains the evaluated artefacts
-        git checkout evaluated -- ./doc
-        tree doc -L 2
+        # Checkout only the /doc/gallery folder than contains the evaluated artefacts
+        git checkout evaluated -- ./doc/gallery
+        tree doc/gallery -L 2
     - name: sync dev evaluated
       # workflow_call events (coming from pr_flow.yml) that did update at least one project
       if: inputs.type == 'workflow_call' && (inputs.changedprojects != '[]' || inputs.removedprojects != '[]' )
@@ -238,23 +238,23 @@ jobs:
           echo "Parse projects to remove them"
           items=$(echo $CHANGEDPROJECTS | jq -c -r '.[]')
           for item in ${items[@]}; do
-              echo "Removing doc/$item..."
-              rm -rf doc/$item/
-              echo "Removed doc/$item"
+              echo "Removing doc/gallery/$item..."
+              rm -rf doc/gallery/$item/
+              echo "Removed doc/gallery/$item"
           done
-          # Checkout only the /doc folder than contains the evaluated artefacts
+          # Checkout only the /doc/gallery folder than contains the evaluated artefacts
           # we want to add to the evaluated branch, albeit just temporarily for this docs build
-          git checkout $DEVBRANCH -- doc/
+          git checkout $DEVBRANCH -- doc/gallery/
           git diff
           # This isn't meant to be pushed, it's just for this docs build
-          git add './doc/'
+          git add './doc/gallery/'
           git commit -m "Add $CHANGEDPROJECTS"
           git checkout ${{ inputs.branch }}
-          # Checkout only the /doc folder than contains the evaluated artefacts
-          git checkout evaluated -- ./doc
+          # Checkout only the /doc/gallery folder than contains the evaluated artefacts
+          git checkout evaluated -- ./doc/gallery
           git diff
           git log -n 10 --oneline
-          tree doc -D -h
+          tree doc/gallery -D -h
           ls
         elif [ "$REMOVEDPROJECTS" != "[]" ]; then
 
@@ -272,29 +272,27 @@ jobs:
           echo "Parse projects to remove them"
           items=$(echo $REMOVEDPROJECTS | jq -c -r '.[]')
           for item in ${items[@]}; do
-              echo "Removing doc/$item..."
-              rm -rf doc/$item/
-              echo "Removed doc/$item"
-              git add './doc/$item'
+              echo "Removing doc/gallery/$item..."
+              rm -rf doc/gallery/$item/
+              echo "Removed doc/gallery/$item"
+              git add './doc/gallery/$item'
           done
           git diff
           # This isn't meant to be pushed, it's just for this docs build
           git commit -m "Remove $REMOVEDPROJECTS"
           git checkout ${{ inputs.branch }}
-          # Checkout only the /doc folder than contains the evaluated artefacts
-          git checkout evaluated -- ./doc
+          # Checkout only the /doc/gallery folder than contains the evaluated artefacts
+          git checkout evaluated -- ./doc/gallery
           git diff
           git log -n 10 --oneline
-          tree doc -D -h
+          tree doc/gallery -D -h
           ls
         fi
     - name: archive projects
       run: |
         doit doc_archive_projects
-    - name: move thumbnails
-      run: doit doc_move_thumbnails
-    - name: make assets
-      run: doit doc_move_assets
+    - name: move content
+      run: doit doc_move_content
     - name: "temp: remove non evaluated projects"
       run: doit doc_remove_not_evaluated
     - name: build dev website
@@ -310,18 +308,18 @@ jobs:
       # ZIP and upload the built site:
       # Only when called from pr_flow.yml. Done as multiple PRs can update the dev website
       # concurrently, this offers a way to download the site and see it locally.
-    - name: zip built site
+    - name: tar built site
       if: inputs.type == 'workflow_call'
-      run: zip -r builtdocs.zip builtdocs/
+      run: tar czf builtdocs.tar.gz builtdocs/
     - uses: actions/upload-artifact@v3
       if: inputs.type == 'workflow_call'
       with:
         name: website
-        path: builtdocs.zip
+        path: builtdocs.tar.gz
         retention-days: 3
     - name: delete zip
       if: inputs.type == 'workflow_call'
-      run: rm builtdocs.zip
+      run: rm builtdocs.tar.gz
     - name: Deploy dev
       # workflow_call, by pr_flow.yml
       # workflow_dispatch and dev target

.gitignore

@@ -29,15 +29,12 @@ __pycache__/
 # OSX
 *.DS_Store
 
-# nbsite
-doc/index.rst
-
 # Ignore output of get_evaluated_doc
-doc/*/
+doc/gallery/*
 !doc/_static/
 !doc/_templates/
 
-# Ignore output of doc_move_assets
+# Ignore output of doc_archive_projects
 assets/
 # but don't ignore the projname/assets/
 !*/assets/

_extensions/gallery.py

@@ -1,6 +1,9 @@
 import glob
 import os
 
+from pathlib import Path
+
+import nbformat
 import sphinx.util
 
 logger = sphinx.util.logging.getLogger('gallery-extension')
@@ -26,7 +29,6 @@
     'default_extensions': ['*.ipynb'],
     'examples_dir': os.path.join('..', 'examples'),
     'labels_dir': 'labels',
-    'alternative_toctree': [],
     'github_project': None,
     'intro': 'Sample intro',
     'title': 'A sample gallery title',
@@ -62,47 +64,27 @@ def sort_index_first(files):
     return sorted_files
 
 
-def generate_file_rst(
-    gallery_conf, src_dir, dest_dir, page, section, skip, prolog,
-):
-    proj = gallery_conf['github_project']
-    examples_dir = gallery_conf['examples_dir']
-    skip_execute = gallery_conf.get('skip_execute', [])
-    extensions = gallery_conf['default_extensions']
-
-    components = [examples_dir.split(os.path.sep)[-1], page]
-    components.append(section)
-
-    files = []
-    for extension in extensions:
-        files += glob.glob(os.path.join(src_dir, extension))
-
-    for f in files:
-        if isinstance(skip, list) and os.path.basename(f) in skip:
+def generate_project_toctree(files):
+    toctree = '.. toctree::\n'
+    toctree += '   :hidden:\n\n'
+    for file in files:
+        name = Path(file).stem
+        if name == 'index':
             continue
-        extension = f.split('.')[-1]
-        basename = os.path.basename(f)
-        rel_path = os.path.relpath(os.path.join(src_dir, basename), dest_dir)
-        rst_path = os.path.join(dest_dir, basename[:-len(extension)].replace(' ', '_') + 'rst')
-
-        if os.path.isfile(rst_path):
-            with open(rst_path) as existing:
-                if not 'Originally generated by gallery-extension' in existing.read():
-                    continue
-
-        with open(rst_path, 'w') as rst_file:
-            if prolog:
-                # Used by examples.holoviz.org to link to the viewed notebook
-                if '/notebooks/{template_notebook_filename}' in prolog:
-                    prolog = prolog.format(
-                        template_notebook_filename=basename,
-                    )
-                rst_file.write(prolog)
+        toctree += f'   {name}\n'
+    return toctree
 
-            rst_file.write(".. notebook:: %s %s" % (proj, rel_path))
-            if (isinstance(skip, bool) and skip) or any(basename.strip().endswith(skipped) for skipped in skip_execute):
-                rst_file.write('\n    :skip_execute: True\n')
 
+def insert_toctree(nb_path, toctree):
+    nb = nbformat.read(nb_path, as_version=4)
+    last_cell = nb['cells'][-1]
+    toctree = "```{eval-rst}\n" + toctree + "\n```"
+    toctree_cell = nbformat.v4.new_markdown_cell(source=toctree)
+    if "```{eval-rst}" in last_cell['source']:
+        nb['cells'][-1] = toctree_cell
+    else:
+        nb['cells'].append(toctree_cell)
+    nbformat.write(nb, nb_path, version=nbformat.NO_CONVERT)
 
 
 def generate_gallery(app):
@@ -113,7 +95,6 @@ def generate_gallery(app):
     # Get config
     gallery_conf = app.config.gallery_conf
     extensions = gallery_conf['default_extensions']
-    alternative_toctree = gallery_conf['alternative_toctree']
 
     gallery_path = gallery_conf['path']
 
@@ -159,22 +140,13 @@ def generate_gallery(app):
         description = section.get('description', None)
         labels = section.get('labels', [])
         skip = section.get('skip', [])
-        prolog = section.get('prolog', '')
 
-        path_components = [gallery_path]
-        path_components.append(section_path)
-
-        path = os.path.join(examples_dir, *path_components)
-        dest_dir = os.path.join(doc_dir, *path_components)
-        try:
-            os.makedirs(dest_dir)
-        except:
-            pass
+        dest_dir = os.path.join(doc_dir, gallery_path, section_path)
 
         # Collect examples
         files = []
         for extension in extensions:
-            files += glob.glob(os.path.join(path, extension))
+            files += glob.glob(os.path.join(dest_dir, extension))
         if skip:
             files = [f for f in files if os.path.basename(f) not in skip]
 
@@ -197,17 +169,6 @@ def generate_gallery(app):
         basenames = []
         for f in files:
 
-            # Generate the notebook rst
-            generate_file_rst(
-                gallery_conf=app.config.gallery_conf,
-                src_dir=path,
-                dest_dir=dest_dir,
-                page=gallery_path,
-                section=section_path,
-                skip=skip,
-                prolog=prolog
-            )
-
             extension = f.split('.')[-1]
             basename = os.path.basename(f)[:-(len(extension)+1)]
             basenames.append(basename)
@@ -248,32 +209,21 @@ def generate_gallery(app):
             )
             gallery_rst += this_entry
 
-        if not alternative_toctree and len(files) > 1:
-            # Append a toctree to the section index.rst file
-            rst_path = os.path.join(dest_dir, 'index.rst')
-            assert os.path.isfile(rst_path), f'index.rst file not found at {rst_path}'
-
-            with open(rst_path, 'a') as rst_file:
-                rst_file.write('\n\n.. toctree::\n   :hidden:\n\n')
-                for basename_ in basenames:
-                    target = 'self' if basename_ == 'index' else basename_
-                    rst_file.write(f'   {target}\n')
+        if len(files) > 1:
+            index_nb = next(file for file in files if file.endswith('index.ipynb'))
+            project_toctree = generate_project_toctree(files)
+            insert_toctree(index_nb, project_toctree)
 
         # Gallery toctree: just put the index file or the only notebook available.
         target = 'index' if 'index' in basenames else basenames[0]
         toctree_entries.append(f'{section_title} <{section_path}/{target}>')
 
     # Add gallery toctree
-    if not alternative_toctree:
-        assert toctree_entries, 'Empty toctree entries.'
-        toctree_rst = '.. toctree::\n   :hidden:\n\n'
-        for toctree_entry in toctree_entries:
-            toctree_entry = 'self' if toctree_entry == 'index' else toctree_entry
-            toctree_rst += f'   {toctree_entry}\n'
-    else:
-        toctree_rst = '.. toctree::\n   :hidden:\n\n'
-        for toctree_entry in alternative_toctree:
-            toctree_rst += f'   {toctree_entry}\n'
+    assert toctree_entries, 'Empty toctree entries.'
+    toctree_rst = '.. toctree::\n   :hidden:\n\n'
+    for toctree_entry in toctree_entries:
+        toctree_entry = 'self' if toctree_entry == 'index' else toctree_entry
+        toctree_rst += f'   {toctree_entry}\n'
 
     gallery_rst += toctree_rst