diff --git a/.github/workflows/build_one.yml b/.github/workflows/build_one.yml index 17bb83f84..70900ece8 100644 --- a/.github/workflows/build_one.yml +++ b/.github/workflows/build_one.yml @@ -77,7 +77,7 @@ jobs: - uses: actions/upload-artifact@v3 with: name: ${{ inputs.project }} - path: doc/${{ inputs.project }}/ + path: doc/gallery/${{ inputs.project }}/ retention-days: 3 - name: clean project folder run: doit clean build_list_existing_files:${{ inputs.project }} @@ -97,7 +97,7 @@ jobs: git config user.name "travis" # Move doc to move to a tmp directory - mv ./doc/$DIR ./tmp + mv ./doc/gallery/$DIR ./tmp # Checkout tmp dev branch @@ -112,13 +112,13 @@ jobs: git switch --orphan $BRANCHNAME fi - mkdir -p doc + mkdir -p doc/gallery git diff - if [ -d ./doc/$DIR ]; then rm -rf ./doc/$DIR; fi - mkdir ./doc/$DIR - mv ./tmp/* ./doc/$DIR + if [ -d ./doc/gallery/$DIR ]; then rm -rf ./doc/gallery/$DIR; fi + mkdir ./doc/gallery/$DIR + mv ./tmp/* ./doc/gallery/$DIR rmdir ./tmp - git add ./doc/$DIR + git add ./doc/gallery/$DIR git commit -m "adding $DIR" git push --force "https://pyviz-developers:${{ secrets.GITHUB_TOKEN }}@github.com/holoviz-topics/examples.git" HEAD:$BRANCHNAME git checkout local_branch_qpeori diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3e8699e7e..c01dcc257 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -163,14 +163,14 @@ jobs: echo "Parse projects to remove them" items=$(echo $CHANGEDPROJECTS | jq -c -r '.[]') for item in ${items[@]}; do - echo "Removing doc/$item..." - rm -rf doc/$item/ - echo "Removed doc/$item" + echo "Removing doc/gallery/$item..." + rm -rf doc/gallery/$item/ + echo "Removed doc/gallery/$item" done - echo "Pull evaluated docs from the deb branch" - git checkout $DEVBRANCH -- doc/ + echo "Pull evaluated docs from the dev branch" + git checkout $DEVBRANCH -- doc/gallery/ git diff - git add './doc/' + git add './doc/gallery/' git commit -m "Add $CHANGEDPROJECTS" git log -n 10 --oneline echo "Push changes to evaluated" @@ -192,10 +192,10 @@ jobs: echo "Parse projects to remove them" items=$(echo $REMOVEDPROJECTS | jq -c -r '.[]') for item in ${items[@]}; do - echo "Removing doc/$item..." - rm -rf doc/$item/ - echo "Removed doc/$item" - git add './doc/$item' + echo "Removing doc/gallery/$item..." + rm -rf doc/gallery/$item/ + echo "Removed doc/gallery/$item" + git add './doc/gallery/$item' done git commit -m "Remove $REMOVEDPROJECTS" git log -n 10 --oneline @@ -210,9 +210,9 @@ jobs: # Work from a temporary branch git checkout -b deploy--temp-asdfghjkl git fetch https://github.com/${GITHUB_REPOSITORY}.git evaluated:refs/remotes/evaluated - # Checkout only the /doc folder than contains the evaluated artefacts - git checkout evaluated -- ./doc - tree doc -L 2 + # Checkout only the /doc/gallery folder than contains the evaluated artefacts + git checkout evaluated -- ./doc/gallery + tree doc/gallery -L 2 - name: sync dev evaluated # workflow_call events (coming from pr_flow.yml) that did update at least one project if: inputs.type == 'workflow_call' && (inputs.changedprojects != '[]' || inputs.removedprojects != '[]' ) @@ -238,23 +238,23 @@ jobs: echo "Parse projects to remove them" items=$(echo $CHANGEDPROJECTS | jq -c -r '.[]') for item in ${items[@]}; do - echo "Removing doc/$item..." - rm -rf doc/$item/ - echo "Removed doc/$item" + echo "Removing doc/gallery/$item..." + rm -rf doc/gallery/$item/ + echo "Removed doc/gallery/$item" done - # Checkout only the /doc folder than contains the evaluated artefacts + # Checkout only the /doc/gallery folder than contains the evaluated artefacts # we want to add to the evaluated branch, albeit just temporarily for this docs build - git checkout $DEVBRANCH -- doc/ + git checkout $DEVBRANCH -- doc/gallery/ git diff # This isn't meant to be pushed, it's just for this docs build - git add './doc/' + git add './doc/gallery/' git commit -m "Add $CHANGEDPROJECTS" git checkout ${{ inputs.branch }} - # Checkout only the /doc folder than contains the evaluated artefacts - git checkout evaluated -- ./doc + # Checkout only the /doc/gallery folder than contains the evaluated artefacts + git checkout evaluated -- ./doc/gallery git diff git log -n 10 --oneline - tree doc -D -h + tree doc/gallery -D -h ls elif [ "$REMOVEDPROJECTS" != "[]" ]; then @@ -272,29 +272,27 @@ jobs: echo "Parse projects to remove them" items=$(echo $REMOVEDPROJECTS | jq -c -r '.[]') for item in ${items[@]}; do - echo "Removing doc/$item..." - rm -rf doc/$item/ - echo "Removed doc/$item" - git add './doc/$item' + echo "Removing doc/gallery/$item..." + rm -rf doc/gallery/$item/ + echo "Removed doc/gallery/$item" + git add './doc/gallery/$item' done git diff # This isn't meant to be pushed, it's just for this docs build git commit -m "Remove $REMOVEDPROJECTS" git checkout ${{ inputs.branch }} - # Checkout only the /doc folder than contains the evaluated artefacts - git checkout evaluated -- ./doc + # Checkout only the /doc/gallery folder than contains the evaluated artefacts + git checkout evaluated -- ./doc/gallery git diff git log -n 10 --oneline - tree doc -D -h + tree doc/gallery -D -h ls fi - name: archive projects run: | doit doc_archive_projects - - name: move thumbnails - run: doit doc_move_thumbnails - - name: make assets - run: doit doc_move_assets + - name: move content + run: doit doc_move_content - name: "temp: remove non evaluated projects" run: doit doc_remove_not_evaluated - name: build dev website @@ -310,18 +308,18 @@ jobs: # ZIP and upload the built site: # Only when called from pr_flow.yml. Done as multiple PRs can update the dev website # concurrently, this offers a way to download the site and see it locally. - - name: zip built site + - name: tar built site if: inputs.type == 'workflow_call' - run: zip -r builtdocs.zip builtdocs/ + run: tar czf builtdocs.tar.gz builtdocs/ - uses: actions/upload-artifact@v3 if: inputs.type == 'workflow_call' with: name: website - path: builtdocs.zip + path: builtdocs.tar.gz retention-days: 3 - name: delete zip if: inputs.type == 'workflow_call' - run: rm builtdocs.zip + run: rm builtdocs.tar.gz - name: Deploy dev # workflow_call, by pr_flow.yml # workflow_dispatch and dev target diff --git a/.gitignore b/.gitignore index 1dd7e89a6..6e76e6d0a 100644 --- a/.gitignore +++ b/.gitignore @@ -29,15 +29,12 @@ __pycache__/ # OSX *.DS_Store -# nbsite -doc/index.rst - # Ignore output of get_evaluated_doc -doc/*/ +doc/gallery/* !doc/_static/ !doc/_templates/ -# Ignore output of doc_move_assets +# Ignore output of doc_archive_projects assets/ # but don't ignore the projname/assets/ !*/assets/ diff --git a/_extensions/gallery.py b/_extensions/gallery.py index cb33ca9ad..9a4446d10 100644 --- a/_extensions/gallery.py +++ b/_extensions/gallery.py @@ -1,6 +1,9 @@ import glob import os +from pathlib import Path + +import nbformat import sphinx.util logger = sphinx.util.logging.getLogger('gallery-extension') @@ -26,7 +29,6 @@ 'default_extensions': ['*.ipynb'], 'examples_dir': os.path.join('..', 'examples'), 'labels_dir': 'labels', - 'alternative_toctree': [], 'github_project': None, 'intro': 'Sample intro', 'title': 'A sample gallery title', @@ -62,47 +64,27 @@ def sort_index_first(files): return sorted_files -def generate_file_rst( - gallery_conf, src_dir, dest_dir, page, section, skip, prolog, -): - proj = gallery_conf['github_project'] - examples_dir = gallery_conf['examples_dir'] - skip_execute = gallery_conf.get('skip_execute', []) - extensions = gallery_conf['default_extensions'] - - components = [examples_dir.split(os.path.sep)[-1], page] - components.append(section) - - files = [] - for extension in extensions: - files += glob.glob(os.path.join(src_dir, extension)) - - for f in files: - if isinstance(skip, list) and os.path.basename(f) in skip: +def generate_project_toctree(files): + toctree = '.. toctree::\n' + toctree += ' :hidden:\n\n' + for file in files: + name = Path(file).stem + if name == 'index': continue - extension = f.split('.')[-1] - basename = os.path.basename(f) - rel_path = os.path.relpath(os.path.join(src_dir, basename), dest_dir) - rst_path = os.path.join(dest_dir, basename[:-len(extension)].replace(' ', '_') + 'rst') - - if os.path.isfile(rst_path): - with open(rst_path) as existing: - if not 'Originally generated by gallery-extension' in existing.read(): - continue - - with open(rst_path, 'w') as rst_file: - if prolog: - # Used by examples.holoviz.org to link to the viewed notebook - if '/notebooks/{template_notebook_filename}' in prolog: - prolog = prolog.format( - template_notebook_filename=basename, - ) - rst_file.write(prolog) + toctree += f' {name}\n' + return toctree - rst_file.write(".. notebook:: %s %s" % (proj, rel_path)) - if (isinstance(skip, bool) and skip) or any(basename.strip().endswith(skipped) for skipped in skip_execute): - rst_file.write('\n :skip_execute: True\n') +def insert_toctree(nb_path, toctree): + nb = nbformat.read(nb_path, as_version=4) + last_cell = nb['cells'][-1] + toctree = "```{eval-rst}\n" + toctree + "\n```" + toctree_cell = nbformat.v4.new_markdown_cell(source=toctree) + if "```{eval-rst}" in last_cell['source']: + nb['cells'][-1] = toctree_cell + else: + nb['cells'].append(toctree_cell) + nbformat.write(nb, nb_path, version=nbformat.NO_CONVERT) def generate_gallery(app): @@ -113,7 +95,6 @@ def generate_gallery(app): # Get config gallery_conf = app.config.gallery_conf extensions = gallery_conf['default_extensions'] - alternative_toctree = gallery_conf['alternative_toctree'] gallery_path = gallery_conf['path'] @@ -159,22 +140,13 @@ def generate_gallery(app): description = section.get('description', None) labels = section.get('labels', []) skip = section.get('skip', []) - prolog = section.get('prolog', '') - path_components = [gallery_path] - path_components.append(section_path) - - path = os.path.join(examples_dir, *path_components) - dest_dir = os.path.join(doc_dir, *path_components) - try: - os.makedirs(dest_dir) - except: - pass + dest_dir = os.path.join(doc_dir, gallery_path, section_path) # Collect examples files = [] for extension in extensions: - files += glob.glob(os.path.join(path, extension)) + files += glob.glob(os.path.join(dest_dir, extension)) if skip: files = [f for f in files if os.path.basename(f) not in skip] @@ -197,17 +169,6 @@ def generate_gallery(app): basenames = [] for f in files: - # Generate the notebook rst - generate_file_rst( - gallery_conf=app.config.gallery_conf, - src_dir=path, - dest_dir=dest_dir, - page=gallery_path, - section=section_path, - skip=skip, - prolog=prolog - ) - extension = f.split('.')[-1] basename = os.path.basename(f)[:-(len(extension)+1)] basenames.append(basename) @@ -248,32 +209,21 @@ def generate_gallery(app): ) gallery_rst += this_entry - if not alternative_toctree and len(files) > 1: - # Append a toctree to the section index.rst file - rst_path = os.path.join(dest_dir, 'index.rst') - assert os.path.isfile(rst_path), f'index.rst file not found at {rst_path}' - - with open(rst_path, 'a') as rst_file: - rst_file.write('\n\n.. toctree::\n :hidden:\n\n') - for basename_ in basenames: - target = 'self' if basename_ == 'index' else basename_ - rst_file.write(f' {target}\n') + if len(files) > 1: + index_nb = next(file for file in files if file.endswith('index.ipynb')) + project_toctree = generate_project_toctree(files) + insert_toctree(index_nb, project_toctree) # Gallery toctree: just put the index file or the only notebook available. target = 'index' if 'index' in basenames else basenames[0] toctree_entries.append(f'{section_title} <{section_path}/{target}>') # Add gallery toctree - if not alternative_toctree: - assert toctree_entries, 'Empty toctree entries.' - toctree_rst = '.. toctree::\n :hidden:\n\n' - for toctree_entry in toctree_entries: - toctree_entry = 'self' if toctree_entry == 'index' else toctree_entry - toctree_rst += f' {toctree_entry}\n' - else: - toctree_rst = '.. toctree::\n :hidden:\n\n' - for toctree_entry in alternative_toctree: - toctree_rst += f' {toctree_entry}\n' + assert toctree_entries, 'Empty toctree entries.' + toctree_rst = '.. toctree::\n :hidden:\n\n' + for toctree_entry in toctree_entries: + toctree_entry = 'self' if toctree_entry == 'index' else toctree_entry + toctree_rst += f' {toctree_entry}\n' gallery_rst += toctree_rst diff --git a/_extensions/nbheader.py b/_extensions/nbheader.py new file mode 100644 index 000000000..46b519efd --- /dev/null +++ b/_extensions/nbheader.py @@ -0,0 +1,56 @@ +import glob +import os + +from pathlib import Path + +import nbformat +import sphinx.util + +logger = sphinx.util.logging.getLogger('nbheader-extension') + + +def insert_prolog(nb_path, prolog): + nb = nbformat.read(nb_path, as_version=4) + first_cell = nb['cells'][0] + prolog = "```{eval-rst}\n" + prolog + "\n```" + prolog_cell = nbformat.v4.new_markdown_cell(source=prolog) + if "```{eval-rst}" in first_cell['source']: + nb['cells'][0] = prolog_cell + else: + nb['cells'].insert(0, prolog_cell) + nbformat.write(nb, nb_path, version=nbformat.NO_CONVERT) + + +def add_nbheader(app): + """ + This if for now re-using gallery_conf from the gallery extensions. + Configurations could be decoupled if need be. + """ + + logger.info('Adding notebook headers...', color='white') + + # Get config + gallery_conf = app.config.gallery_conf + sections = gallery_conf['sections'] + doc_dir = Path(app.builder.srcdir) + gallery_path = doc_dir / gallery_conf['path'] + for section in sections: + prolog = section['prolog'] + project_path = gallery_path / section['path'] + nb_files = glob.glob(os.path.join(project_path, '*.ipynb')) + for nb_file in nb_files: + nb_file = Path(nb_file) + # Used by examples.holoviz.org to link to the viewed notebook + nb_prolog = prolog + if '/notebooks/{template_notebook_filename}' in prolog: + nb_prolog = prolog.format( + template_notebook_filename=nb_file.name, + ) + insert_prolog(nb_file, nb_prolog) + + +def setup(app): + app.connect('builder-inited', add_nbheader) + metadata = {'parallel_read_safe': True, + 'version': '0.0.1'} + return metadata diff --git a/doc/_static/css/custom.css b/doc/_static/css/custom.css new file mode 100644 index 000000000..ebb2ad06f --- /dev/null +++ b/doc/_static/css/custom.css @@ -0,0 +1,22 @@ +/* +Custom CSS for the examples gallery website +*/ + +/* Resize the thumbnails to be 200px high and keep their aspect ratio */ +.extension-gallery-img { + width: 100%; + height: 200px; + object-fit: contain; +} + + +/* Paragraphs in the metadata header displayed above each notebook +had their margin-bottom set by the PyData Sphinx Theme. + +This sets it to 0 and is inspired by what Sphinx-Design does for its article-info +directive. Sphinx-Design is what is used on examples to implement the header as a grid. +https://github.com/executablebooks/sphinx-design/blob/cca2cfb1b4c0a55ecf6661889c52ea320d42f58f/sphinx_design/article_info.py#L46 +*/ +div.nbsite-metadata > p { + margin-bottom: 0 !important; +} diff --git a/doc/_static/site.css b/doc/_static/site.css deleted file mode 100644 index 260d771e3..000000000 --- a/doc/_static/site.css +++ /dev/null @@ -1,150 +0,0 @@ -/* -Custom CSS for the examples gallery website -*/ - -/* Resize the thumbnails to be 200px high and keep their aspect ratio */ -.extension-gallery-img { - width: 100%; - height: 200px; - object-fit: cover; -} - -/* Bunch of CSS copied from Panel :( */ -:root[data-theme="light"] { - --pst-color-primary: rgb(47, 47, 47); - --pst-color-link: rgb(0, 170, 65); -} - -.nav-link { - white-space: nowrap; -} - -.showcase-table { - border-spacing: 15px -} - -.showcase-table td { - border: 0px; - vertical-align: top; -} - -.pl-md-5, .px-md-5 { - padding-left: 1rem !important; -} - -.pt-md-5, .py-md-5 { - padding-top: 1rem !important; -} - -.cell_output { - padding-left: 0; -} - -@media (min-width: 1200px) { - .container, .container-lg, .container-md, .container-sm, .container-xl { -max-width: 1600px; - } -} - -#scroller-right { - max-width: 14%; -} - -@media (max-width: 1400px) { - #scroller-right { -position: relative; -right: unset; -top: unset; -max-width: 100%; -transform: unset; - } -} - -#navbar-icon-links i.fa-github-square:before { - color: white; -} - -.fa-discourse:before { - color: white; -} - -button.toggle-button { - display: none; -} - -.toggle-hidden:not(.admonition) { - height: 0; -} - -.tag_hide-input { - margin-bottom: 0 !important; -} - -details.hide.above-input { - display: none; -} - -.toggle-hidden + .cell_output { - margin-top: 0 !important; -} - -dl.field-list { - display: none -} - -/* Improve styling */ - -div.cell div.cell_input { - border: none; -} - -.highlight { - border-radius: 4px; -} - -html[data-theme="light"] .highlight { - background-color: #263238; - color: #f8f8f2; -} - -pre[id^='codecell'] { - background-color: unset; - border: none; - border-radius: 0.5em; - color: #f8f8f2; - box-shadow: none; - padding: 1.5em; -} - -button.copybtn { - background-color: #263238; -} - -button.copybtn:hover { - background-color: #263238; - color: #f8f8f2; -} - -.highlight button.copybtn:hover { - background: none -} - -.o-tooltip--left:after { - background: none; -} - -ul.current.nav.bd-sidenav { - padding: 0; -} - - -/* Paragraphs in the metadata header displayed above each notebook -had their margin-bottom set by the PyData Sphinx Theme. - -This sets it to 0 and is inspired by what Sphinx-Design does for its article-info -directive. Sphinx-Design is what is used on examples to implement the header as a grid. -https://github.com/executablebooks/sphinx-design/blob/cca2cfb1b4c0a55ecf6661889c52ea320d42f58f/sphinx_design/article_info.py#L46 -*/ -div.nbsite-metadata > p { - margin-bottom: 0 !important; -} diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html deleted file mode 100644 index 6e863380b..000000000 --- a/doc/_templates/layout.html +++ /dev/null @@ -1,52 +0,0 @@ -{%- extends "!layout.html" %} - -{% block docs_navbar %} - -{% endblock %} - -{% block docs_sidebar %} -{% if sidebars %} - -
-{% else %} - -{% endif %} -{% endblock %} - -{% block docs_toc %} -