ci: build sphinx docs

.github/labeler.yml

@@ -1,8 +1,57 @@
-
-
 # Add labels to any any pull request with changes to the specified paths
 
 # Pull requests that update GitHub Actions code. If you navigate to the folder, you will have a README of what it does
 📚 Documentation:
 - changed-files:
   - any-glob-to-any-file: 'docs/how-to-install.md'
+  - any-glob-to-any-file: 'docs/sphinx/_static/.empty'
+  - any-glob-to-any-file: 'docs/sphinx/_templates/.empty'
+  - any-glob-to-any-file: 'docs/sphinx/api.rst'
+  - any-glob-to-any-file: 'docs/sphinx/cli.rst'
+  - any-glob-to-any-file: 'docs/sphinx/conf.py'
+  - any-glob-to-any-file: 'docs/sphinx/config.rst'
+  - any-glob-to-any-file: 'docs/sphinx/index.rst'
+  - any-glob-to-any-file: 'docs/sphinx/misc.rst'
+  - any-glob-to-any-file: 'docs/sphinx/searching.rst'
+  - any-glob-to-any-file: 'docs/sphinx/types.rst'
+  - any-glob-to-any-file: 'docs/users/ref-python.md'
+
+API:
+- changed-files:
+  - any-glob-to-any-file: 'docs/sphinx/api.rst'
+
+Build Scripts:
+- changed-files:
+  - any-glob-to-any-file: 'scripts/Dockerfile.sphinx'
+  - any-glob-to-any-file: 'scripts/build_sphinx.sh'
+  - any-glob-to-any-file: 'scripts/generate_doc.sh'
+  - any-glob-to-any-file: 'scripts/sphinx/Makefile'
+
+🐳 docker:
+- changed-files:
+  - any-glob-to-any-file: 'Dockerfile'
+  - any-glob-to-any-file: 'docker-compose.yml'
+
+Indexation:
+- changed-files:
+  - any-glob-to-any-file: 'app/indexing.py'
+  - any-glob-to-any-file: 'docs/sphinx/index.rst'
+
+Taxonomies:
+- changed-files:
+  - any-glob-to-any-file: 'app/taxonomy.py'
+  - any-glob-to-any-file: 'app/taxonomy_es.py'
+
+CLI:
+- changed-files:
+  - any-glob-to-any-file: 'app/cli/main.py'
+  - any-glob-to-any-file: 'docs/sphinx/cli.rst'
+
+Config:
+- changed-files:
+  - any-glob-to-any-file: 'docs/sphinx/config.rst'
+
+#  - any-glob-to-any-file: 'app/query.py'
+#  - any-glob-to-any-file: 'app/utils/analyzers.py'
+#  - any-glob-to-any-file: 'app/utils/download.py'
+#  - any-glob-to-any-file: 'app/validations.py'

Dockerfile

@@ -19,7 +19,7 @@ ENV PYTHONUNBUFFERED=1 \
     PYSETUP_PATH="/opt/pysetup" \
     VENV_PATH="/opt/pysetup/.venv" \
     POETRY_HOME="/opt/poetry" \
-    POETRY_VERSION=1.6.1 \
+    POETRY_VERSION=1.8.3 \
     POETRY_VIRTUALENVS_IN_PROJECT=true \
     POETRY_NO_INTERACTION=1
     ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"

app/cli/main.py

@@ -1,3 +1,8 @@
+"""This module provides different commands to help
+setting up search-a-licious
+or doing maintenance operations.
+"""
+
 from pathlib import Path
 from typing import Optional
 
@@ -8,6 +13,13 @@
 cli = typer.Typer()
 
 
+INDEX_ID_HELP = (
+    "Each index has its own configuration in the configuration file, "
+    "and the ID is used to know which index to use. "
+    "If there is only one index, this option is not needed."
+)
+
+
 def _get_index_config(
     config_path: Optional[Path], index_id: Optional[str]
 ) -> tuple[str, "app.config.IndexConfig"]:
@@ -57,12 +69,16 @@ def import_data(
     ),
     index_id: Optional[str] = typer.Option(
         default=None,
-        help="Each index has its own configuration in the configuration file, "
-        "and the ID is used to know which index to use. "
-        "If there is only one index, this option is not needed.",
+        help=INDEX_ID_HELP,
     ),
 ):
-    """Import data into Elasticsearch."""
+    """Import data into Elasticsearch.
+
+    This command is used to initialize or refresh your index with data.
+
+    File must contains one JSON document per line,
+    each document must have same format as a document returned by the API.
+    """
     import time
 
     from app._import import run_full_import
@@ -95,12 +111,13 @@ def import_taxonomies(
     ),
     index_id: Optional[str] = typer.Option(
         default=None,
-        help="Each index has its own configuration in the configuration file, "
-        "and the ID is used to know which index to use. "
-        "If there is only one index, this option is not needed.",
+        help=INDEX_ID_HELP,
     ),
 ):
-    """Import taxonomies into Elasticsearch."""
+    """Import taxonomies into Elasticsearch.
+
+    It get taxonomies json files as specified in the configuration file.
+    """
     import time
 
     from app._import import perform_taxonomy_import
@@ -127,11 +144,13 @@ def sync_scripts(
     ),
     index_id: Optional[str] = typer.Option(
         default=None,
-        help="Each index has its own configuration in the configuration file, "
-        "and the ID is used to know which index to use. "
-        "If there is only one index, this option is not needed.",
+        help=INDEX_ID_HELP,
     ),
 ):
+    """Synchronize scripts defined in configuration with Elasticsearch.
+
+    This command must be run after adding, modifying or removing scripts.
+    """
     from app import es_scripts
     from app.utils import connection, get_logger
 
@@ -155,7 +174,13 @@ def run_update_daemon(
     ),
 ):
     """Run the daemon responsible for listening to document updates from Redis
-    Stream and updating the Elasticsearch index."""
+    Stream and updating the Elasticsearch index.
+
+    This command must be run in a separate process to the api server.
+
+    It is optional but enables having an always up-to-date index,
+    for applications where data changes.
+    """
     from typing import cast
 
     from app import config
@@ -182,9 +207,10 @@ def export_openapi(
         exists=None,
         file_okay=True,
         dir_okay=False,
-        help="Path of target_path the YAML data file",
+        help="Path of target_path the YAML or JSON data file",
     )
 ):
+    """Export OpenAPI specification to a file."""
     import json
 
     import yaml

app/indexing.py

@@ -189,12 +189,14 @@ def process_taxonomy_field(
 
     Generates a dict ready to be indexed by Elasticsearch, with a subfield for
     each language. Two other subfields are added:
+
     - `original`: the original value of the field. For example, if the field
-        name is `categories` and `categories` already exist in the document,
-        we will save its value in the `original` subfield. This subfield is
-        only added if the field is present in the input data.
+       name is `categories` and `categories` already exist in the document,
+       we will save its value in the `original` subfield. This subfield is
+       only added if the field is present in the input data.
+
     - `other`: the value of the field for languages that are not supported by
-        the project (no elasticsearch specific analyzers)
+       the project (no elasticsearch specific analyzers)
 
     :param data: input data, as a dict
     :param field: the field config
@@ -260,7 +262,7 @@ def process_taxonomy_field(
 
 
 class DocumentProcessor:
-    """`DocumentProcessor`s are responsible of converting an item to index
+    """`DocumentProcessor` is responsible of converting an item to index
     into a dict that is ready to be indexed by Elasticsearch.
     """
 

app/query.py

@@ -185,7 +185,7 @@ def compute_facets_filters(q: QueryAnalysis) -> QueryAnalysis:
     that can be done at a later stage
 
     :return: a new QueryAnalysis with facets_filters attribute
-    as a dictionary of field names and list of values to filter on
+       as a dictionary of field names and list of values to filter on
     """
     if q.luqum_tree is None:
         return q

app/taxonomy.py

@@ -1,3 +1,8 @@
+"""Fetching taxonomies files and representing them
+
+See also :py:mod:`app.taxonomy_es`
+"""
+
 from pathlib import Path
 from typing import Any, Dict, Iterable, List, Optional, Set, Union
 
@@ -212,7 +217,7 @@ def is_parent_of_any(
         :param item: The item to compare
         :param candidates: A list of candidates
         :param raises: if True, raises a ValueError if item is not in the
-        taxonomy, defaults to True.
+            taxonomy, defaults to True.
         """
         node: TaxonomyNode = self[item]
 

app/taxonomy_es.py

@@ -1,4 +1,7 @@
-"""Operations on taxonomies in ElasticSearch"""
+"""Operations on taxonomies in ElasticSearch
+
+See also :py:mod:`app.taxonomy`
+"""
 
 from elasticsearch_dsl import Search
 from elasticsearch_dsl.query import Q

app/utils/analyzers.py

@@ -1,5 +1,8 @@
+"""Defines some analyzers for the elesaticsearch fields."""
+
 from elasticsearch_dsl import analyzer
 
+#: An analyzer for the autocomplete field
 AUTOCOMPLETE_ANALYZERS = {
     "fr": analyzer(
         "autocomplete_fr", tokenizer="standard", filter=["lowercase", "asciifolding"]

app/utils/download.py

@@ -1,3 +1,8 @@
+"""Various utilities for downloading files
+
+Used for example for downloading taxonomies
+"""
+
 import json
 import shutil
 import time

app/validations.py

@@ -49,7 +49,7 @@ def check_fields_are_numeric(
     """
     * If field exists in global_config, check that it's numeric.
     * If field looks like x.y and x.y does not exist in global config,
-    check that x is an object field
+      check that x is an object field
     """
     errors: list[str] = []
     if values is None:

docs/sphinx/api.rst

@@ -0,0 +1,8 @@
+API
+###
+
+API Module
+==========
+
+.. automodule:: app.api
+   :members:

docs/sphinx/cli.rst

@@ -0,0 +1,19 @@
+Command Line Interface
+######################
+
+CLI commands
+============
+
+.. typer:: app.cli.main:cli
+    :prog: python3 -m app
+    :width: 80
+    :show-nested:
+    :make-sections:
+
+
+
+CLI Implementation
+==================
+
+.. automodule:: app.cli.main
+    :members:

docs/sphinx/conf.py

@@ -0,0 +1,56 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = "Search-a-licious"
+copyright = "2024, Open Food Facts"
+author = "Open Food Facts"
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.linkcode",
+    "alabaster",
+    "sphinxcontrib.autodoc_pydantic",
+    "sphinxcontrib.typer",
+]
+
+templates_path = ["_templates"]
+exclude_patterns: list[str] = []
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = "alabaster"
+html_static_path = ["_static"]
+html_theme_options = {
+    "github_user": "openfoodfacts",
+    "github_repo": "search-a-licious",
+    "github_banner": True,
+    "extra_nav_links": {
+        "Back to main doc": "/search-a-licious",
+    },
+}
+
+# -- Options for autodoc pydantic --------------------------------------------
+autodoc_pydantic_model_show_json = True
+autodoc_pydantic_settings_show_json = False
+
+
+def linkcode_resolve(domain, info):
+    """Getting url for source file for sphinx.ext.linkcode"""
+    if domain != "py":
+        return None
+    if not info["module"]:
+        return None
+    filename = info["module"].replace(".", "/")
+    return (
+        "https://github.com/openfoodfacts/search-a-licious/tree/main/%s.py" % filename
+    )

docs/sphinx/config.rst

@@ -0,0 +1,8 @@
+Configuration
+#############
+
+Config Module
+=============
+
+.. automodule:: app.config
+   :members:

docs/sphinx/index.rst

@@ -0,0 +1,34 @@
+.. Search-a-licious documentation master file, created by
+   sphinx-quickstart on Tue Jul 16 09:14:56 2024.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Search-a-licious code documentation
+===================================
+
+This documents the backend code of search-alicious.
+
+For `general documentation about search-a-licious Follow this link`__
+
+.. note: this only work on github pages
+.. __: /search-a-licious
+
+
+Contents:
+
+.. toctree::
+   :maxdepth: 2
+
+   api
+   config
+   cli
+   searching
+   types
+   misc
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`