From 66ee9311ad235cc8c01347006422f3e1729b5a74 Mon Sep 17 00:00:00 2001 From: Alex Garel Date: Tue, 16 Jul 2024 18:04:24 +0200 Subject: [PATCH 1/7] ci: build sphinx docs --- Dockerfile | 2 +- app/indexing.py | 12 ++--- app/query.py | 2 +- app/taxonomy.py | 7 ++- app/taxonomy_es.py | 5 ++- app/utils/analyzers.py | 3 ++ app/utils/constants.py | 0 app/utils/download.py | 5 +++ app/validations.py | 2 +- docs/sphinx/api.rst | 8 ++++ docs/sphinx/conf.py | 47 ++++++++++++++++++++ docs/sphinx/config.rst | 8 ++++ docs/sphinx/index.rst | 27 +++++++++++ docs/sphinx/misc.rst | 16 +++++++ docs/sphinx/searching.rst | 59 ++++++++++++++++++++++++ docs/sphinx/types.rst | 22 +++++++++ docs/users/explain-scripts.md | 84 +++++++++++++++++++++++++++++++++++ docs/users/ref-python.md | 5 +++ scripts/Dockerfile.sphinx | 24 ++++++++++ scripts/build_sphinx.sh | 28 ++++++++++++ scripts/generate_doc.sh | 3 ++ scripts/sphinx/Makefile | 20 +++++++++ scripts/sphinx/make.bat | 35 +++++++++++++++ 23 files changed, 414 insertions(+), 10 deletions(-) delete mode 100644 app/utils/constants.py create mode 100644 docs/sphinx/api.rst create mode 100644 docs/sphinx/conf.py create mode 100644 docs/sphinx/config.rst create mode 100644 docs/sphinx/index.rst create mode 100644 docs/sphinx/misc.rst create mode 100644 docs/sphinx/searching.rst create mode 100644 docs/sphinx/types.rst create mode 100644 docs/users/ref-python.md create mode 100644 scripts/Dockerfile.sphinx create mode 100755 scripts/build_sphinx.sh create mode 100644 scripts/sphinx/Makefile create mode 100644 scripts/sphinx/make.bat diff --git a/Dockerfile b/Dockerfile index 4b93cdc5..ab7830f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ ENV PYTHONUNBUFFERED=1 \ PYSETUP_PATH="/opt/pysetup" \ VENV_PATH="/opt/pysetup/.venv" \ POETRY_HOME="/opt/poetry" \ - POETRY_VERSION=1.6.1 \ + POETRY_VERSION=1.8.3 \ POETRY_VIRTUALENVS_IN_PROJECT=true \ POETRY_NO_INTERACTION=1 ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH" diff --git a/app/indexing.py b/app/indexing.py index 3eec8c76..5d28edf8 100644 --- a/app/indexing.py +++ b/app/indexing.py @@ -189,12 +189,14 @@ def process_taxonomy_field( Generates a dict ready to be indexed by Elasticsearch, with a subfield for each language. Two other subfields are added: + - `original`: the original value of the field. For example, if the field - name is `categories` and `categories` already exist in the document, - we will save its value in the `original` subfield. This subfield is - only added if the field is present in the input data. + name is `categories` and `categories` already exist in the document, + we will save its value in the `original` subfield. This subfield is + only added if the field is present in the input data. + - `other`: the value of the field for languages that are not supported by - the project (no elasticsearch specific analyzers) + the project (no elasticsearch specific analyzers) :param data: input data, as a dict :param field: the field config @@ -260,7 +262,7 @@ def process_taxonomy_field( class DocumentProcessor: - """`DocumentProcessor`s are responsible of converting an item to index + """`DocumentProcessor` is responsible of converting an item to index into a dict that is ready to be indexed by Elasticsearch. """ diff --git a/app/query.py b/app/query.py index 37a699f5..66fa7968 100644 --- a/app/query.py +++ b/app/query.py @@ -185,7 +185,7 @@ def compute_facets_filters(q: QueryAnalysis) -> QueryAnalysis: that can be done at a later stage :return: a new QueryAnalysis with facets_filters attribute - as a dictionary of field names and list of values to filter on + as a dictionary of field names and list of values to filter on """ if q.luqum_tree is None: return q diff --git a/app/taxonomy.py b/app/taxonomy.py index c85ade20..6b7b6d05 100644 --- a/app/taxonomy.py +++ b/app/taxonomy.py @@ -1,3 +1,8 @@ +"""Fetching taxonomies files and representing them + +See also :py:mod:`app.taxonomy_es` +""" + from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Set, Union @@ -212,7 +217,7 @@ def is_parent_of_any( :param item: The item to compare :param candidates: A list of candidates :param raises: if True, raises a ValueError if item is not in the - taxonomy, defaults to True. + taxonomy, defaults to True. """ node: TaxonomyNode = self[item] diff --git a/app/taxonomy_es.py b/app/taxonomy_es.py index fb02d7ef..24361c6e 100644 --- a/app/taxonomy_es.py +++ b/app/taxonomy_es.py @@ -1,4 +1,7 @@ -"""Operations on taxonomies in ElasticSearch""" +"""Operations on taxonomies in ElasticSearch + +See also :py:mod:`app.taxonomy` +""" from elasticsearch_dsl import Search from elasticsearch_dsl.query import Q diff --git a/app/utils/analyzers.py b/app/utils/analyzers.py index 943d5d26..f091daba 100644 --- a/app/utils/analyzers.py +++ b/app/utils/analyzers.py @@ -1,5 +1,8 @@ +"""Defines some analyzers for the elesaticsearch fields.""" + from elasticsearch_dsl import analyzer +#: An analyzer for the autocomplete field AUTOCOMPLETE_ANALYZERS = { "fr": analyzer( "autocomplete_fr", tokenizer="standard", filter=["lowercase", "asciifolding"] diff --git a/app/utils/constants.py b/app/utils/constants.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/utils/download.py b/app/utils/download.py index 34466821..ae15fbd4 100644 --- a/app/utils/download.py +++ b/app/utils/download.py @@ -1,3 +1,8 @@ +"""Various utilities for downloading files + +Used for example for downloading taxonomies +""" + import json import shutil import time diff --git a/app/validations.py b/app/validations.py index 88f219bf..b3b764d7 100644 --- a/app/validations.py +++ b/app/validations.py @@ -49,7 +49,7 @@ def check_fields_are_numeric( """ * If field exists in global_config, check that it's numeric. * If field looks like x.y and x.y does not exist in global config, - check that x is an object field + check that x is an object field """ errors: list[str] = [] if values is None: diff --git a/docs/sphinx/api.rst b/docs/sphinx/api.rst new file mode 100644 index 00000000..f8025b46 --- /dev/null +++ b/docs/sphinx/api.rst @@ -0,0 +1,8 @@ +API +### + +API Module +========== + +.. automodule:: app.api + :members: diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py new file mode 100644 index 00000000..4d7d3b56 --- /dev/null +++ b/docs/sphinx/conf.py @@ -0,0 +1,47 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "Search-a-licious" +copyright = "2024, Open Food Facts" +author = "Open Food Facts" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.linkcode", + "alabaster", + "sphinxcontrib.autodoc_pydantic", +] + +templates_path = ["_templates"] +exclude_patterns: list[str] = [] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "alabaster" +html_static_path = ["_static"] + +# -- Options for autodoc pydantic -------------------------------------------- +autodoc_pydantic_model_show_json = True +autodoc_pydantic_settings_show_json = False + + +def linkcode_resolve(domain, info): + """Getting url for source file for sphinx.ext.linkcode""" + if domain != "py": + return None + if not info["module"]: + return None + filename = info["module"].replace(".", "/") + return ( + "https://github.com/openfoodfacts/search-a-licious/tree/main/%s.py" % filename + ) diff --git a/docs/sphinx/config.rst b/docs/sphinx/config.rst new file mode 100644 index 00000000..3ba09a8c --- /dev/null +++ b/docs/sphinx/config.rst @@ -0,0 +1,8 @@ +Configuration +############# + +Config Module +============= + +.. automodule:: app.config + :members: diff --git a/docs/sphinx/index.rst b/docs/sphinx/index.rst new file mode 100644 index 00000000..c3e1e2a0 --- /dev/null +++ b/docs/sphinx/index.rst @@ -0,0 +1,27 @@ +.. Search-a-licious documentation master file, created by + sphinx-quickstart on Tue Jul 16 09:14:56 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Search-a-licious code documentation +=================================== + +This documents the backend code of search-alicious. + +Contents: + +.. toctree:: + :maxdepth: 2 + + api + config + searching + types + misc + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` \ No newline at end of file diff --git a/docs/sphinx/misc.rst b/docs/sphinx/misc.rst new file mode 100644 index 00000000..46e13b03 --- /dev/null +++ b/docs/sphinx/misc.rst @@ -0,0 +1,16 @@ +Misc +#### + +Taxonomy +======== + +See also :py:mod:`app.taxonomy_es` + +.. automodule:: app.taxonomy + :members: + + +Health check +============ +.. automodule:: app.health + :members: diff --git a/docs/sphinx/searching.rst b/docs/sphinx/searching.rst new file mode 100644 index 00000000..681615e2 --- /dev/null +++ b/docs/sphinx/searching.rst @@ -0,0 +1,59 @@ +Search Modules +############## + +Indexing +======== + +.. automodule:: app.indexing + :members: + +Query +===== + +.. automodule:: app.query + :members: + +Search +====== + +.. automodule:: app.search + :members: + +Facets +====== + +.. automodule:: app.facets + :members: + +Charts +====== + +.. automodule:: app.charts + :members: + +ES Scripts +========== + +.. automodule:: app.es_scripts + :members: + +Taxonomy ES +=========== + +.. automodule:: app.taxonomy_es + :members: + + +Analyzers (Utils) +================= + +.. automodule:: app.utils.analyzers + :members: + + +Connection (Utils) +================== + +.. automodule:: app.utils.connection + :members: + diff --git a/docs/sphinx/types.rst b/docs/sphinx/types.rst new file mode 100644 index 00000000..c7836dd6 --- /dev/null +++ b/docs/sphinx/types.rst @@ -0,0 +1,22 @@ +Types +##### + +Main types +========== + +.. we need to exclude QueryAnalysis because autodoc pydantic doesn't support it, we add it below +.. automodule:: app._types + :members: + +.. autopydantic_model:: app._types.QueryAnalysis + :model-show-json: False + +.. autoclass:: app._types.QueryAnalysis + :members: + :no-index: + +Validations +=========== + +.. automodule:: app.validations + :members: diff --git a/docs/users/explain-scripts.md b/docs/users/explain-scripts.md index e69de29b..dfe5b12f 100644 --- a/docs/users/explain-scripts.md +++ b/docs/users/explain-scripts.md @@ -0,0 +1,84 @@ +# Explain scripts + +You can use scripts to sort results in your search requests. + +This leverage a possibility of Elasticsearch of [script based sorting](https://www.elastic.co/guide/en/elasticsearch/reference/current/sort-search-results.html#script-based-sorting). + +Using scripts needs the following steps: + +1. [declare the scripts than can be used in the configuration](#declare-the-script-in-the-configuration) +2. [import the scripts in Elasticsearch](#import-the-script-in-elasticsearch) +3. call the search API using the script name, and providing parameters + + +## Declare the script in the configuration + +You have to declare the scripts that can be used for sorting in your configuration. + +This has two advantages: +* this keeps the API call simple, by just refering to the script by name +* this is more secure as you are in full control of scripts that are allowed to be used. + +The scripts section can look something like this: +```yaml + scripts: + personal_score: # see 1 + # see https://www.elastic.co/guide/en/elasticsearch/painless/8.14/index.html + lang: painless # see 2 + # the script source, here a trivial example + # see 3 + source: |- + doc[params["preferred_field"]].size > 0 ? doc[params["preferred_field"]].value : (doc[params["secondary_field"]].size > 0 ? doc[params["secondary_field"]].value : 0) + # gives an example of parameters + # see 4 + params: + preferred_field: "field1" + secondary_field: "field2" + # more non editable parameters, can be easier than to declare constants in the script + # see 5 + static_params: + param1 : "foo" +``` + +Here: +1. we declare a script named `personal_score`, this is the name you will use in your API requests and/or web-components attributes + +2. we declare the language of the script, in this case `painless`, search-a-licious supports [painless](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting-painless.html) and [Lucene expressions](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting-expression.html) + +3. this is the source of the script. It can be a bit tedious to write those scripts. You can use the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting-painless.html) to get a better understanding of the language. + + In this example we are using a one liner, but your scripts can be far more complex. + +4. Parameters are a way to add inputs to the script. + You can declare them using an example. You can provide more complex structures, as allowed by JSON. + Those parameters will be given through the API requests + +5. static_params are parameters that are not allowed to change through the API. + It's mostly a way to declare constants in the script. + (hopefully more convenient than declaring them in the script) + +See [introduction to script in Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting-using.html) + +## Import the scripts in Elasticsearch + +Each time you change the configuration, you have to import the scripts in Elasticsearch. + +For this you only need to run the sync-scripts command. + +```bash +docker compose run --rm api python -m app sync-scripts +``` + + +## Using the script in the API + +Afet + +## Performance considerations + +When you use scripts for sorting, bare in mind that they needs to be executed on each document. + +Tests your results on your full dataset to make sure performances are not an issue. + +An heavy load on scripts sorting might affect other requests as well under an heavy load. + diff --git a/docs/users/ref-python.md b/docs/users/ref-python.md new file mode 100644 index 00000000..5e4570df --- /dev/null +++ b/docs/users/ref-python.md @@ -0,0 +1,5 @@ +# Ref: Python Code documentation 🡕 + +**Important:** do not edit this file directly, +it is autogenerated by `scripts/build_sphinx.sh` +using `docs/sphinx` content. \ No newline at end of file diff --git a/scripts/Dockerfile.sphinx b/scripts/Dockerfile.sphinx new file mode 100644 index 00000000..626b0fee --- /dev/null +++ b/scripts/Dockerfile.sphinx @@ -0,0 +1,24 @@ +# in your Dockerfile +FROM sphinxdoc/sphinx:latest + +ARG USER_UID=1000 +ARG USER_GID=1000 + +WORKDIR /docs +# add user with right id +RUN addgroup --gid $USER_GID user && adduser --uid $USER_UID --ingroup user --no-create-home --disabled-password --quiet user +# ensure directories and permisisons +RUN mkdir -p /docs/build && mkdir -p /docs/source && chown user:user /docs +# install poetry, and export dependencies as a requirement.txt +COPY poetry.lock pyproject.toml ./ +RUN apt update && apt install -y curl +RUN ( curl -sSL https://install.python-poetry.org | python3 - ) && \ + /root/.local/bin/poetry self add poetry-plugin-export && \ + /root/.local/bin/poetry export --output requirements.txt +# install those dependencies +RUN pip install -r requirements.txt +# install the autodoc-pydantic plugin for sphinx +RUN pip install autodoc_pydantic +USER user + + diff --git a/scripts/build_sphinx.sh b/scripts/build_sphinx.sh new file mode 100755 index 00000000..124732be --- /dev/null +++ b/scripts/build_sphinx.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Build sphinx documentation + +set -e + +# get group id to use it in the docker +GID=$(id -g) + +# create image +docker build --build-arg "USER_UID=$UID" --build-arg "USER_GID=$GID" --tag 'sphinx-builder' -f scripts/Dockerfile.sphinx . + +# ensure dest dir +mkdir -p gh_pages/sphinx + +# use image to generate documentation +docker run --rm --user user \ + -v $(pwd)/scripts/sphinx:/docs \ + -v $(pwd)/docs/sphinx:/docs/source \ + -v $(pwd)/app:/docs/app \ + -v $(pwd)/gh_pages/sphinx:/docs/build \ + -e PYTHONPATH=/docs/ \ + -e SPHINXOPTS="-W --keep-going" \ + sphinx-builder make html + +# move to the right place and cleanup +rm -rf gh_pages/users/ref-python || true +mv gh_pages/sphinx/html gh_pages/users/ref-python +rm -rf gh_pages/sphinx/ diff --git a/scripts/generate_doc.sh b/scripts/generate_doc.sh index f23ab619..43c4db39 100755 --- a/scripts/generate_doc.sh +++ b/scripts/generate_doc.sh @@ -29,4 +29,7 @@ mkdir -p gh_pages/users/ref-web-components/dist cp frontend/public/dist/custom-elements.json gh_pages/users/ref-web-components/dist/custom-elements.json sudo chown $UID -R gh_pages/users/ref-web-components +echo "Generate python code documentation using sphinx" +scripts/build_sphinx.sh + echo "To see your doc locally, run: python3 -m http.server -d gh_pages 8001" \ No newline at end of file diff --git a/scripts/sphinx/Makefile b/scripts/sphinx/Makefile new file mode 100644 index 00000000..d0c3cbf1 --- /dev/null +++ b/scripts/sphinx/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/scripts/sphinx/make.bat b/scripts/sphinx/make.bat new file mode 100644 index 00000000..747ffb7b --- /dev/null +++ b/scripts/sphinx/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd From d4d0f34ab89e239de02e1d44c145ad982741a80d Mon Sep 17 00:00:00 2001 From: Alex Garel Date: Tue, 16 Jul 2024 18:11:34 +0200 Subject: [PATCH 2/7] docs: remove explain script from this branch --- docs/users/explain-scripts.md | 84 ----------------------------------- 1 file changed, 84 deletions(-) diff --git a/docs/users/explain-scripts.md b/docs/users/explain-scripts.md index dfe5b12f..e69de29b 100644 --- a/docs/users/explain-scripts.md +++ b/docs/users/explain-scripts.md @@ -1,84 +0,0 @@ -# Explain scripts - -You can use scripts to sort results in your search requests. - -This leverage a possibility of Elasticsearch of [script based sorting](https://www.elastic.co/guide/en/elasticsearch/reference/current/sort-search-results.html#script-based-sorting). - -Using scripts needs the following steps: - -1. [declare the scripts than can be used in the configuration](#declare-the-script-in-the-configuration) -2. [import the scripts in Elasticsearch](#import-the-script-in-elasticsearch) -3. call the search API using the script name, and providing parameters - - -## Declare the script in the configuration - -You have to declare the scripts that can be used for sorting in your configuration. - -This has two advantages: -* this keeps the API call simple, by just refering to the script by name -* this is more secure as you are in full control of scripts that are allowed to be used. - -The scripts section can look something like this: -```yaml - scripts: - personal_score: # see 1 - # see https://www.elastic.co/guide/en/elasticsearch/painless/8.14/index.html - lang: painless # see 2 - # the script source, here a trivial example - # see 3 - source: |- - doc[params["preferred_field"]].size > 0 ? doc[params["preferred_field"]].value : (doc[params["secondary_field"]].size > 0 ? doc[params["secondary_field"]].value : 0) - # gives an example of parameters - # see 4 - params: - preferred_field: "field1" - secondary_field: "field2" - # more non editable parameters, can be easier than to declare constants in the script - # see 5 - static_params: - param1 : "foo" -``` - -Here: -1. we declare a script named `personal_score`, this is the name you will use in your API requests and/or web-components attributes - -2. we declare the language of the script, in this case `painless`, search-a-licious supports [painless](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting-painless.html) and [Lucene expressions](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting-expression.html) - -3. this is the source of the script. It can be a bit tedious to write those scripts. You can use the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting-painless.html) to get a better understanding of the language. - - In this example we are using a one liner, but your scripts can be far more complex. - -4. Parameters are a way to add inputs to the script. - You can declare them using an example. You can provide more complex structures, as allowed by JSON. - Those parameters will be given through the API requests - -5. static_params are parameters that are not allowed to change through the API. - It's mostly a way to declare constants in the script. - (hopefully more convenient than declaring them in the script) - -See [introduction to script in Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting-using.html) - -## Import the scripts in Elasticsearch - -Each time you change the configuration, you have to import the scripts in Elasticsearch. - -For this you only need to run the sync-scripts command. - -```bash -docker compose run --rm api python -m app sync-scripts -``` - - -## Using the script in the API - -Afet - -## Performance considerations - -When you use scripts for sorting, bare in mind that they needs to be executed on each document. - -Tests your results on your full dataset to make sure performances are not an issue. - -An heavy load on scripts sorting might affect other requests as well under an heavy load. - From ddebc60645a93146b97056df61673b2cebf37856 Mon Sep 17 00:00:00 2001 From: Alex Garel Date: Tue, 16 Jul 2024 18:52:06 +0200 Subject: [PATCH 3/7] =?UTF-8?q?docs:=C2=A0add=20cli=20documentation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/cli/main.py | 52 +++++++++++++++++++++++++++++---------- docs/sphinx/cli.rst | 19 ++++++++++++++ docs/sphinx/conf.py | 1 + docs/sphinx/index.rst | 1 + docs/sphinx/types.rst | 1 + scripts/Dockerfile.sphinx | 4 +-- scripts/build_sphinx.sh | 2 +- scripts/sphinx/make.bat | 35 -------------------------- 8 files changed, 64 insertions(+), 51 deletions(-) create mode 100644 docs/sphinx/cli.rst delete mode 100644 scripts/sphinx/make.bat diff --git a/app/cli/main.py b/app/cli/main.py index a32ebe92..1d15a67d 100644 --- a/app/cli/main.py +++ b/app/cli/main.py @@ -1,3 +1,8 @@ +"""This module provides different commands to help +setting up search-a-licious +or doing maintenance operations. +""" + from pathlib import Path from typing import Optional @@ -8,6 +13,13 @@ cli = typer.Typer() +INDEX_ID_HELP = ( + "Each index has its own configuration in the configuration file, " + "and the ID is used to know which index to use. " + "If there is only one index, this option is not needed." +) + + def _get_index_config( config_path: Optional[Path], index_id: Optional[str] ) -> tuple[str, "app.config.IndexConfig"]: @@ -57,12 +69,16 @@ def import_data( ), index_id: Optional[str] = typer.Option( default=None, - help="Each index has its own configuration in the configuration file, " - "and the ID is used to know which index to use. " - "If there is only one index, this option is not needed.", + help=INDEX_ID_HELP, ), ): - """Import data into Elasticsearch.""" + """Import data into Elasticsearch. + + This command is used to initialize or refresh your index with data. + + File must contains one JSON document per line, + each document must have same format as a document returned by the API. + """ import time from app._import import run_full_import @@ -95,12 +111,13 @@ def import_taxonomies( ), index_id: Optional[str] = typer.Option( default=None, - help="Each index has its own configuration in the configuration file, " - "and the ID is used to know which index to use. " - "If there is only one index, this option is not needed.", + help=INDEX_ID_HELP, ), ): - """Import taxonomies into Elasticsearch.""" + """Import taxonomies into Elasticsearch. + + It get taxonomies json files as specified in the configuration file. + """ import time from app._import import perform_taxonomy_import @@ -127,11 +144,13 @@ def sync_scripts( ), index_id: Optional[str] = typer.Option( default=None, - help="Each index has its own configuration in the configuration file, " - "and the ID is used to know which index to use. " - "If there is only one index, this option is not needed.", + help=INDEX_ID_HELP, ), ): + """Synchronize scripts defined in configuration with Elasticsearch. + + This command must be run after adding, modifying or removing scripts. + """ from app import es_scripts from app.utils import connection, get_logger @@ -155,7 +174,13 @@ def run_update_daemon( ), ): """Run the daemon responsible for listening to document updates from Redis - Stream and updating the Elasticsearch index.""" + Stream and updating the Elasticsearch index. + + This command must be run in a separate process to the api server. + + It is optional but enables having an always up-to-date index, + for applications where data changes. + """ from typing import cast from app import config @@ -182,9 +207,10 @@ def export_openapi( exists=None, file_okay=True, dir_okay=False, - help="Path of target_path the YAML data file", + help="Path of target_path the YAML or JSON data file", ) ): + """Export OpenAPI specification to a file.""" import json import yaml diff --git a/docs/sphinx/cli.rst b/docs/sphinx/cli.rst new file mode 100644 index 00000000..f3570eed --- /dev/null +++ b/docs/sphinx/cli.rst @@ -0,0 +1,19 @@ +Command Line Interface +###################### + +CLI commands +============ + +.. typer:: app.cli.main:cli + :prog: python3 -m app + :width: 80 + :show-nested: + :make-sections: + + + +CLI Implementation +================== + +.. automodule:: app.cli.main + :members: \ No newline at end of file diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py index 4d7d3b56..bed4c113 100644 --- a/docs/sphinx/conf.py +++ b/docs/sphinx/conf.py @@ -18,6 +18,7 @@ "sphinx.ext.linkcode", "alabaster", "sphinxcontrib.autodoc_pydantic", + "sphinxcontrib.typer", ] templates_path = ["_templates"] diff --git a/docs/sphinx/index.rst b/docs/sphinx/index.rst index c3e1e2a0..669e63f3 100644 --- a/docs/sphinx/index.rst +++ b/docs/sphinx/index.rst @@ -15,6 +15,7 @@ Contents: api config + cli searching types misc diff --git a/docs/sphinx/types.rst b/docs/sphinx/types.rst index c7836dd6..926d5f32 100644 --- a/docs/sphinx/types.rst +++ b/docs/sphinx/types.rst @@ -7,6 +7,7 @@ Main types .. we need to exclude QueryAnalysis because autodoc pydantic doesn't support it, we add it below .. automodule:: app._types :members: + :exclude-members: QueryAnalysis .. autopydantic_model:: app._types.QueryAnalysis :model-show-json: False diff --git a/scripts/Dockerfile.sphinx b/scripts/Dockerfile.sphinx index 626b0fee..1aac9afe 100644 --- a/scripts/Dockerfile.sphinx +++ b/scripts/Dockerfile.sphinx @@ -17,8 +17,8 @@ RUN ( curl -sSL https://install.python-poetry.org | python3 - ) && \ /root/.local/bin/poetry export --output requirements.txt # install those dependencies RUN pip install -r requirements.txt -# install the autodoc-pydantic plugin for sphinx -RUN pip install autodoc_pydantic +# install some useful plugin for sphinx +RUN pip install autodoc_pydantic sphinxcontrib-typer USER user diff --git a/scripts/build_sphinx.sh b/scripts/build_sphinx.sh index 124732be..8a385b68 100755 --- a/scripts/build_sphinx.sh +++ b/scripts/build_sphinx.sh @@ -19,7 +19,7 @@ docker run --rm --user user \ -v $(pwd)/app:/docs/app \ -v $(pwd)/gh_pages/sphinx:/docs/build \ -e PYTHONPATH=/docs/ \ - -e SPHINXOPTS="-W --keep-going" \ + -e SPHINXOPTS="--fail-on-warning --keep-going --fresh-env" \ sphinx-builder make html # move to the right place and cleanup diff --git a/scripts/sphinx/make.bat b/scripts/sphinx/make.bat deleted file mode 100644 index 747ffb7b..00000000 --- a/scripts/sphinx/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.https://www.sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "" goto help - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd From 38c25ac04e1e5cee531d5faac31ebcc3c209313f Mon Sep 17 00:00:00 2001 From: Alex Garel Date: Tue, 16 Jul 2024 19:10:23 +0200 Subject: [PATCH 4/7] =?UTF-8?q?docs:=C2=A0some=20tweaks=20to=20sphinx=20do?= =?UTF-8?q?cs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/sphinx/conf.py | 8 ++++++++ docs/sphinx/index.rst | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py index bed4c113..4c9066ad 100644 --- a/docs/sphinx/conf.py +++ b/docs/sphinx/conf.py @@ -30,6 +30,14 @@ html_theme = "alabaster" html_static_path = ["_static"] +html_theme_options = { + "github_user": "openfoodfacts", + "github_repo": "search-a-licious", + "github_banner": True, + "extra_nav_links": { + "Back to main doc": "/search-a-licious", + }, +} # -- Options for autodoc pydantic -------------------------------------------- autodoc_pydantic_model_show_json = True diff --git a/docs/sphinx/index.rst b/docs/sphinx/index.rst index 669e63f3..61af4ac5 100644 --- a/docs/sphinx/index.rst +++ b/docs/sphinx/index.rst @@ -8,6 +8,12 @@ Search-a-licious code documentation This documents the backend code of search-alicious. +For `general documentation about search-a-licious Follow this link`__ + +.. note: this only work on github pages +.. __: /search-a-licious + + Contents: .. toctree:: From ab4643b3066cd1738d4ae7b1e5e407f61edc56f1 Mon Sep 17 00:00:00 2001 From: Alex Garel Date: Tue, 16 Jul 2024 19:12:23 +0200 Subject: [PATCH 5/7] =?UTF-8?q?ci:=C2=A0add=20missing=20directories?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/sphinx/_static/.empty | 0 docs/sphinx/_templates/.empty | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/sphinx/_static/.empty create mode 100644 docs/sphinx/_templates/.empty diff --git a/docs/sphinx/_static/.empty b/docs/sphinx/_static/.empty new file mode 100644 index 00000000..e69de29b diff --git a/docs/sphinx/_templates/.empty b/docs/sphinx/_templates/.empty new file mode 100644 index 00000000..e69de29b From c567603ac1981e45b92b0e6b4738a4ac84ab949d Mon Sep 17 00:00:00 2001 From: Pierre Slamich Date: Wed, 17 Jul 2024 09:45:40 +0200 Subject: [PATCH 6/7] ci: Update labeler.yml --- .github/labeler.yml | 52 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index cb79f1c4..fed36045 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,8 +1,56 @@ - - # Add labels to any any pull request with changes to the specified paths # Pull requests that update GitHub Actions code. If you navigate to the folder, you will have a README of what it does 📚 Documentation: - changed-files: - any-glob-to-any-file: 'docs/how-to-install.md' + - any-glob-to-any-file: 'docs/sphinx/_static/.empty' + - any-glob-to-any-file: 'docs/sphinx/_templates/.empty' + - any-glob-to-any-file: 'docs/sphinx/api.rst' + - any-glob-to-any-file: 'docs/sphinx/cli.rst' + - any-glob-to-any-file: 'docs/sphinx/conf.py' + - any-glob-to-any-file: 'docs/sphinx/config.rst' + - any-glob-to-any-file: 'docs/sphinx/index.rst' + - any-glob-to-any-file: 'docs/sphinx/misc.rst' + - any-glob-to-any-file: 'docs/sphinx/searching.rst' + - any-glob-to-any-file: 'docs/sphinx/types.rst' + - any-glob-to-any-file: 'docs/users/ref-python.md' + +API: +- changed-files: + - any-glob-to-any-file: 'docs/sphinx/api.rst' + +Build Scripts: +- changed-files: + - any-glob-to-any-file: 'scripts/Dockerfile.sphinx' + - any-glob-to-any-file: 'scripts/build_sphinx.sh' + - any-glob-to-any-file: 'scripts/generate_doc.sh' + - any-glob-to-any-file: 'scripts/sphinx/Makefile' + +🐳 docker: +- changed-files: + - any-glob-to-any-file: 'Dockerfile' + +Indexation: +- changed-files: + - any-glob-to-any-file: 'app/indexing.py' + - any-glob-to-any-file: 'docs/sphinx/index.rst' + +Taxonomies: +- changed-files: + - any-glob-to-any-file: 'app/taxonomy.py' + - any-glob-to-any-file: 'app/taxonomy_es.py' + +CLI: +- changed-files: + - any-glob-to-any-file: 'app/cli/main.py' + - any-glob-to-any-file: 'docs/sphinx/cli.rst' + +Config: +- changed-files: + - any-glob-to-any-file: 'docs/sphinx/config.rst' + +# - any-glob-to-any-file: 'app/query.py' +# - any-glob-to-any-file: 'app/utils/analyzers.py' +# - any-glob-to-any-file: 'app/utils/download.py' +# - any-glob-to-any-file: 'app/validations.py' From d6238e2bb9cef226048e909e647197a2320cf02f Mon Sep 17 00:00:00 2001 From: Pierre Slamich Date: Wed, 17 Jul 2024 09:46:56 +0200 Subject: [PATCH 7/7] ci: Update labeler.yml --- .github/labeler.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/labeler.yml b/.github/labeler.yml index fed36045..25689fc3 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -30,6 +30,7 @@ Build Scripts: 🐳 docker: - changed-files: - any-glob-to-any-file: 'Dockerfile' + - any-glob-to-any-file: 'docker-compose.yml' Indexation: - changed-files: