From d494f3037fedcb86d09036efd0e3ff67721394d8 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Fri, 14 Apr 2023 17:01:59 +0200 Subject: [PATCH 01/19] Added structure for the documentation. --- .gitignore | 2 +- .pre-commit-config.yaml | 2 +- doc/Makefile | 72 ++++++ doc/source/_static/index-images/api.svg | 50 ++++ .../_static/index-images/contributor.svg | 13 ++ .../_static/index-images/getting_started.svg | 5 + .../_static/index-images/image_licences.txt | 4 + .../_static/index-images/user_guide.svg | 47 ++++ doc/source/_static/qtoolkit.css | 147 ++++++++++++ doc/source/api/index.rst | 15 ++ doc/source/conf.py | 216 ++++++++++++++++++ doc/source/dev/index.rst | 7 + doc/source/glossary.rst | 14 ++ doc/source/index.rst | 106 +++++++++ doc/source/license.rst | 6 + doc/source/user/basics.rst | 12 + doc/source/user/building.rst | 7 + doc/source/user/index.rst | 30 +++ doc/source/user/install.rst | 9 + doc/source/user/quickstart.rst | 17 ++ doc/source/user/whatisqtoolkit.rst | 8 + doc_requirements.txt | 14 ++ src/qtoolkit/core/__init__.py | 0 src/qtoolkit/host/__init__.py | 0 src/qtoolkit/queue/__init__.py | 0 25 files changed, 801 insertions(+), 2 deletions(-) create mode 100644 doc/Makefile create mode 100644 doc/source/_static/index-images/api.svg create mode 100644 doc/source/_static/index-images/contributor.svg create mode 100644 doc/source/_static/index-images/getting_started.svg create mode 100644 doc/source/_static/index-images/image_licences.txt create mode 100644 doc/source/_static/index-images/user_guide.svg create mode 100644 doc/source/_static/qtoolkit.css create mode 100644 doc/source/api/index.rst create mode 100644 doc/source/conf.py create mode 100644 doc/source/dev/index.rst create mode 100644 doc/source/glossary.rst create mode 100644 doc/source/index.rst create mode 100644 doc/source/license.rst create mode 100644 doc/source/user/basics.rst create mode 100644 doc/source/user/building.rst create mode 100644 doc/source/user/index.rst create mode 100644 doc/source/user/install.rst create mode 100644 doc/source/user/quickstart.rst create mode 100644 doc/source/user/whatisqtoolkit.rst create mode 100644 doc_requirements.txt create mode 100644 src/qtoolkit/core/__init__.py create mode 100644 src/qtoolkit/host/__init__.py create mode 100644 src/qtoolkit/queue/__init__.py diff --git a/.gitignore b/.gitignore index a14d6d0..373876b 100644 --- a/.gitignore +++ b/.gitignore @@ -69,7 +69,7 @@ instance/ .scrapy # Sphinx documentation -docs/_build/ +doc/_build/ # PyBuilder .pybuilder/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c59ed26..91c5c57 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ default_language_version: python: python3 -#exclude: '^src/{{ package_name }}/some/directory/' +exclude: '^doc/' repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000..90d6037 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,72 @@ +# Makefile for Sphinx documentation +# + +# PYVER needs to be major.minor, just "3" doesn't work - it will result in +# issues with the amendments to PYTHONPATH and install paths (see DIST_VARS). + +# Use explicit "version_info" indexing since make cannot handle colon characters, and +# evaluate it now to allow easier debugging when printing the variable + +PYVER:=$(shell python3 -c 'from sys import version_info as v; print("{0}.{1}".format(v[0], v[1]))') +PYTHON = python$(PYVER) + +# You can set these variables from the command line. +SPHINXOPTS ?= +SPHINXBUILD ?= LANG=C sphinx-build +PAPER ?= +# # For merging a documentation archive into a git checkout of numpy/doc +# # Turn a tag like v1.18.0 into 1.18 +# # Use sed -n -e 's/patttern/match/p' to return a blank value if no match +# TAG ?= $(shell git describe --tag | sed -n -e's,v\([1-9]\.[0-9]*\)\.[0-9].*,\1,p') + +FILES= + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -WT --keep-going -d build/doctrees $(PAPEROPT_$(PAPER)) \ + $(SPHINXOPTS) source + +.PHONY: help clean html version-check html-build + +#------------------------------------------------------------------------------ + +help: + @echo "Please use \`make ' where is one of" + @echo " clean to remove generated doc files and start fresh" + @echo " html to make standalone HTML files" + +clean: + -rm -rf build/* + find . -name generated -type d -prune -exec rm -rf "{}" ";" + + +#------------------------------------------------------------------------------ +# Automated generation of all documents +#------------------------------------------------------------------------------ + +# Build the current QToolKit version, and extract docs from it. +# We have to be careful of some issues: +# +# - Everything must be done using the same Python version +# + +#SPHINXBUILD="LANG=C sphinx-build" + + +#------------------------------------------------------------------------------ +# Basic Sphinx generation rules for different formats +#------------------------------------------------------------------------------ +generate: build/generate-stamp +build/generate-stamp: $(wildcard source/reference/*.rst) + mkdir -p build + touch build/generate-stamp + +html: api-doc html-build +html-build: generate + mkdir -p build/html build/doctrees + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) build/html $(FILES) + @echo + @echo "Build finished. The HTML pages are in build/html." +api-doc: + sphinx-apidoc -f -o source/api ../src/qtoolkit diff --git a/doc/source/_static/index-images/api.svg b/doc/source/_static/index-images/api.svg new file mode 100644 index 0000000..9c88397 --- /dev/null +++ b/doc/source/_static/index-images/api.svg @@ -0,0 +1,50 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/source/_static/index-images/contributor.svg b/doc/source/_static/index-images/contributor.svg new file mode 100644 index 0000000..ffd444e --- /dev/null +++ b/doc/source/_static/index-images/contributor.svg @@ -0,0 +1,13 @@ + + + + + + + + \ No newline at end of file diff --git a/doc/source/_static/index-images/getting_started.svg b/doc/source/_static/index-images/getting_started.svg new file mode 100644 index 0000000..20747f9 --- /dev/null +++ b/doc/source/_static/index-images/getting_started.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/doc/source/_static/index-images/image_licences.txt b/doc/source/_static/index-images/image_licences.txt new file mode 100644 index 0000000..85276bc --- /dev/null +++ b/doc/source/_static/index-images/image_licences.txt @@ -0,0 +1,4 @@ +getting_started.svg: https://www.svgrepo.com/svg/393367/rocket (PD Licence) +user_guide.svg: https://www.svgrepo.com/svg/75531/user-guide (CC0 Licence) +api.svg: https://www.svgrepo.com/svg/157898/gears-configuration-tool (CC0 Licence) +contributor.svg: https://www.svgrepo.com/svg/57189/code-programing-symbol (CC0 Licence) \ No newline at end of file diff --git a/doc/source/_static/index-images/user_guide.svg b/doc/source/_static/index-images/user_guide.svg new file mode 100644 index 0000000..6223a92 --- /dev/null +++ b/doc/source/_static/index-images/user_guide.svg @@ -0,0 +1,47 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/source/_static/qtoolkit.css b/doc/source/_static/qtoolkit.css new file mode 100644 index 0000000..4a77b9c --- /dev/null +++ b/doc/source/_static/qtoolkit.css @@ -0,0 +1,147 @@ +@import url('https://fonts.googleapis.com/css2?family=Lato:ital,wght@0,400;0,700;0,900;1,400;1,700;1,900&family=Open+Sans:ital,wght@0,400;0,600;1,400;1,600&display=swap'); + +.navbar-brand img { + height: 75px; +} +.navbar-brand { + height: 75px; +} + +body { + font-family: 'Open Sans', sans-serif; +} + +pre, code { + font-size: 100%; + line-height: 155%; +} + +h1 { + font-family: "Lato", sans-serif; + color: #013243; /* warm black */ +} + +h2 { + color: #4d77cf; /* han blue */ + letter-spacing: -.03em; +} + +h3 { + color: #013243; /* warm black */ + letter-spacing: -.03em; +} + +/* Style the active version button. + +- dev: orange +- stable: green +- old, PR: red + +Colors from: + +Wong, B. Points of view: Color blindness. +Nat Methods 8, 441 (2011). https://doi.org/10.1038/nmeth.1618 +*/ + +/* If the active version has the name "dev", style it orange */ +#version_switcher_button[data-active-version-name*="dev"] { + background-color: #E69F00; + border-color: #E69F00; + color:#000000; +} + +/* green for `stable` */ +#version_switcher_button[data-active-version-name*="stable"] { + background-color: #009E73; + border-color: #009E73; +} + +/* red for `old` */ +#version_switcher_button:not([data-active-version-name*="stable"], [data-active-version-name*="dev"], [data-active-version-name=""]) { + background-color: #980F0F; + border-color: #980F0F; +} + +/* Main page overview cards */ + +.sd-card { + background: #fff; + border-radius: 0; + padding: 30px 10px 20px 10px; + margin: 10px 0px; +} + +.sd-card .sd-card-header { + text-align: center; +} + +.sd-card .sd-card-header .sd-card-text { + margin: 0px; +} + +.sd-card .sd-card-img-top { + height: 52px; + width: 52px; + margin-left: auto; + margin-right: auto; +} + +.sd-card .sd-card-header { + border: none; + background-color: white; + color: #150458 !important; + font-size: var(--pst-font-size-h5); + font-weight: bold; + padding: 2.5rem 0rem 0.5rem 0rem; +} + +.sd-card .sd-card-footer { + border: none; + background-color: white; +} + +.sd-card .sd-card-footer .sd-card-text { + max-width: 220px; + margin-left: auto; + margin-right: auto; +} + +/* Announcements */ +.bd-header-announcement { + background-color: orange; +} + +/* Dark theme tweaking */ +html[data-theme=dark] .sd-card img[src*='.svg'] { + filter: invert(0.82) brightness(0.8) contrast(1.2); +} + +/* Main index page overview cards */ +html[data-theme=dark] .sd-card { + background-color:var(--pst-color-background); +} + +html[data-theme=dark] .sd-shadow-sm { + box-shadow: 0 .1rem 1rem rgba(250, 250, 250, .6) !important +} + +html[data-theme=dark] .sd-card .sd-card-header { + background-color:var(--pst-color-background); + color: #150458 !important; +} + +html[data-theme=dark] .sd-card .sd-card-footer { + background-color:var(--pst-color-background); +} + +html[data-theme=dark] .bd-header-announcement { + background-color: red; +} + +html[data-theme=dark] h1 { + color: var(--pst-color-primary); +} + +html[data-theme=dark] h3 { + color: #0a6774; +} diff --git a/doc/source/api/index.rst b/doc/source/api/index.rst new file mode 100644 index 0000000..daebaf2 --- /dev/null +++ b/doc/source/api/index.rst @@ -0,0 +1,15 @@ +.. _api: + +============= +API Reference +============= + +This is the API reference + +.. toctree:: + :maxdepth: 1 + + qtoolkit.core + qtoolkit.host + qtoolkit.io + qtoolkit.queue \ No newline at end of file diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 0000000..8ace3c5 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,216 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +# sys.path.insert(0, os.path.abspath('.')) +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +) + +# -- Project information ----------------------------------------------------- + +project = "QToolKit" +copyright = "2023, Matgenix SRL" +author = "Guido Petretto, David Waroquiers" + + +import qtoolkit +# The short X.Y version +version = qtoolkit.__version__ +# The full version, including alpha/beta/rc tags +release = qtoolkit.__version__ + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.viewcode", + "sphinx.ext.napoleon", # For Google Python Style Guide + 'sphinx.ext.coverage', + 'sphinx.ext.doctest', + 'sphinx.ext.autosummary', + 'sphinx.ext.graphviz', + 'sphinx.ext.ifconfig', + 'matplotlib.sphinxext.plot_directive', + 'IPython.sphinxext.ipython_console_highlighting', + 'IPython.sphinxext.ipython_directive', + 'sphinx.ext.mathjax', + 'sphinx_design', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +# html_theme = 'sphinx_book_theme' +html_theme = 'pydata_sphinx_theme' +# html_favicon = '_static/favicon/favicon.ico' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +html_theme_options = { + # "logo": { + # "image_light": "index-image/api.svg", + # "image_dark": "index-image/contributor.svg", + # }, + "collapse_navigation": True, + 'announcement': ( + "

" + "QToolKit is still in beta phase. The API may change at any time." + "

" + ), + # "announcement": "

This is still in development

", + # "navbar_end": ["theme-switcher", "navbar-icon-links"], + # "navbar_end": ["theme-switcher", "version-switcher", "navbar-icon-links"], +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +html_css_files = ["qtoolkit.css"] +html_title = "%s v%s Manual" % (project, version) +html_last_updated_fmt = '%b %d, %Y' +# html_css_files = ["numpy.css"] +html_context = {"default_mode": "light"} +html_use_modindex = True +html_copy_source = False +html_domain_indices = False +html_file_suffix = '.html' + +# Output file base name for HTML help builder. +htmlhelp_basename = "qtoolkitdoc" + + +# # -- Options for LaTeX output ------------------------------------------------ +# +# latex_elements = { +# # The paper size ('letterpaper' or 'a4paper'). +# # +# # 'papersize': 'letterpaper', +# # The font size ('10pt', '11pt' or '12pt'). +# # +# # 'pointsize': '10pt', +# # Additional stuff for the LaTeX preamble. +# # +# # 'preamble': '', +# # Latex figure (float) alignment +# # +# # 'figure_align': 'htbp', +# } +# +# # Grouping the document tree into LaTeX files. List of tuples +# # (source start file, target name, title, +# # author, documentclass [howto, manual, or own class]). +# latex_documents = [ +# # (master_doc, "turbomoleio.tex", "turbomoleio Documentation", author, "manual"), +# ] + + +# # -- Options for manual page output ------------------------------------------ +# +# # One entry per manual page. List of tuples +# # (source start file, name, description, authors, manual section). +# man_pages = [(master_doc, "turbomoleio", "turbomoleio Documentation", [author], 1)] +# +# +# # -- Options for Texinfo output ---------------------------------------------- +# +# # Grouping the document tree into Texinfo files. List of tuples +# # (source start file, target name, title, author, +# # dir menu entry, description, category) +# texinfo_documents = [ +# ( +# master_doc, +# "turbomoleio", +# "turbomoleio Documentation", +# author, +# "turbomoleio", +# "One line description of project.", +# "Miscellaneous", +# ), +# ] + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for intersphinx extension --------------------------------------- + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {"https://docs.python.org/": None} + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + +# To print the content of the docstring of the __init__ method as well. +autoclass_content = "both" diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst new file mode 100644 index 0000000..5bf8af1 --- /dev/null +++ b/doc/source/dev/index.rst @@ -0,0 +1,7 @@ +.. _devindex: + +######################## +Contributing to QToolKit +######################## + +Here are the things that can be done. \ No newline at end of file diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst new file mode 100644 index 0000000..3850067 --- /dev/null +++ b/doc/source/glossary.rst @@ -0,0 +1,14 @@ +******** +Glossary +******** + +.. glossary:: + + + QJob + The representation of a job in the queue. + + + QResources + The description of the resources for a job. + diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000..39690e7 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,106 @@ +.. _qtoolkit_docs_mainpage: + +###################### +QToolKit documentation +###################### + +.. toctree:: + :maxdepth: 1 + :hidden: + + User Guide + API reference + Development + release + + +**Version**: |version| + +**Useful links**: +TO BE ADDED + +QToolKit is an interface to most Distributed Resource Management systems, e.g. +SLURM, PBS, ... + + + +.. grid:: 1 2 2 2 + + .. grid-item-card:: + :img-top: ../source/_static/index-images/getting_started.svg + + Getting Started + ^^^^^^^^^^^^^^^ + + New to QToolKit? Check out the Absolute Beginner's Guide. It contains an + introduction to QToolKit's main concepts and links to additional tutorials. + + +++ + + .. button-ref:: user/absolute_beginners + :expand: + :color: secondary + :click-parent: + + To the absolute beginner's guide + + .. grid-item-card:: + :img-top: ../source/_static/index-images/user_guide.svg + + User Guide + ^^^^^^^^^^ + + The user guide provides in-depth information on the + key concepts of QToolKit with useful background information and explanation. + + +++ + + .. button-ref:: user + :expand: + :color: secondary + :click-parent: + + To the user guide + + .. grid-item-card:: + :img-top: ../source/_static/index-images/api.svg + + API Reference + ^^^^^^^^^^^^^ + + The reference guide contains a detailed description of the functions, + modules, and objects included in QToolKit. The reference describes how the + methods work and which parameters can be used. It assumes that you have an + understanding of the key concepts. + + +++ + + .. button-ref:: api + :expand: + :color: secondary + :click-parent: + + To the reference guide + + .. grid-item-card:: + :img-top: ../source/_static/index-images/contributor.svg + + Contributor's Guide + ^^^^^^^^^^^^^^^^^^^ + + Want to add to the codebase? Can help add support to an additional DRM system? + The contributing guidelines will guide you through the + process of improving QToolKit. + + +++ + + .. button-ref:: devindex + :expand: + :color: secondary + :click-parent: + + To the contributor's guide + +.. This is not really the index page, that is found in + _templates/indexcontent.html The toctree content here will be added to the + top of the template header \ No newline at end of file diff --git a/doc/source/license.rst b/doc/source/license.rst new file mode 100644 index 0000000..f36eb5b --- /dev/null +++ b/doc/source/license.rst @@ -0,0 +1,6 @@ +**************** +QToolKit license +**************** + +.. include:: ../../LICENSE + :literal: diff --git a/doc/source/user/basics.rst b/doc/source/user/basics.rst new file mode 100644 index 0000000..67af9c2 --- /dev/null +++ b/doc/source/user/basics.rst @@ -0,0 +1,12 @@ +********************* +QToolKit fundamentals +********************* + +These documents clarify concepts, design decisions, and technical +constraints in QToolKit. This is a great place to understand the +fundamental QToolKit ideas and philosophy. + +.. + .. toctree:: + :maxdepth: 1 + diff --git a/doc/source/user/building.rst b/doc/source/user/building.rst new file mode 100644 index 0000000..93af682 --- /dev/null +++ b/doc/source/user/building.rst @@ -0,0 +1,7 @@ +.. _building-from-source: + +Building from source +==================== + +Get the source from the git repository. +Install it with pip install . \ No newline at end of file diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst new file mode 100644 index 0000000..5155edf --- /dev/null +++ b/doc/source/user/index.rst @@ -0,0 +1,30 @@ +.. _user: + +################### +QToolKit user guide +################### + +This guide is an overview and explains the important features; +details are found in :ref:`reference`. + +.. toctree:: + :caption: Getting started + :maxdepth: 1 + + whatisqtoolkit + install + quickstart + +.. toctree:: + :caption: Advanced usage and interoperability + :maxdepth: 1 + + building + + +.. toctree:: + :hidden: + :caption: Extras + + ../glossary + ../license diff --git a/doc/source/user/install.rst b/doc/source/user/install.rst new file mode 100644 index 0000000..50cd00e --- /dev/null +++ b/doc/source/user/install.rst @@ -0,0 +1,9 @@ +.. _install: + +******************* +Installing QToolKit +******************* + +The only prerequisite for installing QToolKit is Python. + +QToolKit can be installed with 'conda', with 'pip', or from source. \ No newline at end of file diff --git a/doc/source/user/quickstart.rst b/doc/source/user/quickstart.rst new file mode 100644 index 0000000..da9978c --- /dev/null +++ b/doc/source/user/quickstart.rst @@ -0,0 +1,17 @@ +.. _quickstart: + +=================== +QToolKit quickstart +=================== + +Prerequisites +============= +You need python + +The Basics +========== + +Create an easy script + +ADD OTHER TOPICS: +remote submission, resources, ... \ No newline at end of file diff --git a/doc/source/user/whatisqtoolkit.rst b/doc/source/user/whatisqtoolkit.rst new file mode 100644 index 0000000..a302ec4 --- /dev/null +++ b/doc/source/user/whatisqtoolkit.rst @@ -0,0 +1,8 @@ +.. _whatisqtoolkit: + +================= +What is QToolKit? +================= + +QToolKit is ... +TODO: add the features that it has. \ No newline at end of file diff --git a/doc_requirements.txt b/doc_requirements.txt new file mode 100644 index 0000000..53b03cc --- /dev/null +++ b/doc_requirements.txt @@ -0,0 +1,14 @@ +# doxygen required, use apt-get or dnf +sphinx>=4.5.0 +numpydoc==1.4 +pydata-sphinx-theme==0.13.3 +sphinx-design +sphinx-apidoc +ipython!=8.1.0 +scipy +matplotlib +pandas +breathe + +# needed to build release notes +towncrier diff --git a/src/qtoolkit/core/__init__.py b/src/qtoolkit/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/qtoolkit/host/__init__.py b/src/qtoolkit/host/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/qtoolkit/queue/__init__.py b/src/qtoolkit/queue/__init__.py new file mode 100644 index 0000000..e69de29 From 8f5a4b84883bb89fa05dff57f621ba7ca773c26c Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Fri, 14 Apr 2023 17:10:26 +0200 Subject: [PATCH 02/19] Fixed navigation for api. --- doc/source/api/index.rst | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/doc/source/api/index.rst b/doc/source/api/index.rst index daebaf2..484de6a 100644 --- a/doc/source/api/index.rst +++ b/doc/source/api/index.rst @@ -1,15 +1,9 @@ .. _api: -============= +############# API Reference -============= +############# This is the API reference -.. toctree:: - :maxdepth: 1 - - qtoolkit.core - qtoolkit.host - qtoolkit.io - qtoolkit.queue \ No newline at end of file +.. include:: qtoolkit.rst \ No newline at end of file From 3b0a6c518ce60661bab7aaa5d6066a2f5aca9192 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Sat, 29 Apr 2023 23:39:55 +0200 Subject: [PATCH 03/19] Updated css styling (Matgenix colors). --- doc/Makefile | 2 +- doc/source/_static/qtoolkit.css | 15 +++++++++++++++ doc/source/index.rst | 12 ++++++------ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/doc/Makefile b/doc/Makefile index 90d6037..3e5f5da 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -69,4 +69,4 @@ html-build: generate @echo @echo "Build finished. The HTML pages are in build/html." api-doc: - sphinx-apidoc -f -o source/api ../src/qtoolkit + sphinx-apidoc -e -f -o source/api ../src/qtoolkit diff --git a/doc/source/_static/qtoolkit.css b/doc/source/_static/qtoolkit.css index 4a77b9c..4561c96 100644 --- a/doc/source/_static/qtoolkit.css +++ b/doc/source/_static/qtoolkit.css @@ -1,5 +1,10 @@ @import url('https://fonts.googleapis.com/css2?family=Lato:ital,wght@0,400;0,700;0,900;1,400;1,700;1,900&family=Open+Sans:ital,wght@0,400;0,600;1,400;1,600&display=swap'); +:root { + --matgenix-color: #46b3c1; + --matgenix-dark-color: #338d99; +} + .navbar-brand img { height: 75px; } @@ -145,3 +150,13 @@ html[data-theme=dark] h1 { html[data-theme=dark] h3 { color: #0a6774; } + +.sd-btn-secondary { + background-color: var(--matgenix-color) !important; + border-color: var(--matgenix-color) !important; +} + +.sd-btn-secondary:hover, .sd-btn-secondary:focus { + background-color: var(--matgenix-dark-color) !important; + border-color: var(--matgenix-dark-color) !important; +} \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst index 39690e7..46f8eef 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -32,17 +32,17 @@ SLURM, PBS, ... Getting Started ^^^^^^^^^^^^^^^ - New to QToolKit? Check out the Absolute Beginner's Guide. It contains an - introduction to QToolKit's main concepts and links to additional tutorials. + If you want to get started quickly, check out our quickstart section. + It contains an introduction to QToolKit's main concepts. +++ - .. button-ref:: user/absolute_beginners + .. button-ref:: user/quickstart :expand: :color: secondary :click-parent: - To the absolute beginner's guide + Quickstart .. grid-item-card:: :img-top: ../source/_static/index-images/user_guide.svg @@ -60,7 +60,7 @@ SLURM, PBS, ... :color: secondary :click-parent: - To the user guide + User Guide .. grid-item-card:: :img-top: ../source/_static/index-images/api.svg @@ -80,7 +80,7 @@ SLURM, PBS, ... :color: secondary :click-parent: - To the reference guide + API Reference .. grid-item-card:: :img-top: ../source/_static/index-images/contributor.svg From f4512c6ea18740467bef22dac3ac178f6e0dcee6 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Sat, 29 Apr 2023 23:50:19 +0200 Subject: [PATCH 04/19] Added optional deps for docs. --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1432ea9..92453e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,11 @@ tests = [ maintain = [ "git-changelog>=0.6", ] +docs = [ + "sphinx", + "sphinx_design", + "pydata-sphinx-theme", +] strict = [] remote = ["fabric>=3.0.0"] msonable = ["monty>=2022.9.9",] From c588c19fe8779b3efef32c5e3463b82ddec9f8e4 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Fri, 14 Apr 2023 17:01:59 +0200 Subject: [PATCH 05/19] Added structure for the documentation. --- .gitignore | 2 +- .pre-commit-config.yaml | 2 +- doc/Makefile | 72 ++++++ doc/source/_static/index-images/api.svg | 50 ++++ .../_static/index-images/contributor.svg | 13 ++ .../_static/index-images/getting_started.svg | 5 + .../_static/index-images/image_licences.txt | 4 + .../_static/index-images/user_guide.svg | 47 ++++ doc/source/_static/qtoolkit.css | 147 ++++++++++++ doc/source/api/index.rst | 15 ++ doc/source/conf.py | 216 ++++++++++++++++++ doc/source/dev/index.rst | 7 + doc/source/glossary.rst | 14 ++ doc/source/index.rst | 106 +++++++++ doc/source/license.rst | 6 + doc/source/user/basics.rst | 12 + doc/source/user/building.rst | 7 + doc/source/user/index.rst | 30 +++ doc/source/user/install.rst | 9 + doc/source/user/quickstart.rst | 17 ++ doc/source/user/whatisqtoolkit.rst | 8 + doc_requirements.txt | 14 ++ src/qtoolkit/core/__init__.py | 0 src/qtoolkit/host/__init__.py | 0 src/qtoolkit/queue/__init__.py | 0 25 files changed, 801 insertions(+), 2 deletions(-) create mode 100644 doc/Makefile create mode 100644 doc/source/_static/index-images/api.svg create mode 100644 doc/source/_static/index-images/contributor.svg create mode 100644 doc/source/_static/index-images/getting_started.svg create mode 100644 doc/source/_static/index-images/image_licences.txt create mode 100644 doc/source/_static/index-images/user_guide.svg create mode 100644 doc/source/_static/qtoolkit.css create mode 100644 doc/source/api/index.rst create mode 100644 doc/source/conf.py create mode 100644 doc/source/dev/index.rst create mode 100644 doc/source/glossary.rst create mode 100644 doc/source/index.rst create mode 100644 doc/source/license.rst create mode 100644 doc/source/user/basics.rst create mode 100644 doc/source/user/building.rst create mode 100644 doc/source/user/index.rst create mode 100644 doc/source/user/install.rst create mode 100644 doc/source/user/quickstart.rst create mode 100644 doc/source/user/whatisqtoolkit.rst create mode 100644 doc_requirements.txt create mode 100644 src/qtoolkit/core/__init__.py create mode 100644 src/qtoolkit/host/__init__.py create mode 100644 src/qtoolkit/queue/__init__.py diff --git a/.gitignore b/.gitignore index a14d6d0..373876b 100644 --- a/.gitignore +++ b/.gitignore @@ -69,7 +69,7 @@ instance/ .scrapy # Sphinx documentation -docs/_build/ +doc/_build/ # PyBuilder .pybuilder/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c59ed26..91c5c57 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ default_language_version: python: python3 -#exclude: '^src/{{ package_name }}/some/directory/' +exclude: '^doc/' repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000..90d6037 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,72 @@ +# Makefile for Sphinx documentation +# + +# PYVER needs to be major.minor, just "3" doesn't work - it will result in +# issues with the amendments to PYTHONPATH and install paths (see DIST_VARS). + +# Use explicit "version_info" indexing since make cannot handle colon characters, and +# evaluate it now to allow easier debugging when printing the variable + +PYVER:=$(shell python3 -c 'from sys import version_info as v; print("{0}.{1}".format(v[0], v[1]))') +PYTHON = python$(PYVER) + +# You can set these variables from the command line. +SPHINXOPTS ?= +SPHINXBUILD ?= LANG=C sphinx-build +PAPER ?= +# # For merging a documentation archive into a git checkout of numpy/doc +# # Turn a tag like v1.18.0 into 1.18 +# # Use sed -n -e 's/patttern/match/p' to return a blank value if no match +# TAG ?= $(shell git describe --tag | sed -n -e's,v\([1-9]\.[0-9]*\)\.[0-9].*,\1,p') + +FILES= + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -WT --keep-going -d build/doctrees $(PAPEROPT_$(PAPER)) \ + $(SPHINXOPTS) source + +.PHONY: help clean html version-check html-build + +#------------------------------------------------------------------------------ + +help: + @echo "Please use \`make ' where is one of" + @echo " clean to remove generated doc files and start fresh" + @echo " html to make standalone HTML files" + +clean: + -rm -rf build/* + find . -name generated -type d -prune -exec rm -rf "{}" ";" + + +#------------------------------------------------------------------------------ +# Automated generation of all documents +#------------------------------------------------------------------------------ + +# Build the current QToolKit version, and extract docs from it. +# We have to be careful of some issues: +# +# - Everything must be done using the same Python version +# + +#SPHINXBUILD="LANG=C sphinx-build" + + +#------------------------------------------------------------------------------ +# Basic Sphinx generation rules for different formats +#------------------------------------------------------------------------------ +generate: build/generate-stamp +build/generate-stamp: $(wildcard source/reference/*.rst) + mkdir -p build + touch build/generate-stamp + +html: api-doc html-build +html-build: generate + mkdir -p build/html build/doctrees + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) build/html $(FILES) + @echo + @echo "Build finished. The HTML pages are in build/html." +api-doc: + sphinx-apidoc -f -o source/api ../src/qtoolkit diff --git a/doc/source/_static/index-images/api.svg b/doc/source/_static/index-images/api.svg new file mode 100644 index 0000000..9c88397 --- /dev/null +++ b/doc/source/_static/index-images/api.svg @@ -0,0 +1,50 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/source/_static/index-images/contributor.svg b/doc/source/_static/index-images/contributor.svg new file mode 100644 index 0000000..ffd444e --- /dev/null +++ b/doc/source/_static/index-images/contributor.svg @@ -0,0 +1,13 @@ + + + + + + + + \ No newline at end of file diff --git a/doc/source/_static/index-images/getting_started.svg b/doc/source/_static/index-images/getting_started.svg new file mode 100644 index 0000000..20747f9 --- /dev/null +++ b/doc/source/_static/index-images/getting_started.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/doc/source/_static/index-images/image_licences.txt b/doc/source/_static/index-images/image_licences.txt new file mode 100644 index 0000000..85276bc --- /dev/null +++ b/doc/source/_static/index-images/image_licences.txt @@ -0,0 +1,4 @@ +getting_started.svg: https://www.svgrepo.com/svg/393367/rocket (PD Licence) +user_guide.svg: https://www.svgrepo.com/svg/75531/user-guide (CC0 Licence) +api.svg: https://www.svgrepo.com/svg/157898/gears-configuration-tool (CC0 Licence) +contributor.svg: https://www.svgrepo.com/svg/57189/code-programing-symbol (CC0 Licence) \ No newline at end of file diff --git a/doc/source/_static/index-images/user_guide.svg b/doc/source/_static/index-images/user_guide.svg new file mode 100644 index 0000000..6223a92 --- /dev/null +++ b/doc/source/_static/index-images/user_guide.svg @@ -0,0 +1,47 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/source/_static/qtoolkit.css b/doc/source/_static/qtoolkit.css new file mode 100644 index 0000000..4a77b9c --- /dev/null +++ b/doc/source/_static/qtoolkit.css @@ -0,0 +1,147 @@ +@import url('https://fonts.googleapis.com/css2?family=Lato:ital,wght@0,400;0,700;0,900;1,400;1,700;1,900&family=Open+Sans:ital,wght@0,400;0,600;1,400;1,600&display=swap'); + +.navbar-brand img { + height: 75px; +} +.navbar-brand { + height: 75px; +} + +body { + font-family: 'Open Sans', sans-serif; +} + +pre, code { + font-size: 100%; + line-height: 155%; +} + +h1 { + font-family: "Lato", sans-serif; + color: #013243; /* warm black */ +} + +h2 { + color: #4d77cf; /* han blue */ + letter-spacing: -.03em; +} + +h3 { + color: #013243; /* warm black */ + letter-spacing: -.03em; +} + +/* Style the active version button. + +- dev: orange +- stable: green +- old, PR: red + +Colors from: + +Wong, B. Points of view: Color blindness. +Nat Methods 8, 441 (2011). https://doi.org/10.1038/nmeth.1618 +*/ + +/* If the active version has the name "dev", style it orange */ +#version_switcher_button[data-active-version-name*="dev"] { + background-color: #E69F00; + border-color: #E69F00; + color:#000000; +} + +/* green for `stable` */ +#version_switcher_button[data-active-version-name*="stable"] { + background-color: #009E73; + border-color: #009E73; +} + +/* red for `old` */ +#version_switcher_button:not([data-active-version-name*="stable"], [data-active-version-name*="dev"], [data-active-version-name=""]) { + background-color: #980F0F; + border-color: #980F0F; +} + +/* Main page overview cards */ + +.sd-card { + background: #fff; + border-radius: 0; + padding: 30px 10px 20px 10px; + margin: 10px 0px; +} + +.sd-card .sd-card-header { + text-align: center; +} + +.sd-card .sd-card-header .sd-card-text { + margin: 0px; +} + +.sd-card .sd-card-img-top { + height: 52px; + width: 52px; + margin-left: auto; + margin-right: auto; +} + +.sd-card .sd-card-header { + border: none; + background-color: white; + color: #150458 !important; + font-size: var(--pst-font-size-h5); + font-weight: bold; + padding: 2.5rem 0rem 0.5rem 0rem; +} + +.sd-card .sd-card-footer { + border: none; + background-color: white; +} + +.sd-card .sd-card-footer .sd-card-text { + max-width: 220px; + margin-left: auto; + margin-right: auto; +} + +/* Announcements */ +.bd-header-announcement { + background-color: orange; +} + +/* Dark theme tweaking */ +html[data-theme=dark] .sd-card img[src*='.svg'] { + filter: invert(0.82) brightness(0.8) contrast(1.2); +} + +/* Main index page overview cards */ +html[data-theme=dark] .sd-card { + background-color:var(--pst-color-background); +} + +html[data-theme=dark] .sd-shadow-sm { + box-shadow: 0 .1rem 1rem rgba(250, 250, 250, .6) !important +} + +html[data-theme=dark] .sd-card .sd-card-header { + background-color:var(--pst-color-background); + color: #150458 !important; +} + +html[data-theme=dark] .sd-card .sd-card-footer { + background-color:var(--pst-color-background); +} + +html[data-theme=dark] .bd-header-announcement { + background-color: red; +} + +html[data-theme=dark] h1 { + color: var(--pst-color-primary); +} + +html[data-theme=dark] h3 { + color: #0a6774; +} diff --git a/doc/source/api/index.rst b/doc/source/api/index.rst new file mode 100644 index 0000000..daebaf2 --- /dev/null +++ b/doc/source/api/index.rst @@ -0,0 +1,15 @@ +.. _api: + +============= +API Reference +============= + +This is the API reference + +.. toctree:: + :maxdepth: 1 + + qtoolkit.core + qtoolkit.host + qtoolkit.io + qtoolkit.queue \ No newline at end of file diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 0000000..8ace3c5 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,216 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +# sys.path.insert(0, os.path.abspath('.')) +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +) + +# -- Project information ----------------------------------------------------- + +project = "QToolKit" +copyright = "2023, Matgenix SRL" +author = "Guido Petretto, David Waroquiers" + + +import qtoolkit +# The short X.Y version +version = qtoolkit.__version__ +# The full version, including alpha/beta/rc tags +release = qtoolkit.__version__ + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.viewcode", + "sphinx.ext.napoleon", # For Google Python Style Guide + 'sphinx.ext.coverage', + 'sphinx.ext.doctest', + 'sphinx.ext.autosummary', + 'sphinx.ext.graphviz', + 'sphinx.ext.ifconfig', + 'matplotlib.sphinxext.plot_directive', + 'IPython.sphinxext.ipython_console_highlighting', + 'IPython.sphinxext.ipython_directive', + 'sphinx.ext.mathjax', + 'sphinx_design', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +# html_theme = 'sphinx_book_theme' +html_theme = 'pydata_sphinx_theme' +# html_favicon = '_static/favicon/favicon.ico' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +html_theme_options = { + # "logo": { + # "image_light": "index-image/api.svg", + # "image_dark": "index-image/contributor.svg", + # }, + "collapse_navigation": True, + 'announcement': ( + "

" + "QToolKit is still in beta phase. The API may change at any time." + "

" + ), + # "announcement": "

This is still in development

", + # "navbar_end": ["theme-switcher", "navbar-icon-links"], + # "navbar_end": ["theme-switcher", "version-switcher", "navbar-icon-links"], +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +html_css_files = ["qtoolkit.css"] +html_title = "%s v%s Manual" % (project, version) +html_last_updated_fmt = '%b %d, %Y' +# html_css_files = ["numpy.css"] +html_context = {"default_mode": "light"} +html_use_modindex = True +html_copy_source = False +html_domain_indices = False +html_file_suffix = '.html' + +# Output file base name for HTML help builder. +htmlhelp_basename = "qtoolkitdoc" + + +# # -- Options for LaTeX output ------------------------------------------------ +# +# latex_elements = { +# # The paper size ('letterpaper' or 'a4paper'). +# # +# # 'papersize': 'letterpaper', +# # The font size ('10pt', '11pt' or '12pt'). +# # +# # 'pointsize': '10pt', +# # Additional stuff for the LaTeX preamble. +# # +# # 'preamble': '', +# # Latex figure (float) alignment +# # +# # 'figure_align': 'htbp', +# } +# +# # Grouping the document tree into LaTeX files. List of tuples +# # (source start file, target name, title, +# # author, documentclass [howto, manual, or own class]). +# latex_documents = [ +# # (master_doc, "turbomoleio.tex", "turbomoleio Documentation", author, "manual"), +# ] + + +# # -- Options for manual page output ------------------------------------------ +# +# # One entry per manual page. List of tuples +# # (source start file, name, description, authors, manual section). +# man_pages = [(master_doc, "turbomoleio", "turbomoleio Documentation", [author], 1)] +# +# +# # -- Options for Texinfo output ---------------------------------------------- +# +# # Grouping the document tree into Texinfo files. List of tuples +# # (source start file, target name, title, author, +# # dir menu entry, description, category) +# texinfo_documents = [ +# ( +# master_doc, +# "turbomoleio", +# "turbomoleio Documentation", +# author, +# "turbomoleio", +# "One line description of project.", +# "Miscellaneous", +# ), +# ] + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for intersphinx extension --------------------------------------- + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {"https://docs.python.org/": None} + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + +# To print the content of the docstring of the __init__ method as well. +autoclass_content = "both" diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst new file mode 100644 index 0000000..5bf8af1 --- /dev/null +++ b/doc/source/dev/index.rst @@ -0,0 +1,7 @@ +.. _devindex: + +######################## +Contributing to QToolKit +######################## + +Here are the things that can be done. \ No newline at end of file diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst new file mode 100644 index 0000000..3850067 --- /dev/null +++ b/doc/source/glossary.rst @@ -0,0 +1,14 @@ +******** +Glossary +******** + +.. glossary:: + + + QJob + The representation of a job in the queue. + + + QResources + The description of the resources for a job. + diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000..39690e7 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,106 @@ +.. _qtoolkit_docs_mainpage: + +###################### +QToolKit documentation +###################### + +.. toctree:: + :maxdepth: 1 + :hidden: + + User Guide + API reference + Development + release + + +**Version**: |version| + +**Useful links**: +TO BE ADDED + +QToolKit is an interface to most Distributed Resource Management systems, e.g. +SLURM, PBS, ... + + + +.. grid:: 1 2 2 2 + + .. grid-item-card:: + :img-top: ../source/_static/index-images/getting_started.svg + + Getting Started + ^^^^^^^^^^^^^^^ + + New to QToolKit? Check out the Absolute Beginner's Guide. It contains an + introduction to QToolKit's main concepts and links to additional tutorials. + + +++ + + .. button-ref:: user/absolute_beginners + :expand: + :color: secondary + :click-parent: + + To the absolute beginner's guide + + .. grid-item-card:: + :img-top: ../source/_static/index-images/user_guide.svg + + User Guide + ^^^^^^^^^^ + + The user guide provides in-depth information on the + key concepts of QToolKit with useful background information and explanation. + + +++ + + .. button-ref:: user + :expand: + :color: secondary + :click-parent: + + To the user guide + + .. grid-item-card:: + :img-top: ../source/_static/index-images/api.svg + + API Reference + ^^^^^^^^^^^^^ + + The reference guide contains a detailed description of the functions, + modules, and objects included in QToolKit. The reference describes how the + methods work and which parameters can be used. It assumes that you have an + understanding of the key concepts. + + +++ + + .. button-ref:: api + :expand: + :color: secondary + :click-parent: + + To the reference guide + + .. grid-item-card:: + :img-top: ../source/_static/index-images/contributor.svg + + Contributor's Guide + ^^^^^^^^^^^^^^^^^^^ + + Want to add to the codebase? Can help add support to an additional DRM system? + The contributing guidelines will guide you through the + process of improving QToolKit. + + +++ + + .. button-ref:: devindex + :expand: + :color: secondary + :click-parent: + + To the contributor's guide + +.. This is not really the index page, that is found in + _templates/indexcontent.html The toctree content here will be added to the + top of the template header \ No newline at end of file diff --git a/doc/source/license.rst b/doc/source/license.rst new file mode 100644 index 0000000..f36eb5b --- /dev/null +++ b/doc/source/license.rst @@ -0,0 +1,6 @@ +**************** +QToolKit license +**************** + +.. include:: ../../LICENSE + :literal: diff --git a/doc/source/user/basics.rst b/doc/source/user/basics.rst new file mode 100644 index 0000000..67af9c2 --- /dev/null +++ b/doc/source/user/basics.rst @@ -0,0 +1,12 @@ +********************* +QToolKit fundamentals +********************* + +These documents clarify concepts, design decisions, and technical +constraints in QToolKit. This is a great place to understand the +fundamental QToolKit ideas and philosophy. + +.. + .. toctree:: + :maxdepth: 1 + diff --git a/doc/source/user/building.rst b/doc/source/user/building.rst new file mode 100644 index 0000000..93af682 --- /dev/null +++ b/doc/source/user/building.rst @@ -0,0 +1,7 @@ +.. _building-from-source: + +Building from source +==================== + +Get the source from the git repository. +Install it with pip install . \ No newline at end of file diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst new file mode 100644 index 0000000..5155edf --- /dev/null +++ b/doc/source/user/index.rst @@ -0,0 +1,30 @@ +.. _user: + +################### +QToolKit user guide +################### + +This guide is an overview and explains the important features; +details are found in :ref:`reference`. + +.. toctree:: + :caption: Getting started + :maxdepth: 1 + + whatisqtoolkit + install + quickstart + +.. toctree:: + :caption: Advanced usage and interoperability + :maxdepth: 1 + + building + + +.. toctree:: + :hidden: + :caption: Extras + + ../glossary + ../license diff --git a/doc/source/user/install.rst b/doc/source/user/install.rst new file mode 100644 index 0000000..50cd00e --- /dev/null +++ b/doc/source/user/install.rst @@ -0,0 +1,9 @@ +.. _install: + +******************* +Installing QToolKit +******************* + +The only prerequisite for installing QToolKit is Python. + +QToolKit can be installed with 'conda', with 'pip', or from source. \ No newline at end of file diff --git a/doc/source/user/quickstart.rst b/doc/source/user/quickstart.rst new file mode 100644 index 0000000..da9978c --- /dev/null +++ b/doc/source/user/quickstart.rst @@ -0,0 +1,17 @@ +.. _quickstart: + +=================== +QToolKit quickstart +=================== + +Prerequisites +============= +You need python + +The Basics +========== + +Create an easy script + +ADD OTHER TOPICS: +remote submission, resources, ... \ No newline at end of file diff --git a/doc/source/user/whatisqtoolkit.rst b/doc/source/user/whatisqtoolkit.rst new file mode 100644 index 0000000..a302ec4 --- /dev/null +++ b/doc/source/user/whatisqtoolkit.rst @@ -0,0 +1,8 @@ +.. _whatisqtoolkit: + +================= +What is QToolKit? +================= + +QToolKit is ... +TODO: add the features that it has. \ No newline at end of file diff --git a/doc_requirements.txt b/doc_requirements.txt new file mode 100644 index 0000000..53b03cc --- /dev/null +++ b/doc_requirements.txt @@ -0,0 +1,14 @@ +# doxygen required, use apt-get or dnf +sphinx>=4.5.0 +numpydoc==1.4 +pydata-sphinx-theme==0.13.3 +sphinx-design +sphinx-apidoc +ipython!=8.1.0 +scipy +matplotlib +pandas +breathe + +# needed to build release notes +towncrier diff --git a/src/qtoolkit/core/__init__.py b/src/qtoolkit/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/qtoolkit/host/__init__.py b/src/qtoolkit/host/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/qtoolkit/queue/__init__.py b/src/qtoolkit/queue/__init__.py new file mode 100644 index 0000000..e69de29 From b0650d4852cb39c22418749392c813e6b4804e91 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Fri, 14 Apr 2023 17:10:26 +0200 Subject: [PATCH 06/19] Fixed navigation for api. --- doc/source/api/index.rst | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/doc/source/api/index.rst b/doc/source/api/index.rst index daebaf2..484de6a 100644 --- a/doc/source/api/index.rst +++ b/doc/source/api/index.rst @@ -1,15 +1,9 @@ .. _api: -============= +############# API Reference -============= +############# This is the API reference -.. toctree:: - :maxdepth: 1 - - qtoolkit.core - qtoolkit.host - qtoolkit.io - qtoolkit.queue \ No newline at end of file +.. include:: qtoolkit.rst \ No newline at end of file From 9a76d824ee335812e8baa2fa7258d80f062322ce Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Sat, 29 Apr 2023 23:39:55 +0200 Subject: [PATCH 07/19] Updated css styling (Matgenix colors). --- doc/Makefile | 2 +- doc/source/_static/qtoolkit.css | 15 +++++++++++++++ doc/source/index.rst | 12 ++++++------ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/doc/Makefile b/doc/Makefile index 90d6037..3e5f5da 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -69,4 +69,4 @@ html-build: generate @echo @echo "Build finished. The HTML pages are in build/html." api-doc: - sphinx-apidoc -f -o source/api ../src/qtoolkit + sphinx-apidoc -e -f -o source/api ../src/qtoolkit diff --git a/doc/source/_static/qtoolkit.css b/doc/source/_static/qtoolkit.css index 4a77b9c..4561c96 100644 --- a/doc/source/_static/qtoolkit.css +++ b/doc/source/_static/qtoolkit.css @@ -1,5 +1,10 @@ @import url('https://fonts.googleapis.com/css2?family=Lato:ital,wght@0,400;0,700;0,900;1,400;1,700;1,900&family=Open+Sans:ital,wght@0,400;0,600;1,400;1,600&display=swap'); +:root { + --matgenix-color: #46b3c1; + --matgenix-dark-color: #338d99; +} + .navbar-brand img { height: 75px; } @@ -145,3 +150,13 @@ html[data-theme=dark] h1 { html[data-theme=dark] h3 { color: #0a6774; } + +.sd-btn-secondary { + background-color: var(--matgenix-color) !important; + border-color: var(--matgenix-color) !important; +} + +.sd-btn-secondary:hover, .sd-btn-secondary:focus { + background-color: var(--matgenix-dark-color) !important; + border-color: var(--matgenix-dark-color) !important; +} \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst index 39690e7..46f8eef 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -32,17 +32,17 @@ SLURM, PBS, ... Getting Started ^^^^^^^^^^^^^^^ - New to QToolKit? Check out the Absolute Beginner's Guide. It contains an - introduction to QToolKit's main concepts and links to additional tutorials. + If you want to get started quickly, check out our quickstart section. + It contains an introduction to QToolKit's main concepts. +++ - .. button-ref:: user/absolute_beginners + .. button-ref:: user/quickstart :expand: :color: secondary :click-parent: - To the absolute beginner's guide + Quickstart .. grid-item-card:: :img-top: ../source/_static/index-images/user_guide.svg @@ -60,7 +60,7 @@ SLURM, PBS, ... :color: secondary :click-parent: - To the user guide + User Guide .. grid-item-card:: :img-top: ../source/_static/index-images/api.svg @@ -80,7 +80,7 @@ SLURM, PBS, ... :color: secondary :click-parent: - To the reference guide + API Reference .. grid-item-card:: :img-top: ../source/_static/index-images/contributor.svg From 5b1c02ecbccfdeab9a1666409c59a5998608c02c Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Sat, 29 Apr 2023 23:50:19 +0200 Subject: [PATCH 08/19] Added optional deps for docs. --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1432ea9..92453e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,11 @@ tests = [ maintain = [ "git-changelog>=0.6", ] +docs = [ + "sphinx", + "sphinx_design", + "pydata-sphinx-theme", +] strict = [] remote = ["fabric>=3.0.0"] msonable = ["monty>=2022.9.9",] From 9c30d5fb57c11ee25daf7eea38b9d8428c28172a Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 7 Jun 2023 10:45:12 +0200 Subject: [PATCH 09/19] Adding some tests and documentation. --- src/qtoolkit/io/base.py | 2 +- tests/conftest.py | 14 ++- tests/core/__init__.py | 0 tests/core/test_base.py | 10 +- tests/core/test_data_objects.py | 211 ++++++++++++++++++++++++++++++++ tests/io/__init__.py | 0 tests/io/test_base.py | 107 ++++++++++++++++ 7 files changed, 335 insertions(+), 9 deletions(-) create mode 100644 tests/core/__init__.py create mode 100644 tests/core/test_data_objects.py create mode 100644 tests/io/__init__.py create mode 100644 tests/io/test_base.py diff --git a/src/qtoolkit/io/base.py b/src/qtoolkit/io/base.py index 3c4ee5d..5848478 100644 --- a/src/qtoolkit/io/base.py +++ b/src/qtoolkit/io/base.py @@ -37,7 +37,7 @@ def get_identifiers(self) -> list: return ids -class BaseSchedulerIO(QBase): +class BaseSchedulerIO(QBase, abc.ABC): """Base class for job queues. Attributes diff --git a/tests/conftest.py b/tests/conftest.py index c9305da..372d7a6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -82,15 +82,23 @@ class TestUtils: from monty.serialization import MontyDecoder, MontyEncoder @classmethod - def is_msonable(cls, obj): + def is_msonable(cls, obj, obj_cls=None): if not isinstance(obj, cls.MSONable): return False obj_dict = obj.as_dict() if not obj_dict == obj.__class__.from_dict(obj_dict).as_dict(): return False json_string = cls.json.dumps(obj_dict, cls=cls.MontyEncoder) - obj_dict_from_json = cls.json.loads(json_string, cls=cls.MontyDecoder) - return obj_dict_from_json == obj_dict + obj_from_json = cls.json.loads(json_string, cls=cls.MontyDecoder) + # When the class is defined as an inner class, the MontyDecoder is unable + # to find it automatically. This is only used in the core/test_base tests. + # The next check on the type of the obj_from_json is of course not relevant + # in that specific case. + if obj_cls is not None: + obj_from_json = obj_cls.from_dict(obj_from_json) + if not isinstance(obj_from_json, obj.__class__): + return False + return obj_from_json == obj @classmethod def inkwargs_outref(cls, in_out_ref, inkey, outkey): diff --git a/tests/core/__init__.py b/tests/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/core/test_base.py b/tests/core/test_base.py index 052930e..1ed4e8b 100644 --- a/tests/core/test_base.py +++ b/tests/core/test_base.py @@ -51,7 +51,7 @@ class QClass(qbase.QBase): name: str = "name" qc = QClass() - assert test_utils.is_msonable(qc) + assert test_utils.is_msonable(qc, obj_cls=QClass) def test_not_msonable(self, test_utils, qtk_core_base_mocked_monty_not_found): @dataclass @@ -72,11 +72,11 @@ class SomeEnum(qbase.QEnum): VAL2 = "VAL2" se = SomeEnum("VAL1") - assert test_utils.is_msonable(se) + assert test_utils.is_msonable(se, obj_cls=SomeEnum) assert isinstance(se, enum.Enum) se = SomeEnum.VAL2 - assert test_utils.is_msonable(se) + assert test_utils.is_msonable(se, obj_cls=SomeEnum) assert isinstance(se, enum.Enum) class SomeEnum(qbase.QEnum): @@ -84,11 +84,11 @@ class SomeEnum(qbase.QEnum): VAL2 = 4 se = SomeEnum(3) - assert test_utils.is_msonable(se) + assert test_utils.is_msonable(se, obj_cls=SomeEnum) assert isinstance(se, enum.Enum) se = SomeEnum.VAL2 - assert test_utils.is_msonable(se) + assert test_utils.is_msonable(se, obj_cls=SomeEnum) assert isinstance(se, enum.Enum) def test_not_msonable(self, test_utils, qtk_core_base_mocked_monty_not_found): diff --git a/tests/core/test_data_objects.py b/tests/core/test_data_objects.py new file mode 100644 index 0000000..a680e33 --- /dev/null +++ b/tests/core/test_data_objects.py @@ -0,0 +1,211 @@ +"""Unit tests for the core.data_objects module of QToolKit.""" +import pytest + +from qtoolkit.core.data_objects import ( + CancelResult, + CancelStatus, + ProcessPlacement, + QState, + QSubState, + SubmissionResult, + SubmissionStatus, +) + +try: + import monty +except ModuleNotFoundError: + monty = None + + +class TestSubmissionStatus: + def test_status_list(self): + all_status = [status.value for status in SubmissionStatus] + assert set(all_status) == {"SUCCESSFUL", "FAILED", "JOB_ID_UNKNOWN"} + + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + status = SubmissionStatus("SUCCESSFUL") + assert test_utils.is_msonable(status) + + def test_equal(self): + assert SubmissionStatus("SUCCESSFUL") == SubmissionStatus.SUCCESSFUL + + +class TestSubmissionResult: + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + sr = SubmissionResult( + job_id="abc123", + step_id=1, + exit_code=1, + stdout="mystdout", + stderr="mystderr", + status=SubmissionStatus.FAILED, + ) + assert test_utils.is_msonable(sr) + + def test_equality(self): + sr1 = SubmissionResult( + job_id=None, + step_id=1, + exit_code=1, + stdout="mystdout", + stderr="mystderr", + status=SubmissionStatus.JOB_ID_UNKNOWN, + ) + sr2 = SubmissionResult( + job_id="abc123", + step_id=1, + exit_code=1, + stdout="mystdout", + stderr="mystderr", + status=SubmissionStatus.FAILED, + ) + sr3 = SubmissionResult( + job_id="abc123", + step_id=1, + exit_code=1, + stdout="mystdout", + stderr="mystderr", + status=SubmissionStatus.FAILED, + ) + assert sr2 == sr3 + assert sr1 != sr2 + assert sr1 != sr3 + + +class TestCancelStatus: + def test_status_list(self): + all_status = [status.value for status in CancelStatus] + assert set(all_status) == {"SUCCESSFUL", "FAILED", "JOB_ID_UNKNOWN"} + + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + status = CancelStatus("FAILED") + assert test_utils.is_msonable(status) + + def test_equal(self): + assert CancelStatus("JOB_ID_UNKNOWN") == CancelStatus.JOB_ID_UNKNOWN + assert CancelStatus("JOB_ID_UNKNOWN") != SubmissionStatus.JOB_ID_UNKNOWN + + +class TestCancelResult: + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + cr = CancelResult( + job_id="abc123", + step_id=1, + exit_code=1, + stdout="mystdout", + stderr="mystderr", + status=CancelStatus.FAILED, + ) + assert test_utils.is_msonable(cr) + + def test_equality(self): + cr1 = CancelResult( + job_id="abc123", + step_id=1, + exit_code=0, + stdout="mystdout", + stderr="mystderr", + status=CancelStatus.SUCCESSFUL, + ) + cr2 = CancelResult( + job_id="abc123", + step_id=1, + exit_code=1, + stdout="mystdout", + stderr="mystderr", + status=CancelStatus.FAILED, + ) + cr3 = CancelResult( + job_id="abc123", + step_id=1, + exit_code=1, + stdout="mystdout", + stderr="mystderr", + status=CancelStatus.FAILED, + ) + assert cr2 == cr3 + assert cr1 != cr2 + assert cr1 != cr3 + + +class TestQState: + def test_states_list(self): + all_states = [state.value for state in QState] + assert set(all_states) == { + "UNDETERMINED", + "QUEUED", + "QUEUED_HELD", + "RUNNING", + "SUSPENDED", + "REQUEUED", + "REQUEUED_HELD", + "DONE", + "FAILED", + } + + def test_equality(self): + state1 = QState("QUEUED") + state2 = QState("RUNNING") + state3 = QState.RUNNING + assert state1 != state2 + assert state1 != state3 + assert state2 == state3 + + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + state = QState.DONE + assert test_utils.is_msonable(state) + + +class SomeSubState(QSubState): + STATE_A = "stateA", "STA" + STATE_B = "stateB", "STB", "sb" + STATE_C = "sc" + + def qstate(self) -> QState: + return QState.DONE + + +class TestQSubState: + def test_equality(self): + sst1 = SomeSubState.STATE_B + sst2 = SomeSubState("stateB") + sst3 = SomeSubState("STB") + sst4 = SomeSubState("sb") + assert sst1 == sst2 + assert sst1 == sst3 + assert sst1 == sst4 + + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + sst1 = SomeSubState.STATE_A + assert test_utils.is_msonable(sst1) + sst2 = SomeSubState("sb") + assert test_utils.is_msonable(sst2) + + def test_repr(self): + sst1 = SomeSubState.STATE_B + sst2 = SomeSubState("stateB") + sst3 = SomeSubState("STB") + sst4 = SomeSubState("sb") + assert repr(sst1) == "" + assert repr(sst1) == repr(sst2) + assert repr(sst1) == repr(sst3) + assert repr(sst1) == repr(sst4) + + +class TestProcessPlacement: + def test_process_placement_list(self): + all_process_placements = [ + process_placement.value for process_placement in ProcessPlacement + ] + assert set(all_process_placements) == { + "NO_CONSTRAINTS", + "SCATTERED", + "SAME_NODE", + "EVENLY_DISTRIBUTED", + } diff --git a/tests/io/__init__.py b/tests/io/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/io/test_base.py b/tests/io/test_base.py new file mode 100644 index 0000000..b2f7365 --- /dev/null +++ b/tests/io/test_base.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest +from monty.serialization import loadfn + +from qtoolkit.core.data_objects import CancelResult, QJob, QResources, SubmissionResult +from qtoolkit.io.base import BaseSchedulerIO, QTemplate + +TEST_DIR = Path(__file__).resolve().parents[1] / "test_data" +ref_file = TEST_DIR / "io" / "slurm" / "parse_submit_cmd_inout.yaml" +in_out_ref_list = loadfn(ref_file) + + +def test_qtemplate(): + template_str = """This is a template + with some $$substitutions + another $${tata}te""" + template = QTemplate(template_str) + identifiers = set(template.get_identifiers()) + assert identifiers == {"substitutions", "tata"} + substituted_template = template.safe_substitute({"substitutions": "mysubstitution"}) + assert ( + substituted_template + == """This is a template + with some mysubstitution + another $${tata}te""" + ) + substituted_template = template.safe_substitute({}) + assert ( + substituted_template + == """This is a template + with some $$substitutions + another $${tata}te""" + ) + substituted_template = template.safe_substitute({"tata": "pata"}) + assert ( + substituted_template + == """This is a template + with some $$substitutions + another patate""" + ) + + template_str = """Multi template $$subst + $${subst1}$${subst2} + $${subst3}$${subst3}""" + template = QTemplate(template_str) + identifiers = template.get_identifiers() + assert len(identifiers) == 4 + assert set(identifiers) == {"subst", "subst1", "subst2", "subst3"} + substituted_template = template.safe_substitute({"subst3": "to", "subst": "bla"}) + assert ( + substituted_template + == """Multi template bla + $${subst1}$${subst2} + toto""" + ) + + +def test_base_scheduler(): + class MyScheduler(BaseSchedulerIO): + pass + + with pytest.raises(TypeError): + MyScheduler() + + class MyScheduler(BaseSchedulerIO): + header_template = """#SPECCMD --option1=$${option1} +#SPECCMD --option2=$${option2} +#SPECCMD --option3=$${option3}""" + + SUBMIT_CMD = "mysubmit" + CANCEL_CMD = "mycancel" + + def parse_submit_output(self, exit_code, stdout, stderr) -> SubmissionResult: + pass + + def parse_cancel_output(self, exit_code, stdout, stderr) -> CancelResult: + pass + + def _get_job_cmd(self, job_id: str) -> str: + pass + + def parse_job_output(self, exit_code, stdout, stderr) -> QJob | None: + pass + + def _convert_qresources(self, resources: QResources) -> dict: + pass + + @property + def supported_qresources_keys(self) -> list: + return [] + + def _get_jobs_list_cmd( + self, job_ids: list[str] | None = None, user: str | None = None + ) -> str: + pass + + def parse_jobs_list_output(self, exit_code, stdout, stderr) -> list[QJob]: + pass + + scheduler = MyScheduler() + + header = scheduler.generate_header({"option2": "value_option2"}) + assert header == """#SPECCMD --option2=value_option2""" + scheduler.parse_submit_output(0, "", "") From 5d39bf58b79d6c3fc7164d39802d6884cd45a83f Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 7 Jun 2023 14:51:03 +0200 Subject: [PATCH 10/19] More unit tests for data objects. --- src/qtoolkit/core/data_objects.py | 1 + tests/core/test_data_objects.py | 124 ++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) diff --git a/src/qtoolkit/core/data_objects.py b/src/qtoolkit/core/data_objects.py index 65129ef..9e1c042 100644 --- a/src/qtoolkit/core/data_objects.py +++ b/src/qtoolkit/core/data_objects.py @@ -151,6 +151,7 @@ def __post_init__(self): "plus processes_per_node or only processes" ) raise UnsupportedResourcesError(msg) + self.kwargs = self.kwargs or {} @classmethod def no_constraints(cls, processes, **kwargs): diff --git a/tests/core/test_data_objects.py b/tests/core/test_data_objects.py index a680e33..7466199 100644 --- a/tests/core/test_data_objects.py +++ b/tests/core/test_data_objects.py @@ -5,11 +5,13 @@ CancelResult, CancelStatus, ProcessPlacement, + QResources, QState, QSubState, SubmissionResult, SubmissionStatus, ) +from qtoolkit.core.exceptions import UnsupportedResourcesError try: import monty @@ -209,3 +211,125 @@ def test_process_placement_list(self): "SAME_NODE", "EVENLY_DISTRIBUTED", } + + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + pp1 = ProcessPlacement.SCATTERED + assert test_utils.is_msonable(pp1) + pp2 = ProcessPlacement("SAME_NODE") + assert test_utils.is_msonable(pp2) + + +class TestQResources: + def test_no_process_placement(self): + with pytest.raises( + UnsupportedResourcesError, + match=r"When process_placement is None either define only nodes " + r"plus processes_per_node or only processes", + ): + QResources(processes=8, nodes=2) + + with pytest.raises( + UnsupportedResourcesError, + match=r"When process_placement is None either define only nodes " + r"plus processes_per_node or only processes", + ): + QResources(processes=8, processes_per_node=2) + + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + qr1 = QResources( + queue_name="main", + job_name="myjob", + memory_per_thread=1024, + processes=16, + time_limit=86400, + ) + assert test_utils.is_msonable(qr1) + qr2 = QResources.evenly_distributed(nodes=4, processes_per_node=8) + assert test_utils.is_msonable(qr2) + + def test_no_constraints(self): + qr = QResources.no_constraints(processes=16, queue_name="main") + assert qr.queue_name == "main" + assert qr.process_placement == ProcessPlacement.NO_CONSTRAINTS + + with pytest.raises( + UnsupportedResourcesError, + match=r"nodes and processes_per_node are incompatible with no constraints jobs", + ): + QResources.no_constraints(processes=16, nodes=4) + + with pytest.raises( + UnsupportedResourcesError, + match=r"nodes and processes_per_node are incompatible with no constraints jobs", + ): + QResources.no_constraints(processes=16, processes_per_node=2) + + def test_evenly_distributed(self): + qr = QResources.evenly_distributed( + nodes=4, processes_per_node=2, queue_name="main" + ) + assert qr.queue_name == "main" + assert qr.process_placement == ProcessPlacement.EVENLY_DISTRIBUTED + + with pytest.raises( + UnsupportedResourcesError, + match=r"processes is incompatible with evenly distributed jobs", + ): + QResources.evenly_distributed(nodes=4, processes_per_node=2, processes=12) + + def test_scattered(self): + qr = QResources.scattered(processes=16, queue_name="main") + assert qr.queue_name == "main" + assert qr.process_placement == ProcessPlacement.SCATTERED + + with pytest.raises( + UnsupportedResourcesError, + match=r"nodes and processes_per_node are incompatible with scattered jobs", + ): + QResources.scattered(processes=16, nodes=4) + + with pytest.raises( + UnsupportedResourcesError, + match=r"nodes and processes_per_node are incompatible with scattered jobs", + ): + QResources.scattered(processes=16, processes_per_node=4) + + def test_same_node(self): + qr = QResources.same_node(processes=16, queue_name="main") + assert qr.queue_name == "main" + assert qr.process_placement == ProcessPlacement.SAME_NODE + + with pytest.raises( + UnsupportedResourcesError, + match=r"nodes and processes_per_node are incompatible with same node jobs", + ): + QResources.same_node(processes=16, nodes=4) + + with pytest.raises( + UnsupportedResourcesError, + match=r"nodes and processes_per_node are incompatible with same node jobs", + ): + QResources.same_node(processes=16, processes_per_node=4) + + def test_equality(self): + qr1 = QResources.evenly_distributed( + nodes=4, processes_per_node=4, job_name="myjob" + ) + qr2 = QResources( + nodes=4, + processes_per_node=4, + job_name="myjob", + process_placement=ProcessPlacement.EVENLY_DISTRIBUTED, + ) + qr3 = QResources(nodes=4, processes_per_node=4, job_name="myjob") + qr4 = QResources( + nodes=4, + processes_per_node=4, + job_name="myjob", + process_placement=ProcessPlacement.SAME_NODE, + ) + assert qr1 == qr2 + assert qr1 == qr3 + assert qr1 != qr4 From 498dffafe9e743866f0bca9364d4ef587322a672 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 7 Jun 2023 15:18:45 +0200 Subject: [PATCH 11/19] Unit tests for data objects. --- tests/core/test_data_objects.py | 150 ++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/tests/core/test_data_objects.py b/tests/core/test_data_objects.py index 7466199..8d628b8 100644 --- a/tests/core/test_data_objects.py +++ b/tests/core/test_data_objects.py @@ -5,6 +5,8 @@ CancelResult, CancelStatus, ProcessPlacement, + QJob, + QJobInfo, QResources, QState, QSubState, @@ -333,3 +335,151 @@ def test_equality(self): assert qr1 == qr2 assert qr1 == qr3 assert qr1 != qr4 + + def test_get_processes_distribution(self): + qr = QResources(nodes=4, processes_per_node=2) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [4, None, 2] + qr = QResources(processes=18) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [None, 18, None] + qr = QResources.scattered(processes=12) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [12, 12, 1] + qr = QResources.same_node(processes=12) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [1, 12, 12] + qr = QResources.evenly_distributed(nodes=4, processes_per_node=8) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [4, None, 8] + qr = QResources.no_constraints(processes=14) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [None, 14, None] + + +class TestQJobInfo: + def test_equality(self): + qji1 = QJobInfo( + memory=2000, + memory_per_cpu=500, + nodes=2, + cpus=4, + threads_per_process=2, + time_limit=3600, + ) + qji2 = QJobInfo( + memory=2000, + memory_per_cpu=500, + nodes=2, + cpus=4, + threads_per_process=2, + time_limit=3600, + ) + qji3 = QJobInfo( + memory=4000, + memory_per_cpu=500, + nodes=2, + cpus=4, + threads_per_process=2, + time_limit=3600, + ) + assert qji1 == qji2 + assert qji1 != qji3 + + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + qji = QJobInfo( + memory=2000, + memory_per_cpu=500, + nodes=2, + cpus=4, + threads_per_process=2, + time_limit=3600, + ) + assert test_utils.is_msonable(qji) + + +class TestQJob: + def test_equality(self): + qji1 = QJobInfo( + memory=2000, + memory_per_cpu=500, + nodes=2, + cpus=4, + threads_per_process=2, + time_limit=3600, + ) + qji2 = QJobInfo( + memory=2000, + memory_per_cpu=500, + nodes=2, + cpus=4, + threads_per_process=2, + time_limit=3600, + ) + qji3 = QJobInfo( + memory=4000, + memory_per_cpu=500, + nodes=2, + cpus=4, + threads_per_process=2, + time_limit=3600, + ) + qjob1 = QJob( + name="job1", + job_id="id1", + exit_status=0, + state=QState.DONE, + sub_state=SomeSubState.STATE_A, + info=qji1, + account="myacc", + runtime=2541, + queue_name="mymain", + ) + qjob2 = QJob( + name="job1", + job_id="id1", + exit_status=0, + state=QState.DONE, + sub_state=SomeSubState.STATE_A, + info=qji2, + account="myacc", + runtime=2541, + queue_name="mymain", + ) + qjob3 = QJob( + name="job1", + job_id="id1", + exit_status=0, + state=QState.DONE, + sub_state=SomeSubState.STATE_A, + info=qji3, + account="myacc", + runtime=2541, + queue_name="mymain", + ) + assert qjob1 == qjob2 + assert qjob1 != qjob3 + + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + qji = QJobInfo( + memory=2000, + memory_per_cpu=500, + nodes=2, + cpus=4, + threads_per_process=2, + time_limit=3600, + ) + qjob = QJob( + name="job1", + job_id="id1", + exit_status=0, + state=QState.DONE, + sub_state=SomeSubState.STATE_A, + info=qji, + account="myacc", + runtime=2541, + queue_name="mymain", + ) + assert test_utils.is_msonable(qjob) From 624bb08b54272e4f5e42e78d1e16bc9bab14c888 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 7 Jun 2023 15:21:17 +0200 Subject: [PATCH 12/19] Removed unused core.jobs module --- src/qtoolkit/core/jobs.py | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 src/qtoolkit/core/jobs.py diff --git a/src/qtoolkit/core/jobs.py b/src/qtoolkit/core/jobs.py deleted file mode 100644 index e559c61..0000000 --- a/src/qtoolkit/core/jobs.py +++ /dev/null @@ -1,20 +0,0 @@ -# from __future__ import annotations -# -# from dataclasses import dataclass -# -# from qtoolkit.core.base import QBase -# from qtoolkit.core.data_objects import QState, QSubState, QResources, QJobInfo -# -# -# TODO: this has been moved to data_objects for now. See if it should be here -# for some reason ? -# @dataclass -# class QJob(QBase): -# name: str -# qid: str | None -# exit_status: int | None -# state: QState | None # Standard -# sub_state: QSubState | None -# queue: str | None -# resources: QResources | None -# job_info: QJobInfo | None From b55f16748164e9ac7adb8bef190d353a8c9fc4da Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 7 Jun 2023 15:28:55 +0200 Subject: [PATCH 13/19] Removed unused queue module (directory with __init__.py) --- src/qtoolkit/queue/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/qtoolkit/queue/__init__.py diff --git a/src/qtoolkit/queue/__init__.py b/src/qtoolkit/queue/__init__.py deleted file mode 100644 index e69de29..0000000 From cdff324c0d9ebddfc7ab048aa701106b46615b81 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 7 Jun 2023 15:46:43 +0200 Subject: [PATCH 14/19] Unit tests for base scheduler io. --- src/qtoolkit/io/base.py | 2 +- tests/io/test_base.py | 26 +++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/qtoolkit/io/base.py b/src/qtoolkit/io/base.py index 5848478..b437215 100644 --- a/src/qtoolkit/io/base.py +++ b/src/qtoolkit/io/base.py @@ -243,7 +243,7 @@ def generate_ids_list(self, jobs: list[QJob | int | str] | None) -> list[str]: ids_list = [] for j in jobs: if isinstance(j, QJob): - ids_list.append(j.job_id) + ids_list.append(str(j.job_id)) else: ids_list.append(str(j)) diff --git a/tests/io/test_base.py b/tests/io/test_base.py index b2f7365..e549ef8 100644 --- a/tests/io/test_base.py +++ b/tests/io/test_base.py @@ -104,4 +104,28 @@ def parse_jobs_list_output(self, exit_code, stdout, stderr) -> list[QJob]: header = scheduler.generate_header({"option2": "value_option2"}) assert header == """#SPECCMD --option2=value_option2""" - scheduler.parse_submit_output(0, "", "") + + ids_list = scheduler.generate_ids_list( + [QJob(job_id=4), QJob(job_id="job_id_abc1"), 215, "job12345"] + ) + assert ids_list == ["4", "job_id_abc1", "215", "job12345"] + + submit_cmd = scheduler.get_submit_cmd() + assert submit_cmd == "mysubmit submit.script" + submit_cmd = scheduler.get_submit_cmd(script_file="sub.sh") + assert submit_cmd == "mysubmit sub.sh" + + cancel_cmd = scheduler.get_cancel_cmd(QJob(job_id=5)) + assert cancel_cmd == "mycancel 5" + cancel_cmd = scheduler.get_cancel_cmd(QJob(job_id="abc1")) + assert cancel_cmd == "mycancel abc1" + cancel_cmd = scheduler.get_cancel_cmd("jobid2") + assert cancel_cmd == "mycancel jobid2" + cancel_cmd = scheduler.get_cancel_cmd(632) + assert cancel_cmd == "mycancel 632" + + with pytest.raises( + ValueError, + match=r"The id of the job to be cancelled should be defined. Received: None", + ): + scheduler.get_cancel_cmd(job=None) From bc84aa772ea763615ed161aa1cdcbb0f2fc3b67e Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 7 Jun 2023 16:02:17 +0200 Subject: [PATCH 15/19] Started tests for shell io. --- tests/conftest.py | 6 +++++- tests/io/test_shell.py | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 tests/io/test_shell.py diff --git a/tests/conftest.py b/tests/conftest.py index 372d7a6..a267056 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,5 @@ +from dataclasses import is_dataclass +from enum import Enum from pathlib import Path import pytest @@ -98,7 +100,9 @@ def is_msonable(cls, obj, obj_cls=None): obj_from_json = obj_cls.from_dict(obj_from_json) if not isinstance(obj_from_json, obj.__class__): return False - return obj_from_json == obj + if is_dataclass(obj) or isinstance(obj, Enum): + return obj_from_json == obj + return obj_from_json.as_dict() == obj.as_dict() @classmethod def inkwargs_outref(cls, in_out_ref, inkey, outkey): diff --git a/tests/io/test_shell.py b/tests/io/test_shell.py new file mode 100644 index 0000000..aedfd18 --- /dev/null +++ b/tests/io/test_shell.py @@ -0,0 +1,24 @@ +import pytest + +try: + import monty +except ModuleNotFoundError: + monty = None + + +from qtoolkit.io.shell import ShellIO + + +class TestShellIO: + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + shell_io = ShellIO() + assert test_utils.is_msonable(shell_io) + + def test_get_submit_cmd(self): + shell_io = ShellIO(blocking=True) + submit_cmd = shell_io.get_submit_cmd(script_file="myscript.sh") + assert submit_cmd == "bash myscript.sh > stdout 2> stderr" + shell_io = ShellIO(blocking=False) + submit_cmd = shell_io.get_submit_cmd(script_file="myscript.sh") + assert submit_cmd == "nohup bash myscript.sh > stdout 2> stderr & echo $!" From c2f19ed5f04d7d6445276a88e6bd8830e2af7313 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Thu, 8 Jun 2023 09:37:17 +0200 Subject: [PATCH 16/19] Set sphinx language to english (en) in conf.py --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 8ace3c5..3c58151 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -78,7 +78,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. From a21a80673324bcb8ff1e39cf7e908953210de250 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Thu, 8 Jun 2023 12:50:30 +0200 Subject: [PATCH 17/19] More unit tests for shell IO. --- src/qtoolkit/io/shell.py | 5 +- tests/io/test_shell.py | 146 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 147 insertions(+), 4 deletions(-) diff --git a/src/qtoolkit/io/shell.py b/src/qtoolkit/io/shell.py index d3fc6cd..0a7ab7b 100644 --- a/src/qtoolkit/io/shell.py +++ b/src/qtoolkit/io/shell.py @@ -37,7 +37,7 @@ class ShellState(QSubState): STOPPED = "T" STOPPED_DEBUGGER = "t" PAGING = "W" - DEAD = "D" + DEAD = "X" DEFUNCT = "Z" @property @@ -172,6 +172,9 @@ def _get_jobs_list_cmd( return " ".join(command) def parse_jobs_list_output(self, exit_code, stdout, stderr) -> list[QJob]: + print("exit_code", (type(exit_code), repr(exit_code))) + print("stdout", (type(stdout), repr(stdout))) + print("stderr", (type(stderr), repr(stderr))) if isinstance(stdout, bytes): stdout = stdout.decode() diff --git a/tests/io/test_shell.py b/tests/io/test_shell.py index aedfd18..30017e0 100644 --- a/tests/io/test_shell.py +++ b/tests/io/test_shell.py @@ -6,13 +6,54 @@ monty = None -from qtoolkit.io.shell import ShellIO +from qtoolkit.core.data_objects import ( + CancelResult, + CancelStatus, + QJob, + QState, + SubmissionResult, + SubmissionStatus, +) +from qtoolkit.io.shell import ShellIO, ShellState -class TestShellIO: +@pytest.fixture(scope="module") +def shell_io(): + return ShellIO() + + +class TestShellState: @pytest.mark.skipif(monty is None, reason="monty is not installed") def test_msonable(self, test_utils): - shell_io = ShellIO() + shell_state = ShellState.DEFUNCT + assert test_utils.is_msonable(shell_state) + + def test_states_list(self): + all_states = [state.value for state in ShellState] + assert set(all_states) == {"D", "R", "S", "T", "t", "W", "X", "Z"} + + def test_qstate(self): + shell_state = ShellState.UNINTERRUPTIBLE_SLEEP + assert shell_state.qstate == QState.RUNNING + shell_state = ShellState.RUNNING + assert shell_state.qstate == QState.RUNNING + shell_state = ShellState.INTERRUPTIBLE_SLEEP + assert shell_state.qstate == QState.RUNNING + shell_state = ShellState.STOPPED + assert shell_state.qstate == QState.SUSPENDED + shell_state = ShellState.STOPPED_DEBUGGER + assert shell_state.qstate == QState.SUSPENDED + shell_state = ShellState.PAGING + assert shell_state.qstate == QState.RUNNING + shell_state = ShellState.DEAD + assert shell_state.qstate == QState.DONE + shell_state = ShellState.DEFUNCT + assert shell_state.qstate == QState.DONE + + +class TestShellIO: + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils, shell_io): assert test_utils.is_msonable(shell_io) def test_get_submit_cmd(self): @@ -22,3 +63,102 @@ def test_get_submit_cmd(self): shell_io = ShellIO(blocking=False) submit_cmd = shell_io.get_submit_cmd(script_file="myscript.sh") assert submit_cmd == "nohup bash myscript.sh > stdout 2> stderr & echo $!" + + def test_parse_submit_output(self, shell_io): + sr = shell_io.parse_submit_output(exit_code=0, stdout="13647\n", stderr="") + assert sr == SubmissionResult( + job_id="13647", + step_id=None, + exit_code=0, + stdout="13647\n", + stderr="", + status=SubmissionStatus.SUCCESSFUL, + ) + sr = shell_io.parse_submit_output(exit_code=0, stdout=b"13647\n", stderr=b"") + assert sr == SubmissionResult( + job_id="13647", + step_id=None, + exit_code=0, + stdout="13647\n", + stderr="", + status=SubmissionStatus.SUCCESSFUL, + ) + sr = shell_io.parse_submit_output(exit_code=104, stdout="tata", stderr=b"titi") + assert sr == SubmissionResult( + job_id=None, + step_id=None, + exit_code=104, + stdout="tata", + stderr="titi", + status=SubmissionStatus.FAILED, + ) + sr = shell_io.parse_submit_output(exit_code=0, stdout=b"\n", stderr="") + assert sr == SubmissionResult( + job_id=None, + step_id=None, + exit_code=0, + stdout="\n", + stderr="", + status=SubmissionStatus.JOB_ID_UNKNOWN, + ) + + def test_parse_cancel_output(self, shell_io): + cr = shell_io.parse_cancel_output(exit_code=0, stdout="", stderr="") + assert cr == CancelResult( + job_id=None, + step_id=None, + exit_code=0, + stdout="", + stderr="", + status=CancelStatus.SUCCESSFUL, + ) + cr = shell_io.parse_cancel_output( + exit_code=1, + stdout=b"", + stderr=b"/bin/sh: line 1: kill: (14020) - No such process\n", + ) + assert cr == CancelResult( + job_id=None, + step_id=None, + exit_code=1, + stdout="", + stderr="/bin/sh: line 1: kill: (14020) - No such process\n", + status=CancelStatus.FAILED, + ) + + def test_get_job_cmd(self, shell_io): + get_job_cmd = shell_io.get_job_cmd(123) + assert get_job_cmd == "ps -o pid,user,etimes,state,comm -p 123" + get_job_cmd = shell_io.get_job_cmd(456) + assert get_job_cmd == "ps -o pid,user,etimes,state,comm -p 456" + get_job_cmd = shell_io.get_job_cmd(QJob(job_id="789")) + assert get_job_cmd == "ps -o pid,user,etimes,state,comm -p 789" + + def test_get_jobs_list_cmd(self, shell_io): + get_jobs_list_cmd = shell_io.get_jobs_list_cmd( + jobs=[QJob(job_id=125), 126, "127"], user=None + ) + assert get_jobs_list_cmd == "ps -o pid,user,etimes,state,comm -p 125,126,127" + + def test_parse_jobs_list_output(self, shell_io): + joblist = shell_io.parse_jobs_list_output( + exit_code=0, + stdout=" PID USER ELAPSED S COMMAND\n 18092 davidwa+ 465 S bash\n 18112 davidwa+ 461 S bash\n", + stderr="", + ) + assert joblist == [ + QJob( + job_id="18092", + runtime=465, + name="bash", + state=QState.RUNNING, + sub_state=ShellState.INTERRUPTIBLE_SLEEP, + ), + QJob( + job_id="18112", + runtime=461, + name="bash", + state=QState.RUNNING, + sub_state=ShellState.INTERRUPTIBLE_SLEEP, + ), + ] From d1646b532ba9cbec3db1e9e35bbd6dac33cbd163 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Thu, 8 Jun 2023 15:38:23 +0200 Subject: [PATCH 18/19] More tests for shell io + few docstrings additions. --- src/qtoolkit/io/base.py | 2 +- src/qtoolkit/io/shell.py | 27 +++++++++++++++++++++++---- tests/io/test_shell.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/qtoolkit/io/base.py b/src/qtoolkit/io/base.py index b437215..f442f35 100644 --- a/src/qtoolkit/io/base.py +++ b/src/qtoolkit/io/base.py @@ -311,7 +311,7 @@ def check_convert_qresources(self, resources: QResources) -> dict: unsupported_options = not_none.difference(self.supported_qresources_keys) if unsupported_options: - msg = f"Keys not supported: {', '.join(unsupported_options)}" + msg = f"Keys not supported: {', '.join(sorted(unsupported_options))}" raise UnsupportedResourcesError(msg) return self._convert_qresources(resources) diff --git a/src/qtoolkit/io/shell.py b/src/qtoolkit/io/shell.py index 0a7ab7b..f605482 100644 --- a/src/qtoolkit/io/shell.py +++ b/src/qtoolkit/io/shell.py @@ -65,6 +65,17 @@ class ShellIO(BaseSchedulerIO): CANCEL_CMD: str | None = "kill -9" def __init__(self, blocking=False, stdout_path="stdout", stderr_path="stderr"): + """Construct the ShellIO object. + + Parameters + ---------- + blocking: bool + Whether the execution should be blocking. + stdout_path: str or Path + Path to the standard output file. + stderr_path: str or Path + Path to the standard error file. + """ self.blocking = blocking self.stdout_path = stdout_path self.stderr_path = stderr_path @@ -75,7 +86,8 @@ def get_submit_cmd(self, script_file: str | Path | None = "submit.script") -> st Parameters ---------- - script_file: (str) path of the script file to use. + script_file: str or Path + Path of the script file to use. """ script_file = script_file or "" @@ -172,10 +184,17 @@ def _get_jobs_list_cmd( return " ".join(command) def parse_jobs_list_output(self, exit_code, stdout, stderr) -> list[QJob]: - print("exit_code", (type(exit_code), repr(exit_code))) - print("stdout", (type(stdout), repr(stdout))) - print("stderr", (type(stderr), repr(stderr))) + """Parse the output of the ps command to list jobs. + Parameters + ---------- + exit_code : int + Exit code of the ps command. + stdout : str + Standard output of the ps command. + stderr : str + Standard error of the ps command. + """ if isinstance(stdout, bytes): stdout = stdout.decode() if isinstance(stderr, bytes): diff --git a/tests/io/test_shell.py b/tests/io/test_shell.py index 30017e0..32de498 100644 --- a/tests/io/test_shell.py +++ b/tests/io/test_shell.py @@ -10,10 +10,16 @@ CancelResult, CancelStatus, QJob, + QResources, QState, SubmissionResult, SubmissionStatus, ) +from qtoolkit.core.exceptions import ( + CommandFailedError, + OutputParsingError, + UnsupportedResourcesError, +) from qtoolkit.io.shell import ShellIO, ShellState @@ -162,3 +168,27 @@ def test_parse_jobs_list_output(self, shell_io): sub_state=ShellState.INTERRUPTIBLE_SLEEP, ), ] + with pytest.raises( + CommandFailedError, match=r"command ps failed: string in stderr" + ): + shell_io.parse_jobs_list_output( + exit_code=1, + stdout=b"", + stderr=b"string in stderr", + ) + with pytest.raises( + OutputParsingError, match=r"Unknown job state K for job id 18112" + ): + shell_io.parse_jobs_list_output( + exit_code=0, + stdout=b" PID USER ELAPSED S COMMAND\n 18092 davidwa+ 465 S bash\n 18112 davidwa+ 461 K bash\n", + stderr=b"", + ) + + def test_check_convert_qresources(self, shell_io): + qr = QResources(processes=1) + with pytest.raises( + UnsupportedResourcesError, + match=r"Keys not supported: kwargs, process_placement, processes", + ): + shell_io.check_convert_qresources(qr) From 469f2c30205ed56735bf59dc38addaf9216658dc Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Thu, 8 Jun 2023 15:43:21 +0200 Subject: [PATCH 19/19] Renamed QBase to QTKObject and QEnum to QTKEnum. --- src/qtoolkit/core/base.py | 4 ++-- src/qtoolkit/core/data_objects.py | 24 ++++++++++++------------ src/qtoolkit/host/base.py | 6 +++--- src/qtoolkit/io/base.py | 4 ++-- src/qtoolkit/io/slurm.py | 2 +- src/qtoolkit/manager.py | 4 ++-- tests/core/test_base.py | 14 +++++++------- 7 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/qtoolkit/core/base.py b/src/qtoolkit/core/base.py index 35ec24e..1e63040 100644 --- a/src/qtoolkit/core/base.py +++ b/src/qtoolkit/core/base.py @@ -11,9 +11,9 @@ enum_superclses = (Enum,) # type: ignore -class QBase(supercls): # type: ignore +class QTKObject(supercls): # type: ignore pass -class QEnum(*enum_superclses): # type: ignore +class QTKEnum(*enum_superclses): # type: ignore pass diff --git a/src/qtoolkit/core/data_objects.py b/src/qtoolkit/core/data_objects.py index 9e1c042..ac4bc40 100644 --- a/src/qtoolkit/core/data_objects.py +++ b/src/qtoolkit/core/data_objects.py @@ -5,18 +5,18 @@ from datetime import timedelta from pathlib import Path -from qtoolkit.core.base import QBase, QEnum +from qtoolkit.core.base import QTKEnum, QTKObject from qtoolkit.core.exceptions import UnsupportedResourcesError -class SubmissionStatus(QEnum): +class SubmissionStatus(QTKEnum): SUCCESSFUL = "SUCCESSFUL" FAILED = "FAILED" JOB_ID_UNKNOWN = "JOB_ID_UNKNOWN" @dataclass -class SubmissionResult(QBase): +class SubmissionResult(QTKObject): job_id: int | str | None = None step_id: int | None = None exit_code: int | None = None @@ -25,14 +25,14 @@ class SubmissionResult(QBase): status: SubmissionStatus | None = None -class CancelStatus(QEnum): +class CancelStatus(QTKEnum): SUCCESSFUL = "SUCCESSFUL" FAILED = "FAILED" JOB_ID_UNKNOWN = "JOB_ID_UNKNOWN" @dataclass -class CancelResult(QBase): +class CancelResult(QTKObject): job_id: int | str | None = None step_id: int | None = None exit_code: int | None = None @@ -41,7 +41,7 @@ class CancelResult(QBase): status: CancelStatus | None = None -class QState(QEnum): +class QState(QTKEnum): """Enumeration of possible ("standardized") job states. These "standardized" states are based on the drmaa specification. @@ -64,7 +64,7 @@ class QState(QEnum): FAILED = "FAILED" -class QSubState(QEnum): +class QSubState(QTKEnum): """QSubState class defined without any enum values so it can be subclassed. These sub-states should be the actual job states in a given queuing system @@ -93,7 +93,7 @@ def qstate(self) -> QState: raise NotImplementedError -class ProcessPlacement(QEnum): +class ProcessPlacement(QTKEnum): NO_CONSTRAINTS = "NO_CONSTRAINTS" SCATTERED = "SCATTERED" SAME_NODE = "SAME_NODE" @@ -101,7 +101,7 @@ class ProcessPlacement(QEnum): @dataclass -class QResources(QBase): +class QResources(QTKObject): """Data defining resources for a given job (submitted or to be submitted). Attributes @@ -142,7 +142,7 @@ class QResources(QBase): def __post_init__(self): if self.process_placement is None: if self.processes and not self.processes_per_node and not self.nodes: - self.process_placement = ProcessPlacement.NO_CONSTRAINTS # type: ignore # due to QEnum + self.process_placement = ProcessPlacement.NO_CONSTRAINTS # type: ignore # due to QTKEnum elif self.nodes and self.processes_per_node and not self.processes: self.process_placement = ProcessPlacement.EVENLY_DISTRIBUTED else: @@ -239,7 +239,7 @@ def get_processes_distribution(self) -> list: @dataclass -class QJobInfo(QBase): +class QJobInfo(QTKObject): memory: int | None = None # in Kb memory_per_cpu: int | None = None # in Kb nodes: int | None = None @@ -249,7 +249,7 @@ class QJobInfo(QBase): @dataclass -class QJob(QBase): +class QJob(QTKObject): name: str | None = None job_id: str | None = None exit_status: int | None = None diff --git a/src/qtoolkit/host/base.py b/src/qtoolkit/host/base.py index 12218d6..fb29816 100644 --- a/src/qtoolkit/host/base.py +++ b/src/qtoolkit/host/base.py @@ -4,15 +4,15 @@ from dataclasses import dataclass from pathlib import Path -from qtoolkit.core.base import QBase +from qtoolkit.core.base import QTKObject @dataclass -class HostConfig(QBase): +class HostConfig(QTKObject): root_dir: str | Path -class BaseHost(QBase): +class BaseHost(QTKObject): """Base Host class.""" # def __init__(self, config, user): diff --git a/src/qtoolkit/io/base.py b/src/qtoolkit/io/base.py index f442f35..c9fd0d4 100644 --- a/src/qtoolkit/io/base.py +++ b/src/qtoolkit/io/base.py @@ -6,7 +6,7 @@ from pathlib import Path from string import Template -from qtoolkit.core.base import QBase +from qtoolkit.core.base import QTKObject from qtoolkit.core.data_objects import CancelResult, QJob, QResources, SubmissionResult from qtoolkit.core.exceptions import UnsupportedResourcesError @@ -37,7 +37,7 @@ def get_identifiers(self) -> list: return ids -class BaseSchedulerIO(QBase, abc.ABC): +class BaseSchedulerIO(QTKObject, abc.ABC): """Base class for job queues. Attributes diff --git a/src/qtoolkit/io/slurm.py b/src/qtoolkit/io/slurm.py index 41b1e99..8d81ad7 100644 --- a/src/qtoolkit/io/slurm.py +++ b/src/qtoolkit/io/slurm.py @@ -103,7 +103,7 @@ class SlurmState(QSubState): @property def qstate(self) -> QState: - # the type:ignore is required due to the dynamic class creation of QEnum + # the type:ignore is required due to the dynamic class creation of QTKEnum return _STATUS_MAPPING[self] # type: ignore diff --git a/src/qtoolkit/manager.py b/src/qtoolkit/manager.py index 80f7a07..ed50b77 100644 --- a/src/qtoolkit/manager.py +++ b/src/qtoolkit/manager.py @@ -2,14 +2,14 @@ from pathlib import Path -from qtoolkit.core.base import QBase +from qtoolkit.core.base import QTKObject from qtoolkit.core.data_objects import CancelResult, QJob, QResources, SubmissionResult from qtoolkit.host.base import BaseHost from qtoolkit.host.local import LocalHost from qtoolkit.io.base import BaseSchedulerIO -class QueueManager(QBase): +class QueueManager(QTKObject): """Base class for job queues. Attributes diff --git a/tests/core/test_base.py b/tests/core/test_base.py index 1ed4e8b..722d8a1 100644 --- a/tests/core/test_base.py +++ b/tests/core/test_base.py @@ -16,7 +16,7 @@ def qtk_core_base_mocked_monty_not_found(mocker): # Note: # Here we use importlib to dynamically import the qtoolkit.core.base module. - # We want to test the QBase and QEnum super classes with monty present or not. + # We want to test the QTKObject and QTKEnum super classes with monty present or not. # This is done by mocking the import. We then need to use importlib to reload # the qtoolkit.core.base module when we want to change the behaviour of the # the monty.json import inside the qtoolkit.core.base module (i.e. mocking the @@ -47,7 +47,7 @@ def test_msonable(self, test_utils): import qtoolkit.core.base as qbase @dataclass - class QClass(qbase.QBase): + class QClass(qbase.QTKObject): name: str = "name" qc = QClass() @@ -55,7 +55,7 @@ class QClass(qbase.QBase): def test_not_msonable(self, test_utils, qtk_core_base_mocked_monty_not_found): @dataclass - class QClass(qtk_core_base_mocked_monty_not_found.QBase): + class QClass(qtk_core_base_mocked_monty_not_found.QTKObject): name: str = "name" qc = QClass() @@ -67,7 +67,7 @@ class TestQEnum: def test_msonable(self, test_utils): import qtoolkit.core.base as qbase - class SomeEnum(qbase.QEnum): + class SomeEnum(qbase.QTKEnum): VAL1 = "VAL1" VAL2 = "VAL2" @@ -79,7 +79,7 @@ class SomeEnum(qbase.QEnum): assert test_utils.is_msonable(se, obj_cls=SomeEnum) assert isinstance(se, enum.Enum) - class SomeEnum(qbase.QEnum): + class SomeEnum(qbase.QTKEnum): VAL1 = 3 VAL2 = 4 @@ -92,7 +92,7 @@ class SomeEnum(qbase.QEnum): assert isinstance(se, enum.Enum) def test_not_msonable(self, test_utils, qtk_core_base_mocked_monty_not_found): - class SomeEnum(qtk_core_base_mocked_monty_not_found.QEnum): + class SomeEnum(qtk_core_base_mocked_monty_not_found.QTKEnum): VAL1 = "VAL1" VAL2 = "VAL2" @@ -104,7 +104,7 @@ class SomeEnum(qtk_core_base_mocked_monty_not_found.QEnum): assert not test_utils.is_msonable(se) assert isinstance(se, enum.Enum) - class SomeEnum(qtk_core_base_mocked_monty_not_found.QEnum): + class SomeEnum(qtk_core_base_mocked_monty_not_found.QTKEnum): VAL1 = 3 VAL2 = 4