From d494f3037fedcb86d09036efd0e3ff67721394d8 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Fri, 14 Apr 2023 17:01:59 +0200 Subject: [PATCH 01/34] Added structure for the documentation. --- .gitignore | 2 +- .pre-commit-config.yaml | 2 +- doc/Makefile | 72 ++++++ doc/source/_static/index-images/api.svg | 50 ++++ .../_static/index-images/contributor.svg | 13 ++ .../_static/index-images/getting_started.svg | 5 + .../_static/index-images/image_licences.txt | 4 + .../_static/index-images/user_guide.svg | 47 ++++ doc/source/_static/qtoolkit.css | 147 ++++++++++++ doc/source/api/index.rst | 15 ++ doc/source/conf.py | 216 ++++++++++++++++++ doc/source/dev/index.rst | 7 + doc/source/glossary.rst | 14 ++ doc/source/index.rst | 106 +++++++++ doc/source/license.rst | 6 + doc/source/user/basics.rst | 12 + doc/source/user/building.rst | 7 + doc/source/user/index.rst | 30 +++ doc/source/user/install.rst | 9 + doc/source/user/quickstart.rst | 17 ++ doc/source/user/whatisqtoolkit.rst | 8 + doc_requirements.txt | 14 ++ src/qtoolkit/core/__init__.py | 0 src/qtoolkit/host/__init__.py | 0 src/qtoolkit/queue/__init__.py | 0 25 files changed, 801 insertions(+), 2 deletions(-) create mode 100644 doc/Makefile create mode 100644 doc/source/_static/index-images/api.svg create mode 100644 doc/source/_static/index-images/contributor.svg create mode 100644 doc/source/_static/index-images/getting_started.svg create mode 100644 doc/source/_static/index-images/image_licences.txt create mode 100644 doc/source/_static/index-images/user_guide.svg create mode 100644 doc/source/_static/qtoolkit.css create mode 100644 doc/source/api/index.rst create mode 100644 doc/source/conf.py create mode 100644 doc/source/dev/index.rst create mode 100644 doc/source/glossary.rst create mode 100644 doc/source/index.rst create mode 100644 doc/source/license.rst create mode 100644 doc/source/user/basics.rst create mode 100644 doc/source/user/building.rst create mode 100644 doc/source/user/index.rst create mode 100644 doc/source/user/install.rst create mode 100644 doc/source/user/quickstart.rst create mode 100644 doc/source/user/whatisqtoolkit.rst create mode 100644 doc_requirements.txt create mode 100644 src/qtoolkit/core/__init__.py create mode 100644 src/qtoolkit/host/__init__.py create mode 100644 src/qtoolkit/queue/__init__.py diff --git a/.gitignore b/.gitignore index a14d6d0..373876b 100644 --- a/.gitignore +++ b/.gitignore @@ -69,7 +69,7 @@ instance/ .scrapy # Sphinx documentation -docs/_build/ +doc/_build/ # PyBuilder .pybuilder/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c59ed26..91c5c57 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ default_language_version: python: python3 -#exclude: '^src/{{ package_name }}/some/directory/' +exclude: '^doc/' repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000..90d6037 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,72 @@ +# Makefile for Sphinx documentation +# + +# PYVER needs to be major.minor, just "3" doesn't work - it will result in +# issues with the amendments to PYTHONPATH and install paths (see DIST_VARS). + +# Use explicit "version_info" indexing since make cannot handle colon characters, and +# evaluate it now to allow easier debugging when printing the variable + +PYVER:=$(shell python3 -c 'from sys import version_info as v; print("{0}.{1}".format(v[0], v[1]))') +PYTHON = python$(PYVER) + +# You can set these variables from the command line. +SPHINXOPTS ?= +SPHINXBUILD ?= LANG=C sphinx-build +PAPER ?= +# # For merging a documentation archive into a git checkout of numpy/doc +# # Turn a tag like v1.18.0 into 1.18 +# # Use sed -n -e 's/patttern/match/p' to return a blank value if no match +# TAG ?= $(shell git describe --tag | sed -n -e's,v\([1-9]\.[0-9]*\)\.[0-9].*,\1,p') + +FILES= + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -WT --keep-going -d build/doctrees $(PAPEROPT_$(PAPER)) \ + $(SPHINXOPTS) source + +.PHONY: help clean html version-check html-build + +#------------------------------------------------------------------------------ + +help: + @echo "Please use \`make ' where is one of" + @echo " clean to remove generated doc files and start fresh" + @echo " html to make standalone HTML files" + +clean: + -rm -rf build/* + find . -name generated -type d -prune -exec rm -rf "{}" ";" + + +#------------------------------------------------------------------------------ +# Automated generation of all documents +#------------------------------------------------------------------------------ + +# Build the current QToolKit version, and extract docs from it. +# We have to be careful of some issues: +# +# - Everything must be done using the same Python version +# + +#SPHINXBUILD="LANG=C sphinx-build" + + +#------------------------------------------------------------------------------ +# Basic Sphinx generation rules for different formats +#------------------------------------------------------------------------------ +generate: build/generate-stamp +build/generate-stamp: $(wildcard source/reference/*.rst) + mkdir -p build + touch build/generate-stamp + +html: api-doc html-build +html-build: generate + mkdir -p build/html build/doctrees + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) build/html $(FILES) + @echo + @echo "Build finished. The HTML pages are in build/html." +api-doc: + sphinx-apidoc -f -o source/api ../src/qtoolkit diff --git a/doc/source/_static/index-images/api.svg b/doc/source/_static/index-images/api.svg new file mode 100644 index 0000000..9c88397 --- /dev/null +++ b/doc/source/_static/index-images/api.svg @@ -0,0 +1,50 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/source/_static/index-images/contributor.svg b/doc/source/_static/index-images/contributor.svg new file mode 100644 index 0000000..ffd444e --- /dev/null +++ b/doc/source/_static/index-images/contributor.svg @@ -0,0 +1,13 @@ + + + + + + + + \ No newline at end of file diff --git a/doc/source/_static/index-images/getting_started.svg b/doc/source/_static/index-images/getting_started.svg new file mode 100644 index 0000000..20747f9 --- /dev/null +++ b/doc/source/_static/index-images/getting_started.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/doc/source/_static/index-images/image_licences.txt b/doc/source/_static/index-images/image_licences.txt new file mode 100644 index 0000000..85276bc --- /dev/null +++ b/doc/source/_static/index-images/image_licences.txt @@ -0,0 +1,4 @@ +getting_started.svg: https://www.svgrepo.com/svg/393367/rocket (PD Licence) +user_guide.svg: https://www.svgrepo.com/svg/75531/user-guide (CC0 Licence) +api.svg: https://www.svgrepo.com/svg/157898/gears-configuration-tool (CC0 Licence) +contributor.svg: https://www.svgrepo.com/svg/57189/code-programing-symbol (CC0 Licence) \ No newline at end of file diff --git a/doc/source/_static/index-images/user_guide.svg b/doc/source/_static/index-images/user_guide.svg new file mode 100644 index 0000000..6223a92 --- /dev/null +++ b/doc/source/_static/index-images/user_guide.svg @@ -0,0 +1,47 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/source/_static/qtoolkit.css b/doc/source/_static/qtoolkit.css new file mode 100644 index 0000000..4a77b9c --- /dev/null +++ b/doc/source/_static/qtoolkit.css @@ -0,0 +1,147 @@ +@import url('https://fonts.googleapis.com/css2?family=Lato:ital,wght@0,400;0,700;0,900;1,400;1,700;1,900&family=Open+Sans:ital,wght@0,400;0,600;1,400;1,600&display=swap'); + +.navbar-brand img { + height: 75px; +} +.navbar-brand { + height: 75px; +} + +body { + font-family: 'Open Sans', sans-serif; +} + +pre, code { + font-size: 100%; + line-height: 155%; +} + +h1 { + font-family: "Lato", sans-serif; + color: #013243; /* warm black */ +} + +h2 { + color: #4d77cf; /* han blue */ + letter-spacing: -.03em; +} + +h3 { + color: #013243; /* warm black */ + letter-spacing: -.03em; +} + +/* Style the active version button. + +- dev: orange +- stable: green +- old, PR: red + +Colors from: + +Wong, B. Points of view: Color blindness. +Nat Methods 8, 441 (2011). https://doi.org/10.1038/nmeth.1618 +*/ + +/* If the active version has the name "dev", style it orange */ +#version_switcher_button[data-active-version-name*="dev"] { + background-color: #E69F00; + border-color: #E69F00; + color:#000000; +} + +/* green for `stable` */ +#version_switcher_button[data-active-version-name*="stable"] { + background-color: #009E73; + border-color: #009E73; +} + +/* red for `old` */ +#version_switcher_button:not([data-active-version-name*="stable"], [data-active-version-name*="dev"], [data-active-version-name=""]) { + background-color: #980F0F; + border-color: #980F0F; +} + +/* Main page overview cards */ + +.sd-card { + background: #fff; + border-radius: 0; + padding: 30px 10px 20px 10px; + margin: 10px 0px; +} + +.sd-card .sd-card-header { + text-align: center; +} + +.sd-card .sd-card-header .sd-card-text { + margin: 0px; +} + +.sd-card .sd-card-img-top { + height: 52px; + width: 52px; + margin-left: auto; + margin-right: auto; +} + +.sd-card .sd-card-header { + border: none; + background-color: white; + color: #150458 !important; + font-size: var(--pst-font-size-h5); + font-weight: bold; + padding: 2.5rem 0rem 0.5rem 0rem; +} + +.sd-card .sd-card-footer { + border: none; + background-color: white; +} + +.sd-card .sd-card-footer .sd-card-text { + max-width: 220px; + margin-left: auto; + margin-right: auto; +} + +/* Announcements */ +.bd-header-announcement { + background-color: orange; +} + +/* Dark theme tweaking */ +html[data-theme=dark] .sd-card img[src*='.svg'] { + filter: invert(0.82) brightness(0.8) contrast(1.2); +} + +/* Main index page overview cards */ +html[data-theme=dark] .sd-card { + background-color:var(--pst-color-background); +} + +html[data-theme=dark] .sd-shadow-sm { + box-shadow: 0 .1rem 1rem rgba(250, 250, 250, .6) !important +} + +html[data-theme=dark] .sd-card .sd-card-header { + background-color:var(--pst-color-background); + color: #150458 !important; +} + +html[data-theme=dark] .sd-card .sd-card-footer { + background-color:var(--pst-color-background); +} + +html[data-theme=dark] .bd-header-announcement { + background-color: red; +} + +html[data-theme=dark] h1 { + color: var(--pst-color-primary); +} + +html[data-theme=dark] h3 { + color: #0a6774; +} diff --git a/doc/source/api/index.rst b/doc/source/api/index.rst new file mode 100644 index 0000000..daebaf2 --- /dev/null +++ b/doc/source/api/index.rst @@ -0,0 +1,15 @@ +.. _api: + +============= +API Reference +============= + +This is the API reference + +.. toctree:: + :maxdepth: 1 + + qtoolkit.core + qtoolkit.host + qtoolkit.io + qtoolkit.queue \ No newline at end of file diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 0000000..8ace3c5 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,216 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +# sys.path.insert(0, os.path.abspath('.')) +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +) + +# -- Project information ----------------------------------------------------- + +project = "QToolKit" +copyright = "2023, Matgenix SRL" +author = "Guido Petretto, David Waroquiers" + + +import qtoolkit +# The short X.Y version +version = qtoolkit.__version__ +# The full version, including alpha/beta/rc tags +release = qtoolkit.__version__ + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.viewcode", + "sphinx.ext.napoleon", # For Google Python Style Guide + 'sphinx.ext.coverage', + 'sphinx.ext.doctest', + 'sphinx.ext.autosummary', + 'sphinx.ext.graphviz', + 'sphinx.ext.ifconfig', + 'matplotlib.sphinxext.plot_directive', + 'IPython.sphinxext.ipython_console_highlighting', + 'IPython.sphinxext.ipython_directive', + 'sphinx.ext.mathjax', + 'sphinx_design', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +# html_theme = 'sphinx_book_theme' +html_theme = 'pydata_sphinx_theme' +# html_favicon = '_static/favicon/favicon.ico' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +html_theme_options = { + # "logo": { + # "image_light": "index-image/api.svg", + # "image_dark": "index-image/contributor.svg", + # }, + "collapse_navigation": True, + 'announcement': ( + "

" + "QToolKit is still in beta phase. The API may change at any time." + "

" + ), + # "announcement": "

This is still in development

", + # "navbar_end": ["theme-switcher", "navbar-icon-links"], + # "navbar_end": ["theme-switcher", "version-switcher", "navbar-icon-links"], +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +html_css_files = ["qtoolkit.css"] +html_title = "%s v%s Manual" % (project, version) +html_last_updated_fmt = '%b %d, %Y' +# html_css_files = ["numpy.css"] +html_context = {"default_mode": "light"} +html_use_modindex = True +html_copy_source = False +html_domain_indices = False +html_file_suffix = '.html' + +# Output file base name for HTML help builder. +htmlhelp_basename = "qtoolkitdoc" + + +# # -- Options for LaTeX output ------------------------------------------------ +# +# latex_elements = { +# # The paper size ('letterpaper' or 'a4paper'). +# # +# # 'papersize': 'letterpaper', +# # The font size ('10pt', '11pt' or '12pt'). +# # +# # 'pointsize': '10pt', +# # Additional stuff for the LaTeX preamble. +# # +# # 'preamble': '', +# # Latex figure (float) alignment +# # +# # 'figure_align': 'htbp', +# } +# +# # Grouping the document tree into LaTeX files. List of tuples +# # (source start file, target name, title, +# # author, documentclass [howto, manual, or own class]). +# latex_documents = [ +# # (master_doc, "turbomoleio.tex", "turbomoleio Documentation", author, "manual"), +# ] + + +# # -- Options for manual page output ------------------------------------------ +# +# # One entry per manual page. List of tuples +# # (source start file, name, description, authors, manual section). +# man_pages = [(master_doc, "turbomoleio", "turbomoleio Documentation", [author], 1)] +# +# +# # -- Options for Texinfo output ---------------------------------------------- +# +# # Grouping the document tree into Texinfo files. List of tuples +# # (source start file, target name, title, author, +# # dir menu entry, description, category) +# texinfo_documents = [ +# ( +# master_doc, +# "turbomoleio", +# "turbomoleio Documentation", +# author, +# "turbomoleio", +# "One line description of project.", +# "Miscellaneous", +# ), +# ] + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for intersphinx extension --------------------------------------- + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {"https://docs.python.org/": None} + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + +# To print the content of the docstring of the __init__ method as well. +autoclass_content = "both" diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst new file mode 100644 index 0000000..5bf8af1 --- /dev/null +++ b/doc/source/dev/index.rst @@ -0,0 +1,7 @@ +.. _devindex: + +######################## +Contributing to QToolKit +######################## + +Here are the things that can be done. \ No newline at end of file diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst new file mode 100644 index 0000000..3850067 --- /dev/null +++ b/doc/source/glossary.rst @@ -0,0 +1,14 @@ +******** +Glossary +******** + +.. glossary:: + + + QJob + The representation of a job in the queue. + + + QResources + The description of the resources for a job. + diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000..39690e7 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,106 @@ +.. _qtoolkit_docs_mainpage: + +###################### +QToolKit documentation +###################### + +.. toctree:: + :maxdepth: 1 + :hidden: + + User Guide + API reference + Development + release + + +**Version**: |version| + +**Useful links**: +TO BE ADDED + +QToolKit is an interface to most Distributed Resource Management systems, e.g. +SLURM, PBS, ... + + + +.. grid:: 1 2 2 2 + + .. grid-item-card:: + :img-top: ../source/_static/index-images/getting_started.svg + + Getting Started + ^^^^^^^^^^^^^^^ + + New to QToolKit? Check out the Absolute Beginner's Guide. It contains an + introduction to QToolKit's main concepts and links to additional tutorials. + + +++ + + .. button-ref:: user/absolute_beginners + :expand: + :color: secondary + :click-parent: + + To the absolute beginner's guide + + .. grid-item-card:: + :img-top: ../source/_static/index-images/user_guide.svg + + User Guide + ^^^^^^^^^^ + + The user guide provides in-depth information on the + key concepts of QToolKit with useful background information and explanation. + + +++ + + .. button-ref:: user + :expand: + :color: secondary + :click-parent: + + To the user guide + + .. grid-item-card:: + :img-top: ../source/_static/index-images/api.svg + + API Reference + ^^^^^^^^^^^^^ + + The reference guide contains a detailed description of the functions, + modules, and objects included in QToolKit. The reference describes how the + methods work and which parameters can be used. It assumes that you have an + understanding of the key concepts. + + +++ + + .. button-ref:: api + :expand: + :color: secondary + :click-parent: + + To the reference guide + + .. grid-item-card:: + :img-top: ../source/_static/index-images/contributor.svg + + Contributor's Guide + ^^^^^^^^^^^^^^^^^^^ + + Want to add to the codebase? Can help add support to an additional DRM system? + The contributing guidelines will guide you through the + process of improving QToolKit. + + +++ + + .. button-ref:: devindex + :expand: + :color: secondary + :click-parent: + + To the contributor's guide + +.. This is not really the index page, that is found in + _templates/indexcontent.html The toctree content here will be added to the + top of the template header \ No newline at end of file diff --git a/doc/source/license.rst b/doc/source/license.rst new file mode 100644 index 0000000..f36eb5b --- /dev/null +++ b/doc/source/license.rst @@ -0,0 +1,6 @@ +**************** +QToolKit license +**************** + +.. include:: ../../LICENSE + :literal: diff --git a/doc/source/user/basics.rst b/doc/source/user/basics.rst new file mode 100644 index 0000000..67af9c2 --- /dev/null +++ b/doc/source/user/basics.rst @@ -0,0 +1,12 @@ +********************* +QToolKit fundamentals +********************* + +These documents clarify concepts, design decisions, and technical +constraints in QToolKit. This is a great place to understand the +fundamental QToolKit ideas and philosophy. + +.. + .. toctree:: + :maxdepth: 1 + diff --git a/doc/source/user/building.rst b/doc/source/user/building.rst new file mode 100644 index 0000000..93af682 --- /dev/null +++ b/doc/source/user/building.rst @@ -0,0 +1,7 @@ +.. _building-from-source: + +Building from source +==================== + +Get the source from the git repository. +Install it with pip install . \ No newline at end of file diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst new file mode 100644 index 0000000..5155edf --- /dev/null +++ b/doc/source/user/index.rst @@ -0,0 +1,30 @@ +.. _user: + +################### +QToolKit user guide +################### + +This guide is an overview and explains the important features; +details are found in :ref:`reference`. + +.. toctree:: + :caption: Getting started + :maxdepth: 1 + + whatisqtoolkit + install + quickstart + +.. toctree:: + :caption: Advanced usage and interoperability + :maxdepth: 1 + + building + + +.. toctree:: + :hidden: + :caption: Extras + + ../glossary + ../license diff --git a/doc/source/user/install.rst b/doc/source/user/install.rst new file mode 100644 index 0000000..50cd00e --- /dev/null +++ b/doc/source/user/install.rst @@ -0,0 +1,9 @@ +.. _install: + +******************* +Installing QToolKit +******************* + +The only prerequisite for installing QToolKit is Python. + +QToolKit can be installed with 'conda', with 'pip', or from source. \ No newline at end of file diff --git a/doc/source/user/quickstart.rst b/doc/source/user/quickstart.rst new file mode 100644 index 0000000..da9978c --- /dev/null +++ b/doc/source/user/quickstart.rst @@ -0,0 +1,17 @@ +.. _quickstart: + +=================== +QToolKit quickstart +=================== + +Prerequisites +============= +You need python + +The Basics +========== + +Create an easy script + +ADD OTHER TOPICS: +remote submission, resources, ... \ No newline at end of file diff --git a/doc/source/user/whatisqtoolkit.rst b/doc/source/user/whatisqtoolkit.rst new file mode 100644 index 0000000..a302ec4 --- /dev/null +++ b/doc/source/user/whatisqtoolkit.rst @@ -0,0 +1,8 @@ +.. _whatisqtoolkit: + +================= +What is QToolKit? +================= + +QToolKit is ... +TODO: add the features that it has. \ No newline at end of file diff --git a/doc_requirements.txt b/doc_requirements.txt new file mode 100644 index 0000000..53b03cc --- /dev/null +++ b/doc_requirements.txt @@ -0,0 +1,14 @@ +# doxygen required, use apt-get or dnf +sphinx>=4.5.0 +numpydoc==1.4 +pydata-sphinx-theme==0.13.3 +sphinx-design +sphinx-apidoc +ipython!=8.1.0 +scipy +matplotlib +pandas +breathe + +# needed to build release notes +towncrier diff --git a/src/qtoolkit/core/__init__.py b/src/qtoolkit/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/qtoolkit/host/__init__.py b/src/qtoolkit/host/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/qtoolkit/queue/__init__.py b/src/qtoolkit/queue/__init__.py new file mode 100644 index 0000000..e69de29 From 8f5a4b84883bb89fa05dff57f621ba7ca773c26c Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Fri, 14 Apr 2023 17:10:26 +0200 Subject: [PATCH 02/34] Fixed navigation for api. --- doc/source/api/index.rst | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/doc/source/api/index.rst b/doc/source/api/index.rst index daebaf2..484de6a 100644 --- a/doc/source/api/index.rst +++ b/doc/source/api/index.rst @@ -1,15 +1,9 @@ .. _api: -============= +############# API Reference -============= +############# This is the API reference -.. toctree:: - :maxdepth: 1 - - qtoolkit.core - qtoolkit.host - qtoolkit.io - qtoolkit.queue \ No newline at end of file +.. include:: qtoolkit.rst \ No newline at end of file From 3b0a6c518ce60661bab7aaa5d6066a2f5aca9192 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Sat, 29 Apr 2023 23:39:55 +0200 Subject: [PATCH 03/34] Updated css styling (Matgenix colors). --- doc/Makefile | 2 +- doc/source/_static/qtoolkit.css | 15 +++++++++++++++ doc/source/index.rst | 12 ++++++------ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/doc/Makefile b/doc/Makefile index 90d6037..3e5f5da 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -69,4 +69,4 @@ html-build: generate @echo @echo "Build finished. The HTML pages are in build/html." api-doc: - sphinx-apidoc -f -o source/api ../src/qtoolkit + sphinx-apidoc -e -f -o source/api ../src/qtoolkit diff --git a/doc/source/_static/qtoolkit.css b/doc/source/_static/qtoolkit.css index 4a77b9c..4561c96 100644 --- a/doc/source/_static/qtoolkit.css +++ b/doc/source/_static/qtoolkit.css @@ -1,5 +1,10 @@ @import url('https://fonts.googleapis.com/css2?family=Lato:ital,wght@0,400;0,700;0,900;1,400;1,700;1,900&family=Open+Sans:ital,wght@0,400;0,600;1,400;1,600&display=swap'); +:root { + --matgenix-color: #46b3c1; + --matgenix-dark-color: #338d99; +} + .navbar-brand img { height: 75px; } @@ -145,3 +150,13 @@ html[data-theme=dark] h1 { html[data-theme=dark] h3 { color: #0a6774; } + +.sd-btn-secondary { + background-color: var(--matgenix-color) !important; + border-color: var(--matgenix-color) !important; +} + +.sd-btn-secondary:hover, .sd-btn-secondary:focus { + background-color: var(--matgenix-dark-color) !important; + border-color: var(--matgenix-dark-color) !important; +} \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst index 39690e7..46f8eef 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -32,17 +32,17 @@ SLURM, PBS, ... Getting Started ^^^^^^^^^^^^^^^ - New to QToolKit? Check out the Absolute Beginner's Guide. It contains an - introduction to QToolKit's main concepts and links to additional tutorials. + If you want to get started quickly, check out our quickstart section. + It contains an introduction to QToolKit's main concepts. +++ - .. button-ref:: user/absolute_beginners + .. button-ref:: user/quickstart :expand: :color: secondary :click-parent: - To the absolute beginner's guide + Quickstart .. grid-item-card:: :img-top: ../source/_static/index-images/user_guide.svg @@ -60,7 +60,7 @@ SLURM, PBS, ... :color: secondary :click-parent: - To the user guide + User Guide .. grid-item-card:: :img-top: ../source/_static/index-images/api.svg @@ -80,7 +80,7 @@ SLURM, PBS, ... :color: secondary :click-parent: - To the reference guide + API Reference .. grid-item-card:: :img-top: ../source/_static/index-images/contributor.svg From f4512c6ea18740467bef22dac3ac178f6e0dcee6 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Sat, 29 Apr 2023 23:50:19 +0200 Subject: [PATCH 04/34] Added optional deps for docs. --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1432ea9..92453e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,11 @@ tests = [ maintain = [ "git-changelog>=0.6", ] +docs = [ + "sphinx", + "sphinx_design", + "pydata-sphinx-theme", +] strict = [] remote = ["fabric>=3.0.0"] msonable = ["monty>=2022.9.9",] From d4b6ca2827382a47a9f800ef22b5e8e6a761160a Mon Sep 17 00:00:00 2001 From: Guido Petretto Date: Fri, 5 May 2023 09:44:09 +0200 Subject: [PATCH 05/34] adding shell io object --- src/qtoolkit/core/data_objects.py | 5 +- src/qtoolkit/io/base.py | 4 +- src/qtoolkit/io/pbs.py | 4 +- src/qtoolkit/io/shell.py | 226 ++++++++++++++++++++++++++++++ src/qtoolkit/io/slurm.py | 4 +- 5 files changed, 239 insertions(+), 4 deletions(-) create mode 100644 src/qtoolkit/io/shell.py diff --git a/src/qtoolkit/core/data_objects.py b/src/qtoolkit/core/data_objects.py index 671349c..65129ef 100644 --- a/src/qtoolkit/core/data_objects.py +++ b/src/qtoolkit/core/data_objects.py @@ -146,7 +146,10 @@ def __post_init__(self): elif self.nodes and self.processes_per_node and not self.processes: self.process_placement = ProcessPlacement.EVENLY_DISTRIBUTED else: - msg = "When process_placement is None either define only nodes plus processes_per_node or only processes" + msg = ( + "When process_placement is None either define only nodes " + "plus processes_per_node or only processes" + ) raise UnsupportedResourcesError(msg) @classmethod diff --git a/src/qtoolkit/io/base.py b/src/qtoolkit/io/base.py index 908d286..9f86962 100644 --- a/src/qtoolkit/io/base.py +++ b/src/qtoolkit/io/base.py @@ -345,7 +345,9 @@ def get_jobs_list_cmd( return self._get_jobs_list_cmd(job_ids, user) @abc.abstractmethod - def _get_jobs_list_cmd(self, job_ids: list[str] | None, user: str | None) -> str: + def _get_jobs_list_cmd( + self, job_ids: list[str] | None = None, user: str | None = None + ) -> str: pass @abc.abstractmethod diff --git a/src/qtoolkit/io/pbs.py b/src/qtoolkit/io/pbs.py index 647df73..81dad5f 100644 --- a/src/qtoolkit/io/pbs.py +++ b/src/qtoolkit/io/pbs.py @@ -168,7 +168,9 @@ def parse_job_output(self, exit_code, stdout, stderr) -> QJob | None: return out[0] return None - def _get_jobs_list_cmd(self, job_ids: list[str] | None, user: str | None) -> str: + def _get_jobs_list_cmd( + self, job_ids: list[str] | None = None, user: str | None = None + ) -> str: if user and job_ids: raise ValueError("Cannot query by user and job(s) in PBS") diff --git a/src/qtoolkit/io/shell.py b/src/qtoolkit/io/shell.py new file mode 100644 index 0000000..01a3164 --- /dev/null +++ b/src/qtoolkit/io/shell.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +from pathlib import Path + +from qtoolkit.core.data_objects import ( + CancelResult, + CancelStatus, + QJob, + QResources, + QState, + QSubState, + SubmissionResult, + SubmissionStatus, +) +from qtoolkit.core.exceptions import ( + CommandFailedError, + OutputParsingError, + UnsupportedResourcesError, +) +from qtoolkit.io.base import BaseSchedulerIO + +# States in from ps command, extracted from man ps. +# D uninterruptible sleep (usually IO) +# R running or runnable (on run queue) +# S interruptible sleep (waiting for an event to complete) +# T stopped by job control signal +# t stopped by debugger during the tracing +# W paging (not valid since the 2.6.xx kernel) +# X dead (should never be seen) +# Z defunct ("zombie") process, terminated but not reaped by its parent + + +class ShellState(QSubState): + UNINTERRUPTIBLE_SLEEP = "D" + RUNNING = "R" + INTERRUPTIBLE_SLEEP = "S" + STOPPED = "T" + STOPPED_DEBUGGER = "t" + PAGING = "W" + DEAD = "D" + DEFUNCT = "Z" + + @property + def qstate(self) -> QState: + return _STATUS_MAPPING[self] # type: ignore + + +_STATUS_MAPPING = { + ShellState.UNINTERRUPTIBLE_SLEEP: QState.RUNNING, + ShellState.RUNNING: QState.RUNNING, + ShellState.INTERRUPTIBLE_SLEEP: QState.RUNNING, + ShellState.STOPPED: QState.SUSPENDED, + ShellState.STOPPED_DEBUGGER: QState.SUSPENDED, + ShellState.PAGING: QState.RUNNING, + ShellState.DEAD: QState.DONE, + ShellState.DEFUNCT: QState.DONE, # TODO should be failed? +} + + +class ShellIO(BaseSchedulerIO): + header_template: str = """ +##!/bin/bash +$${qverbatim} +""" + + CANCEL_CMD: str | None = "kill -9" + + def __init__(self, blocking=False, stdout_path="stdout", stderr_path="stderr"): + self.blocking = blocking + self.stdout_path = stdout_path + self.stderr_path = stderr_path + + def get_submit_cmd(self, script_file: str | Path | None = "submit.script") -> str: + """ + Get the command used to submit a given script to the queue. + + Parameters + ---------- + script_file: (str) path of the script file to use. + """ + script_file = script_file or "" + + # nohup and the redirection of the outputs is needed when running through fabric + # see https://www.fabfile.org/faq.html#why-can-t-i-run-programs-in-the-background-with-it-makes-fabric-hang # noqa + command = f"bash {script_file} > {self.stdout_path} 2> {self.stderr_path}" + if not self.blocking: + command = f"nohup {command} & echo $!" + return command + + def parse_submit_output(self, exit_code, stdout, stderr) -> SubmissionResult: + if isinstance(stdout, bytes): + stdout = stdout.decode() + if isinstance(stderr, bytes): + stderr = stderr.decode() + if exit_code != 0: + return SubmissionResult( + exit_code=exit_code, + stdout=stdout, + stderr=stderr, + status=SubmissionStatus("FAILED"), + ) + job_id = stdout.strip() or None + status = ( + SubmissionStatus("SUCCESSFUL") + if job_id + else SubmissionStatus("JOB_ID_UNKNOWN") + ) + return SubmissionResult( + job_id=job_id, + exit_code=exit_code, + stdout=stdout, + stderr=stderr, + status=status, + ) + + def parse_cancel_output(self, exit_code, stdout, stderr) -> CancelResult: + """Parse the output of the kill command.""" + if isinstance(stdout, bytes): + stdout = stdout.decode() + if isinstance(stderr, bytes): + stderr = stderr.decode() + if exit_code != 0: + return CancelResult( + exit_code=exit_code, + stdout=stdout, + stderr=stderr, + status=CancelStatus("FAILED"), + ) + + status = CancelStatus("SUCCESSFUL") + return CancelResult( + job_id=None, + exit_code=exit_code, + stdout=stdout, + stderr=stderr, + status=status, + ) + + def _get_job_cmd(self, job_id: str): + + cmd = self._get_jobs_list_cmd(job_ids=[job_id]) + + return cmd + + def parse_job_output(self, exit_code, stdout, stderr) -> QJob | None: + out = self.parse_jobs_list_output(exit_code, stdout, stderr) + if out: + return out[0] + return None + + def _get_jobs_list_cmd( + self, job_ids: list[str] | None = None, user: str | None = None + ) -> str: + + if user and job_ids: + msg = ( + "Cannot query by user and job(s) with ps, " + "as the user option will override the ids list" + ) + raise ValueError(msg) + + command = [ + "ps", + "-o pid,user,etimes,state,comm", + ] + + if user: + command.append(f"-U {user}") + + if job_ids: + command.append("-p " + ",".join(job_ids)) + + return " ".join(command) + + def parse_jobs_list_output(self, exit_code, stdout, stderr) -> list[QJob]: + + if isinstance(stdout, bytes): + stdout = stdout.decode() + if isinstance(stderr, bytes): + stderr = stderr.decode() + + if exit_code != 0: + msg = f"command ps failed: {stderr}" + raise CommandFailedError(msg) + + jobs_list = [] + for row in stdout.splitlines()[1:]: + if not row.strip(): + continue + + data = row.split() + + qjob = QJob() + qjob.job_id = data[0] + qjob.username = data[1] + qjob.runtime = int(data[2]) + qjob.name = data[4] + + try: + shell_job_state = ShellState(data[3]) + except ValueError: + msg = f"Unknown job state {data[3]} for job id {qjob.job_id}" + raise OutputParsingError(msg) + qjob.sub_state = shell_job_state + qjob.state = shell_job_state.qstate + + jobs_list.append(qjob) + + return jobs_list + + def _convert_qresources(self, resources: QResources) -> dict: + """ + Converts a QResources instance to a dict that will be used to fill in the + header of the submission script. + Not implemented for ShellIO + """ + raise UnsupportedResourcesError + + @property + def supported_qresources_keys(self) -> list: + """ + List of attributes of QResources that are correctly handled by the + _convert_qresources method. It is used to validate that the user + does not pass an unsupported value, expecting to have an effect. + """ + return [] diff --git a/src/qtoolkit/io/slurm.py b/src/qtoolkit/io/slurm.py index 306447a..d70bb0c 100644 --- a/src/qtoolkit/io/slurm.py +++ b/src/qtoolkit/io/slurm.py @@ -339,7 +339,9 @@ def _parse_scontrol_cmd_output(self, stdout): for data in stdout.split() } - def _get_jobs_list_cmd(self, job_ids: list[str] | None, user: str | None) -> str: + def _get_jobs_list_cmd( + self, job_ids: list[str] | None = None, user: str | None = None + ) -> str: if user and job_ids: raise ValueError("Cannot query by user and job(s) in SLURM") From fd45cad37b1d51678a69ee1a0139cb7e84ef167b Mon Sep 17 00:00:00 2001 From: Guido Petretto Date: Fri, 5 May 2023 15:44:06 +0200 Subject: [PATCH 06/34] remove shabang from template --- src/qtoolkit/io/base.py | 2 -- src/qtoolkit/io/pbs.py | 1 - src/qtoolkit/io/shell.py | 1 - 3 files changed, 4 deletions(-) diff --git a/src/qtoolkit/io/base.py b/src/qtoolkit/io/base.py index 9f86962..b814409 100644 --- a/src/qtoolkit/io/base.py +++ b/src/qtoolkit/io/base.py @@ -204,8 +204,6 @@ def generate_header(self, options: dict | QResources | None) -> str: # execution (rerunnable) # resources (nodes, cores, memory, time, [gpus]) # default values for (almost) everything in the object ? - if not options: - return "" if isinstance(options, QResources): options = self.check_convert_qresources(options) diff --git a/src/qtoolkit/io/pbs.py b/src/qtoolkit/io/pbs.py index 81dad5f..d876912 100644 --- a/src/qtoolkit/io/pbs.py +++ b/src/qtoolkit/io/pbs.py @@ -81,7 +81,6 @@ def qstate(self) -> QState: class PBSIO(BaseSchedulerIO): header_template: str = """ -#!/bin/bash #PBS -q $${queue} #PBS -N $${job_name} #PBS -A $${account} diff --git a/src/qtoolkit/io/shell.py b/src/qtoolkit/io/shell.py index 01a3164..d3fc6cd 100644 --- a/src/qtoolkit/io/shell.py +++ b/src/qtoolkit/io/shell.py @@ -59,7 +59,6 @@ def qstate(self) -> QState: class ShellIO(BaseSchedulerIO): header_template: str = """ -##!/bin/bash $${qverbatim} """ From d6490beabfd85ef18e8ee0e8bb3f7c3ff77f5676 Mon Sep 17 00:00:00 2001 From: Guido Petretto Date: Fri, 5 May 2023 15:57:20 +0200 Subject: [PATCH 07/34] remove old implementation --- src/qtoolkit/queue/base.py | 366 ------------------------------------ src/qtoolkit/queue/slurm.py | 273 --------------------------- 2 files changed, 639 deletions(-) delete mode 100644 src/qtoolkit/queue/base.py delete mode 100644 src/qtoolkit/queue/slurm.py diff --git a/src/qtoolkit/queue/base.py b/src/qtoolkit/queue/base.py deleted file mode 100644 index 121800a..0000000 --- a/src/qtoolkit/queue/base.py +++ /dev/null @@ -1,366 +0,0 @@ -from __future__ import annotations - -import abc -from dataclasses import dataclass, field -from pathlib import Path -from string import Template - -from qtoolkit.core.base import QBase -from qtoolkit.core.data_objects import QJob -from qtoolkit.host.base import BaseHost -from qtoolkit.host.local import LocalHost - - -class QTemplate(Template): - delimiter = "$$" - - -@dataclass -class BaseQueue(QBase): - """Base class for job queues. - - Attributes - ---------- - name : str - Name of the queue - host : BaseHost - Host where the command should be executed. - """ - - header_template: str - name: str = "name of queue" - host: BaseHost = field(default_factory=LocalHost) - default_shebang: str = "#!/bin/bash" - - SCRIPT_FNAME = "submit.script" - SUBMIT_CMD: str | None = None - - # host : QToolKit.Host or paramiko Client or Fabric client or None - # The host where the command should be executed. - - # config: QueueConfig = None - - # scheduler = None, - # name = None, - # cores = None, - # memory = None, - # processes = None, - # nanny = True, - # protocol = None, - # security = None, - # interface = None, - # death_timeout = None, - # local_directory = None, - # extra = None, - # worker_extra_args = None, - # job_extra = None, - # job_extra_directives = None, - # env_extra = None, - # job_script_prologue = None, - # header_skip = None, - # job_directives_skip = None, - # log_directory = None, - # shebang = None, - # python = sys.executable, - # job_name = None, - # config_name = None, - - """ABIPY - Args: - qname: Name of the queue. - qparams: Dictionary with the parameters used in the template. - setup: String or list of commands to execute during the initial setup. - modules: String or list of modules to load before running the application. - shell_env: Dictionary with the environment variables to export before - running the application. - omp_env: Dictionary with the OpenMP variables. - pre_run: String or list of commands to execute before launching the - calculation. - post_run: String or list of commands to execute once the calculation is - completed. - mpi_runner: Path to the MPI runner or :class:`MpiRunner` instance. - None if not used - mpi_runner_options: Optional string with options passed to the mpi_runner. - max_num_launches: Maximum number of submissions that can be done for a - specific task. Defaults to 5 - qverbatim: - min_cores, max_cores, hint_cores: Minimum, maximum, and hint limits of - number of cores that can be used - min_mem_per_proc=Minimum memory per process in megabytes. - max_mem_per_proc=Maximum memory per process in megabytes. - timelimit: initial time limit in seconds - timelimit_hard: hard limelimit for this queue - priority: Priority level, integer number > 0 - condition: Condition object (dictionary) - """ - - def execute_cmd(self, cmd): - """Execute a command. - - Parameters - ---------- - cmd : str - Command to be executed - - Returns - ------- - stdout : str - stderr : str - exit_code : int - """ - return self.host.execute(cmd) - - def get_submission_script( - self, - commands: str | list[str] | None, - resources=None, - submit_dir=None, - environment=None, - ) -> str: - """ - This is roughly what/how it is done in the existing solutions. - - abipy: done with a str template (using $$ as a delimiter). - Remaining "$$" delimiters are then removed at the end. - It uses a ScriptEditor object to add/modify things to the templated script. - The different steps of "get_script_str(...)" in abipy are summarized here: - - _header, based on the str template (includes the shebang line and all - #SBATCH, #PBS, ... directives) - - change directory (added by the script editor) - - setup section, list of commands executed before running (added - by the script editor) - - load modules section, list of modules to be loaded before running - (added by the script editor) - - setting of openmp environment variables (added by the script editor) - - setting of shell environment variables (added by the script editor) - - prerun, i.e. commands to run before execution, again? (added by - the script editor) - - run line (added by the script editor) - - postrun (added by the script editor) - - aiida: done with a class template (JobTemplate) that should contain - the required info to generate the job header. Other class templates - are also used inside the generation, e.g. JobTemplateCodesInfo, which - defines the command(s) to be run. The JobTemplate is only used as a - container of the information and the script is generated not using - templating but rather directly using python methods based on that - "JobTemplate" container. Actually this JobTemplate is based on the - DRMAA v2 specifications and many other objects are based on that too - (e.g. machine, slots, etc ...). - The different steps of "get_submit_script(...)" in aiida are summarized here: - - shebang line - - _header - - all #SBATCH, #PBS etc ... lines defining the resources and other - info for the queuing system - - some custom lines if it is not dealt with by the template - - environment variables - - prepend_text (something to be written before the run lines) - - _run_line (defines the code execution(s) based on a CodeInfo object). - There can be several codes run. - - append_text (something to be written after the run lines) - - _footer (some post commands done after the run) [note this is only - done/needed for LSF in aiida] - - fireworks: done with a str template. similar to abipy (actually abipy took - its initial concept from fireworks) - - dask_jobqueue: quite obscure ... the job header is done in the init of a - given JobCluster (e.g. SLURMCluster) based on something in the actual - Job object itself. Dask is not really meant to be our use case anyway. - - dpdispatcher: uses python's format() with 5 templates, combined into - another python's format "script" template. - Here are the steps: - - header (includes shebang and #SBATCH, #PBS, ... directives) - - custom directives - - script environment (modules, environment variables, source - somefiles, ...) - - run command - - append script lines - In the templates of the different steps, there are some dpdispatcher's - specific things (e.g. tag a job as finished by touching a file, ...) - - jobqueues: Some queues are using pure python (PBS, LSF, ...), some are - using jinja2 templates (SLURM and SGE). Directly written to file. - - myqueue: the job queue directives are directly passed to the submit - command (no #SBATCH, #PBS, ...). - - troika: uses a generic generator with a list of directives as well - as a directive prefix. These directives are defined in specific - files for each type of job queue. - """ - script_blocks = [self.shebang] - if header := self.get_header(resources): - script_blocks.append(header) - if environment_setup := self.get_environment_setup(environment): - script_blocks.append(environment_setup) - if change_dir := self.get_change_dir(): - script_blocks.append(change_dir) - if prerun := self.get_prerun(): - script_blocks.append(prerun) - if run_commands := self.get_run_commands(commands): - script_blocks.append(run_commands) - if postrun := self.get_postrun(): - script_blocks.append(postrun) - if footer := self.get_footer(): - script_blocks.append(footer) - return "\n".join(script_blocks) - - @property - def shebang(self): - return "#!/bin/bash" - - # @abc.abstractmethod - # def get_header(self, job): - # pass - - def get_header(self, resources): - # needs info from self.meta_info (email, job name [also execution]) - # queuing_options (priority, account, qos and submit as hold) - # execution (rerunnable) - # resources (nodes, cores, memory, time, [gpus]) - # default values for (almost) everything in the object ? - mapping = {} - if resources: - mapping.update(resources) - unclean_header = QTemplate(self.header_template).safe_substitute(mapping) - # Remove lines with leftover $$. - clean_header = [] - for line in unclean_header.split("\n"): - if "$$" not in line: - clean_header.append(line) - - return "\n".join(clean_header) - - def get_environment_setup(self, env_config): - if env_config: - env_setup = [] - if "modules" in env_config: - env_setup.append("module purge") - for mod in env_config["modules"]: - env_setup.append(f"module load {mod}") - if "source_files" in env_config: - for source_file in env_config["source_files"]: - env_setup.append(f"source {source_file}") - if "conda_environment" in env_config: - env_setup.append(f'conda activate {env_config["conda_environment"]}') - if "environ" in env_config: - for var, value in env_config["environ"].items(): - env_setup.append(f"export {var}={value}") - return "\n".join(env_setup) - # This is from aiida, maybe we need to think about this escape_for_bash ? - # lines = ['# ENVIRONMENT VARIABLES BEGIN ###'] - # for key, value in template.job_environment.items(): - # lines.append(f'export {key.strip()}={ - # escape_for_bash(value, - # template.environment_variables_double_quotes) - # }') - # lines.append('# ENVIRONMENT VARIABLES END ###') - return None - - def get_change_dir(self): - pass - - def get_prerun(self): - pass - - def get_run_commands(self, commands): - if isinstance(commands, str): - return commands - elif isinstance(commands, list): - return "\n".join(commands) - else: - raise ValueError("commands should be a str or a list of str.") - - def get_postrun(self): - pass - - def get_footer(self): - pass - - def get_submit_cmd(self, script_file: str | Path = SCRIPT_FNAME) -> str: - """ - Get the command used to submit a given script to the queue. - - Parameters - ---------- - script_file: (str) name of the script file to use. - """ - - return f"{self.SUBMIT_CMD} {script_file}" - - def get_cancel_cmd(self, job: QJob | int | str) -> str: - """ - Get the command used to cancel a given job. - - Parameters - ---------- - job: (str) job to be cancelled. - """ - job_id = QJob.job_id if isinstance(job, QJob) else job - return f"{self.CANCEL_CMD} {job_id}" - - def write_script(self, script_fpath: str | Path, script_content: str) -> None: - self.host.write_text_file(script_fpath, script_content) - - @abc.abstractmethod - def _parse_submit_cmd_output(self, exit_code, stdout, stderr): - pass - - @abc.abstractmethod - def _parse_cancel_cmd_output(self, exit_code, stdout, stderr): - pass - - def submit( - self, - commands: str | list[str] | None, - resources=None, - submit_dir=None, - environment=None, - script_fname=SCRIPT_FNAME, - create_submit_dir=False, - ): - script_str = self.get_submission_script( - commands=commands, - resources=resources, - # TODO: Do we need the submit_dir here ? - # Should we distinguish submit_dir and work_dir ? - submit_dir=submit_dir, - environment=environment, - ) - # TODO: deal with remote directory directly on the host here. - # Will currently only work on the localhost. - submit_dir = Path(submit_dir) if submit_dir is not None else Path.cwd() - if create_submit_dir: - self.host.mkdir(submit_dir, recursive=True, exist_ok=True) - script_fpath = Path(submit_dir, script_fname) - self.write_script(script_fpath, script_str) - submit_cmd = self.get_submit_cmd(script_fpath) - print(submit_cmd) - stdout, stderr, returncode = self.execute_cmd(submit_cmd) - return self._parse_submit_cmd_output( - exit_code=returncode, stdout=stdout, stderr=stderr - ) - - def get_job_info(self, job: QJob | int | str): - pass - - def get_jobs(self, jobs: list[QJob | int | str]): - pass - - def cancel(self, job: QJob | int | str): - cancel_cmd = self.get_cancel_cmd(job) - stdout, stderr, returncode = self.execute_cmd(cancel_cmd) - return self._parse_cancel_cmd_output( - exit_code=returncode, stdout=stdout, stderr=stderr - ) - - # @abc.abstractmethod - # def _get_jobs_cmd(self, jobs=None, user=None) -> str: - # """Get multiple jobs at once.""" - # pass - - @abc.abstractmethod - def get_job(self, job: QJob | int | str): - pass diff --git a/src/qtoolkit/queue/slurm.py b/src/qtoolkit/queue/slurm.py deleted file mode 100644 index 8453802..0000000 --- a/src/qtoolkit/queue/slurm.py +++ /dev/null @@ -1,273 +0,0 @@ -from __future__ import annotations - -import re -from dataclasses import dataclass - -from qtoolkit.core.data_objects import ( - CancelResult, - CancelStatus, - QJob, - QResources, - QState, - QSubState, - SubmissionResult, - SubmissionStatus, -) -from qtoolkit.queue.base import BaseQueue - -# States in Slurm from squeue's manual. We currently only take the most important ones. -# JOB STATE CODES -# Jobs typically pass through several states in the course of their execution. -# The typical states are PENDING, RUNNING, SUSPENDED, COMPLETING, and COMPLETED. -# An explanation of each state follows. -# -# BF BOOT_FAIL Job terminated due to launch failure, typically due to a -# hardware failure (e.g. unable to -# boot the node or block and the job can not be requeued). -# -# CA CANCELLED Job was explicitly cancelled by the user or system -# administrator. The job may or may not -# have been initiated. -# -# CD COMPLETED Job has terminated all processes on all nodes with -# an exit code of zero. -# -# CF CONFIGURING Job has been allocated resources, but are waiting for -# them to become ready for use (e.g. booting). -# -# CG COMPLETING Job is in the process of completing. -# Some processes on some nodes may still be active. -# -# DL DEADLINE Job terminated on deadline. -# -# F FAILED Job terminated with non-zero exit code or other -# failure condition. -# -# NF NODE_FAIL Job terminated due to failure of one or more -# allocated nodes. -# -# OOM OUT_OF_MEMORY Job experienced out of memory error. -# -# PD PENDING Job is awaiting resource allocation. -# -# PR PREEMPTED Job terminated due to preemption. -# -# R RUNNING Job currently has an allocation. -# -# RD RESV_DEL_HOLD Job is being held after requested reservation was deleted. -# -# RF REQUEUE_FED Job is being requeued by a federation. -# -# RH REQUEUE_HOLD Held job is being requeued. -# -# RQ REQUEUED Completing job is being requeued. -# -# RS RESIZING Job is about to change size. -# -# RV REVOKED Sibling was removed from cluster due to other cluster -# starting the job. -# -# SI SIGNALING Job is being signaled. -# -# SE SPECIAL_EXIT The job was requeued in a special state. This state -# can be set by users, typically in Epi‐ -# logSlurmctld, if the job has terminated with a particular -# exit value. -# -# SO STAGE_OUT Job is staging out files. -# -# ST STOPPED Job has an allocation, but execution has been stopped -# with SIGSTOP signal. CPUS have been retained by this job. -# -# S SUSPENDED Job has an allocation, but execution has been -# suspended and CPUs have been released for other jobs. -# -# TO TIMEOUT Job terminated upon reaching its time limit. - - -class SlurmState(QSubState): - CANCELLED = "CANCELLED", "CA" - COMPLETING = "COMPLETING", "CG" - COMPLETED = "COMPLETED", "CD" - CONFIGURING = "CONFIGURING", "CF" - DEADLINE = "DEADLINE", "DL" - FAILED = "FAILED", "F" - NODE_FAIL = "NODE_FAIL", "NF" - OUT_OF_MEMORY = "OUT_OF_MEMORY", "OOM" - PENDING = "PENDING", "PD" - RUNNING = "RUNNING", "R" - SUSPENDED = "SUSPENDED", "S" - TIMEOUT = "TIMEOUT", "TO" - - -@dataclass -class SlurmQueue(BaseQueue): - header_template: str = """ -#SBATCH --partition=$${queue_name} -#SBATCH --job-name=$${job_name} -#SBATCH --nodes=$${number_of_nodes} -#SBATCH --ntasks=$${number_of_tasks} -#SBATCH --ntasks-per-node=$${ntasks_per_node} -#SBATCH --cpus-per-task=$${cpus_per_task} -#####SBATCH --mem=$${mem} -#SBATCH --mem-per-cpu=$${mem_per_cpu} -#SBATCH --hint=$${hint} -#SBATCH --time=$${time} -#SBATCH --exclude=$${exclude_nodes} -#SBATCH --account=$${account} -#SBATCH --mail-user=$${mail_user} -#SBATCH --mail-type=$${mail_type} -#SBATCH --constraint=$${constraint} -#SBATCH --gres=$${gres} -#SBATCH --requeue=$${requeue} -#SBATCH --nodelist=$${nodelist} -#SBATCH --propagate=$${propagate} -#SBATCH --licenses=$${licenses} -#SBATCH --output=$${_qout_path} -#SBATCH --error=$${_qerr_path} -#SBATCH --qos=$${qos} -$${qverbatim}""" - - SUBMIT_CMD: str = "sbatch" - CANCEL_CMD = "scancel -v" # The -v is needed as the default is to report nothing - - _STATUS_MAPPING = { - SlurmState.CANCELLED: QState.SUSPENDED, # Should this be failed ? - SlurmState.COMPLETING: QState.RUNNING, - SlurmState.COMPLETED: QState.DONE, - SlurmState.CONFIGURING: QState.QUEUED, - SlurmState.DEADLINE: QState.FAILED, - SlurmState.FAILED: QState.FAILED, - SlurmState.NODE_FAIL: QState.FAILED, - SlurmState.OUT_OF_MEMORY: QState.FAILED, - SlurmState.PENDING: QState.QUEUED, - SlurmState.RUNNING: QState.RUNNING, - SlurmState.SUSPENDED: QState.SUSPENDED, - SlurmState.TIMEOUT: QState.FAILED, - } - - def _parse_submit_cmd_output(self, exit_code, stdout, stderr): - if isinstance(stdout, bytes): - stdout = stdout.decode() - if isinstance(stderr, bytes): - stderr = stderr.decode() - if exit_code != 0: - return SubmissionResult( - exit_code=exit_code, - stdout=stdout, - stderr=stderr, - status=SubmissionStatus("FAILED"), - ) - _SLURM_SUBMITTED_REGEXP = re.compile( - r"(.*:\s*)?([Gg]ranted job allocation|" - r"[Ss]ubmitted batch job)\s+(?P\d+)" - ) - match = _SLURM_SUBMITTED_REGEXP.match(stdout.strip()) - job_id = match.group("jobid") if match else None - status = ( - SubmissionStatus("SUCCESSFUL") - if job_id - else SubmissionStatus("JOB_ID_UNKNOWN") - ) - return SubmissionResult( - job_id=job_id, - exit_code=exit_code, - stdout=stdout, - stderr=stderr, - status=status, - ) - - def _parse_cancel_cmd_output(self, exit_code, stdout, stderr): - """Parse the output of the scancel command.""" - # Possible error messages: - # scancel: error: No job identification provided - # scancel: error: Kill job error on job id 958: Invalid job id specified - # scancel: error: Kill job error on job id 69: - # Job/step already completing or completed - # Correct execution: - # scancel: Terminating job 80 - if isinstance(stdout, bytes): - stdout = stdout.decode() - if isinstance(stderr, bytes): - stderr = stderr.decode() - if exit_code != 0: - return CancelResult( - exit_code=exit_code, - stdout=stdout, - stderr=stderr, - status=CancelStatus("FAILED"), - ) - _SLURM_CANCELLED_REGEXP = re.compile( - r"(.*:\s*)?(Terminating job)\s+(?P\d+)" - ) - match = _SLURM_CANCELLED_REGEXP.match(stderr.strip()) - job_id = match.group("jobid") if match else None - status = ( - CancelStatus("SUCCESSFUL") if job_id else CancelStatus("JOB_ID_UNKNOWN") - ) - return CancelResult( - job_id=job_id, - exit_code=exit_code, - stdout=stdout, - stderr=stderr, - status=status, - ) - - def get_job(self, job: QJob | int | str, inplace=False, get_job_cmd="scontrol"): - # TODO: there are two options to get info on a job in slurm: - # - scontrol show job JOB_ID - # - sacct -j JOB_ID - # sacct is only available when a database is running (slurmdbd). - # I guess most of the time, the clusters - # will have that in place. scontrol is only available for queued - # or running jobs (not completed ones), - # at least it disappears rapidly. Currently I am only - # using/implementing scontrol. - if inplace and not isinstance(job, QJob): - raise RuntimeError( - "Cannot update job information inplace when the QJob is not provided." - ) - job_id = job.job_id if isinstance(job, QJob) else job - if get_job_cmd == "scontrol": - # -o is to get the output as a one-liner - cmd = f"scontrol show job -o {job_id}" - elif get_job_cmd == "sacct": - raise NotImplementedError("sacct for get_job not yet implemented.") - else: - raise RuntimeError(f'"{get_job_cmd}" is not a valid get_job_cmd.') - stdout, stderr, returncode = self.execute_cmd(cmd) - scontrol_out = self._parse_scontrol_cmd_output( - exit_code=returncode, stdout=stdout, stderr=stderr - ) - slurm_state = SlurmState(scontrol_out["JobState"]) - job_state = self._STATUS_MAPPING[slurm_state] - print(type(job_state)) - print("\n".join([f"{k}: {v}" for k, v in scontrol_out.items()])) - resources = QResources( - queue_name=scontrol_out["Partition"], - memory=scontrol_out["MinMemoryCPU"], - # TODO: clarify here what are tasks, cpus, cores, etc ... - # and whether this makes sense - nodes=scontrol_out["NumNodes"], - cpus_per_node=scontrol_out["NumTasks"], - cores_per_cpu=scontrol_out["CPUs/Task"], - ) - return QJob( - name=scontrol_out["JobName"], - job_id=scontrol_out["JobId"], - state=job_state, # type: ignore # mypy thinks job_state is a str - sub_state=slurm_state, - resources=resources, - ) - - def _parse_scontrol_cmd_output(self, exit_code, stdout, stderr): - if isinstance(stdout, bytes): - stdout = stdout.decode() - if isinstance(stderr, bytes): - stderr = stderr.decode() - if exit_code != 0: - return {} - return { - data.split("=", maxsplit=1)[0]: data.split("=", maxsplit=1)[1] - for data in stdout.split() - } From 861e79694aefd0351e6d90ea662043038ae6a0a7 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Fri, 5 May 2023 17:38:09 +0200 Subject: [PATCH 08/34] Fixed unit test. --- tests/{queue => io}/test_slurm.py | 8 ++++---- .../{queue => io}/slurm/parse_submit_cmd_inout.yaml | 0 2 files changed, 4 insertions(+), 4 deletions(-) rename tests/{queue => io}/test_slurm.py (69%) rename tests/test_data/{queue => io}/slurm/parse_submit_cmd_inout.yaml (100%) diff --git a/tests/queue/test_slurm.py b/tests/io/test_slurm.py similarity index 69% rename from tests/queue/test_slurm.py rename to tests/io/test_slurm.py index 21cdcc2..3456629 100644 --- a/tests/queue/test_slurm.py +++ b/tests/io/test_slurm.py @@ -3,10 +3,10 @@ import pytest from monty.serialization import loadfn -from qtoolkit.queue.slurm import SlurmQueue +from qtoolkit.io.slurm import SlurmIO TEST_DIR = Path(__file__).resolve().parents[1] / "test_data" -ref_file = TEST_DIR / "queue" / "slurm" / "parse_submit_cmd_inout.yaml" +ref_file = TEST_DIR / "io" / "slurm" / "parse_submit_cmd_inout.yaml" in_out_ref_list = loadfn(ref_file) @@ -15,6 +15,6 @@ def test_parse_submit_cmd_output(in_out_ref, test_utils): parse_cmd_output, sr_ref = test_utils.inkwargs_outref( in_out_ref, inkey="parse_submit_kwargs", outkey="submission_result_ref" ) - slurm_q = SlurmQueue() - sr = slurm_q._parse_submit_cmd_output(**parse_cmd_output) + slurm_io = SlurmIO() + sr = slurm_io.parse_submit_output(**parse_cmd_output) assert sr == sr_ref diff --git a/tests/test_data/queue/slurm/parse_submit_cmd_inout.yaml b/tests/test_data/io/slurm/parse_submit_cmd_inout.yaml similarity index 100% rename from tests/test_data/queue/slurm/parse_submit_cmd_inout.yaml rename to tests/test_data/io/slurm/parse_submit_cmd_inout.yaml From c588c19fe8779b3efef32c5e3463b82ddec9f8e4 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Fri, 14 Apr 2023 17:01:59 +0200 Subject: [PATCH 09/34] Added structure for the documentation. --- .gitignore | 2 +- .pre-commit-config.yaml | 2 +- doc/Makefile | 72 ++++++ doc/source/_static/index-images/api.svg | 50 ++++ .../_static/index-images/contributor.svg | 13 ++ .../_static/index-images/getting_started.svg | 5 + .../_static/index-images/image_licences.txt | 4 + .../_static/index-images/user_guide.svg | 47 ++++ doc/source/_static/qtoolkit.css | 147 ++++++++++++ doc/source/api/index.rst | 15 ++ doc/source/conf.py | 216 ++++++++++++++++++ doc/source/dev/index.rst | 7 + doc/source/glossary.rst | 14 ++ doc/source/index.rst | 106 +++++++++ doc/source/license.rst | 6 + doc/source/user/basics.rst | 12 + doc/source/user/building.rst | 7 + doc/source/user/index.rst | 30 +++ doc/source/user/install.rst | 9 + doc/source/user/quickstart.rst | 17 ++ doc/source/user/whatisqtoolkit.rst | 8 + doc_requirements.txt | 14 ++ src/qtoolkit/core/__init__.py | 0 src/qtoolkit/host/__init__.py | 0 src/qtoolkit/queue/__init__.py | 0 25 files changed, 801 insertions(+), 2 deletions(-) create mode 100644 doc/Makefile create mode 100644 doc/source/_static/index-images/api.svg create mode 100644 doc/source/_static/index-images/contributor.svg create mode 100644 doc/source/_static/index-images/getting_started.svg create mode 100644 doc/source/_static/index-images/image_licences.txt create mode 100644 doc/source/_static/index-images/user_guide.svg create mode 100644 doc/source/_static/qtoolkit.css create mode 100644 doc/source/api/index.rst create mode 100644 doc/source/conf.py create mode 100644 doc/source/dev/index.rst create mode 100644 doc/source/glossary.rst create mode 100644 doc/source/index.rst create mode 100644 doc/source/license.rst create mode 100644 doc/source/user/basics.rst create mode 100644 doc/source/user/building.rst create mode 100644 doc/source/user/index.rst create mode 100644 doc/source/user/install.rst create mode 100644 doc/source/user/quickstart.rst create mode 100644 doc/source/user/whatisqtoolkit.rst create mode 100644 doc_requirements.txt create mode 100644 src/qtoolkit/core/__init__.py create mode 100644 src/qtoolkit/host/__init__.py create mode 100644 src/qtoolkit/queue/__init__.py diff --git a/.gitignore b/.gitignore index a14d6d0..373876b 100644 --- a/.gitignore +++ b/.gitignore @@ -69,7 +69,7 @@ instance/ .scrapy # Sphinx documentation -docs/_build/ +doc/_build/ # PyBuilder .pybuilder/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c59ed26..91c5c57 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ default_language_version: python: python3 -#exclude: '^src/{{ package_name }}/some/directory/' +exclude: '^doc/' repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000..90d6037 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,72 @@ +# Makefile for Sphinx documentation +# + +# PYVER needs to be major.minor, just "3" doesn't work - it will result in +# issues with the amendments to PYTHONPATH and install paths (see DIST_VARS). + +# Use explicit "version_info" indexing since make cannot handle colon characters, and +# evaluate it now to allow easier debugging when printing the variable + +PYVER:=$(shell python3 -c 'from sys import version_info as v; print("{0}.{1}".format(v[0], v[1]))') +PYTHON = python$(PYVER) + +# You can set these variables from the command line. +SPHINXOPTS ?= +SPHINXBUILD ?= LANG=C sphinx-build +PAPER ?= +# # For merging a documentation archive into a git checkout of numpy/doc +# # Turn a tag like v1.18.0 into 1.18 +# # Use sed -n -e 's/patttern/match/p' to return a blank value if no match +# TAG ?= $(shell git describe --tag | sed -n -e's,v\([1-9]\.[0-9]*\)\.[0-9].*,\1,p') + +FILES= + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -WT --keep-going -d build/doctrees $(PAPEROPT_$(PAPER)) \ + $(SPHINXOPTS) source + +.PHONY: help clean html version-check html-build + +#------------------------------------------------------------------------------ + +help: + @echo "Please use \`make ' where is one of" + @echo " clean to remove generated doc files and start fresh" + @echo " html to make standalone HTML files" + +clean: + -rm -rf build/* + find . -name generated -type d -prune -exec rm -rf "{}" ";" + + +#------------------------------------------------------------------------------ +# Automated generation of all documents +#------------------------------------------------------------------------------ + +# Build the current QToolKit version, and extract docs from it. +# We have to be careful of some issues: +# +# - Everything must be done using the same Python version +# + +#SPHINXBUILD="LANG=C sphinx-build" + + +#------------------------------------------------------------------------------ +# Basic Sphinx generation rules for different formats +#------------------------------------------------------------------------------ +generate: build/generate-stamp +build/generate-stamp: $(wildcard source/reference/*.rst) + mkdir -p build + touch build/generate-stamp + +html: api-doc html-build +html-build: generate + mkdir -p build/html build/doctrees + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) build/html $(FILES) + @echo + @echo "Build finished. The HTML pages are in build/html." +api-doc: + sphinx-apidoc -f -o source/api ../src/qtoolkit diff --git a/doc/source/_static/index-images/api.svg b/doc/source/_static/index-images/api.svg new file mode 100644 index 0000000..9c88397 --- /dev/null +++ b/doc/source/_static/index-images/api.svg @@ -0,0 +1,50 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/source/_static/index-images/contributor.svg b/doc/source/_static/index-images/contributor.svg new file mode 100644 index 0000000..ffd444e --- /dev/null +++ b/doc/source/_static/index-images/contributor.svg @@ -0,0 +1,13 @@ + + + + + + + + \ No newline at end of file diff --git a/doc/source/_static/index-images/getting_started.svg b/doc/source/_static/index-images/getting_started.svg new file mode 100644 index 0000000..20747f9 --- /dev/null +++ b/doc/source/_static/index-images/getting_started.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/doc/source/_static/index-images/image_licences.txt b/doc/source/_static/index-images/image_licences.txt new file mode 100644 index 0000000..85276bc --- /dev/null +++ b/doc/source/_static/index-images/image_licences.txt @@ -0,0 +1,4 @@ +getting_started.svg: https://www.svgrepo.com/svg/393367/rocket (PD Licence) +user_guide.svg: https://www.svgrepo.com/svg/75531/user-guide (CC0 Licence) +api.svg: https://www.svgrepo.com/svg/157898/gears-configuration-tool (CC0 Licence) +contributor.svg: https://www.svgrepo.com/svg/57189/code-programing-symbol (CC0 Licence) \ No newline at end of file diff --git a/doc/source/_static/index-images/user_guide.svg b/doc/source/_static/index-images/user_guide.svg new file mode 100644 index 0000000..6223a92 --- /dev/null +++ b/doc/source/_static/index-images/user_guide.svg @@ -0,0 +1,47 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/source/_static/qtoolkit.css b/doc/source/_static/qtoolkit.css new file mode 100644 index 0000000..4a77b9c --- /dev/null +++ b/doc/source/_static/qtoolkit.css @@ -0,0 +1,147 @@ +@import url('https://fonts.googleapis.com/css2?family=Lato:ital,wght@0,400;0,700;0,900;1,400;1,700;1,900&family=Open+Sans:ital,wght@0,400;0,600;1,400;1,600&display=swap'); + +.navbar-brand img { + height: 75px; +} +.navbar-brand { + height: 75px; +} + +body { + font-family: 'Open Sans', sans-serif; +} + +pre, code { + font-size: 100%; + line-height: 155%; +} + +h1 { + font-family: "Lato", sans-serif; + color: #013243; /* warm black */ +} + +h2 { + color: #4d77cf; /* han blue */ + letter-spacing: -.03em; +} + +h3 { + color: #013243; /* warm black */ + letter-spacing: -.03em; +} + +/* Style the active version button. + +- dev: orange +- stable: green +- old, PR: red + +Colors from: + +Wong, B. Points of view: Color blindness. +Nat Methods 8, 441 (2011). https://doi.org/10.1038/nmeth.1618 +*/ + +/* If the active version has the name "dev", style it orange */ +#version_switcher_button[data-active-version-name*="dev"] { + background-color: #E69F00; + border-color: #E69F00; + color:#000000; +} + +/* green for `stable` */ +#version_switcher_button[data-active-version-name*="stable"] { + background-color: #009E73; + border-color: #009E73; +} + +/* red for `old` */ +#version_switcher_button:not([data-active-version-name*="stable"], [data-active-version-name*="dev"], [data-active-version-name=""]) { + background-color: #980F0F; + border-color: #980F0F; +} + +/* Main page overview cards */ + +.sd-card { + background: #fff; + border-radius: 0; + padding: 30px 10px 20px 10px; + margin: 10px 0px; +} + +.sd-card .sd-card-header { + text-align: center; +} + +.sd-card .sd-card-header .sd-card-text { + margin: 0px; +} + +.sd-card .sd-card-img-top { + height: 52px; + width: 52px; + margin-left: auto; + margin-right: auto; +} + +.sd-card .sd-card-header { + border: none; + background-color: white; + color: #150458 !important; + font-size: var(--pst-font-size-h5); + font-weight: bold; + padding: 2.5rem 0rem 0.5rem 0rem; +} + +.sd-card .sd-card-footer { + border: none; + background-color: white; +} + +.sd-card .sd-card-footer .sd-card-text { + max-width: 220px; + margin-left: auto; + margin-right: auto; +} + +/* Announcements */ +.bd-header-announcement { + background-color: orange; +} + +/* Dark theme tweaking */ +html[data-theme=dark] .sd-card img[src*='.svg'] { + filter: invert(0.82) brightness(0.8) contrast(1.2); +} + +/* Main index page overview cards */ +html[data-theme=dark] .sd-card { + background-color:var(--pst-color-background); +} + +html[data-theme=dark] .sd-shadow-sm { + box-shadow: 0 .1rem 1rem rgba(250, 250, 250, .6) !important +} + +html[data-theme=dark] .sd-card .sd-card-header { + background-color:var(--pst-color-background); + color: #150458 !important; +} + +html[data-theme=dark] .sd-card .sd-card-footer { + background-color:var(--pst-color-background); +} + +html[data-theme=dark] .bd-header-announcement { + background-color: red; +} + +html[data-theme=dark] h1 { + color: var(--pst-color-primary); +} + +html[data-theme=dark] h3 { + color: #0a6774; +} diff --git a/doc/source/api/index.rst b/doc/source/api/index.rst new file mode 100644 index 0000000..daebaf2 --- /dev/null +++ b/doc/source/api/index.rst @@ -0,0 +1,15 @@ +.. _api: + +============= +API Reference +============= + +This is the API reference + +.. toctree:: + :maxdepth: 1 + + qtoolkit.core + qtoolkit.host + qtoolkit.io + qtoolkit.queue \ No newline at end of file diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 0000000..8ace3c5 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,216 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +# sys.path.insert(0, os.path.abspath('.')) +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +) + +# -- Project information ----------------------------------------------------- + +project = "QToolKit" +copyright = "2023, Matgenix SRL" +author = "Guido Petretto, David Waroquiers" + + +import qtoolkit +# The short X.Y version +version = qtoolkit.__version__ +# The full version, including alpha/beta/rc tags +release = qtoolkit.__version__ + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.viewcode", + "sphinx.ext.napoleon", # For Google Python Style Guide + 'sphinx.ext.coverage', + 'sphinx.ext.doctest', + 'sphinx.ext.autosummary', + 'sphinx.ext.graphviz', + 'sphinx.ext.ifconfig', + 'matplotlib.sphinxext.plot_directive', + 'IPython.sphinxext.ipython_console_highlighting', + 'IPython.sphinxext.ipython_directive', + 'sphinx.ext.mathjax', + 'sphinx_design', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +# html_theme = 'sphinx_book_theme' +html_theme = 'pydata_sphinx_theme' +# html_favicon = '_static/favicon/favicon.ico' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +html_theme_options = { + # "logo": { + # "image_light": "index-image/api.svg", + # "image_dark": "index-image/contributor.svg", + # }, + "collapse_navigation": True, + 'announcement': ( + "

" + "QToolKit is still in beta phase. The API may change at any time." + "

" + ), + # "announcement": "

This is still in development

", + # "navbar_end": ["theme-switcher", "navbar-icon-links"], + # "navbar_end": ["theme-switcher", "version-switcher", "navbar-icon-links"], +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +html_css_files = ["qtoolkit.css"] +html_title = "%s v%s Manual" % (project, version) +html_last_updated_fmt = '%b %d, %Y' +# html_css_files = ["numpy.css"] +html_context = {"default_mode": "light"} +html_use_modindex = True +html_copy_source = False +html_domain_indices = False +html_file_suffix = '.html' + +# Output file base name for HTML help builder. +htmlhelp_basename = "qtoolkitdoc" + + +# # -- Options for LaTeX output ------------------------------------------------ +# +# latex_elements = { +# # The paper size ('letterpaper' or 'a4paper'). +# # +# # 'papersize': 'letterpaper', +# # The font size ('10pt', '11pt' or '12pt'). +# # +# # 'pointsize': '10pt', +# # Additional stuff for the LaTeX preamble. +# # +# # 'preamble': '', +# # Latex figure (float) alignment +# # +# # 'figure_align': 'htbp', +# } +# +# # Grouping the document tree into LaTeX files. List of tuples +# # (source start file, target name, title, +# # author, documentclass [howto, manual, or own class]). +# latex_documents = [ +# # (master_doc, "turbomoleio.tex", "turbomoleio Documentation", author, "manual"), +# ] + + +# # -- Options for manual page output ------------------------------------------ +# +# # One entry per manual page. List of tuples +# # (source start file, name, description, authors, manual section). +# man_pages = [(master_doc, "turbomoleio", "turbomoleio Documentation", [author], 1)] +# +# +# # -- Options for Texinfo output ---------------------------------------------- +# +# # Grouping the document tree into Texinfo files. List of tuples +# # (source start file, target name, title, author, +# # dir menu entry, description, category) +# texinfo_documents = [ +# ( +# master_doc, +# "turbomoleio", +# "turbomoleio Documentation", +# author, +# "turbomoleio", +# "One line description of project.", +# "Miscellaneous", +# ), +# ] + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for intersphinx extension --------------------------------------- + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {"https://docs.python.org/": None} + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + +# To print the content of the docstring of the __init__ method as well. +autoclass_content = "both" diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst new file mode 100644 index 0000000..5bf8af1 --- /dev/null +++ b/doc/source/dev/index.rst @@ -0,0 +1,7 @@ +.. _devindex: + +######################## +Contributing to QToolKit +######################## + +Here are the things that can be done. \ No newline at end of file diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst new file mode 100644 index 0000000..3850067 --- /dev/null +++ b/doc/source/glossary.rst @@ -0,0 +1,14 @@ +******** +Glossary +******** + +.. glossary:: + + + QJob + The representation of a job in the queue. + + + QResources + The description of the resources for a job. + diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000..39690e7 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,106 @@ +.. _qtoolkit_docs_mainpage: + +###################### +QToolKit documentation +###################### + +.. toctree:: + :maxdepth: 1 + :hidden: + + User Guide + API reference + Development + release + + +**Version**: |version| + +**Useful links**: +TO BE ADDED + +QToolKit is an interface to most Distributed Resource Management systems, e.g. +SLURM, PBS, ... + + + +.. grid:: 1 2 2 2 + + .. grid-item-card:: + :img-top: ../source/_static/index-images/getting_started.svg + + Getting Started + ^^^^^^^^^^^^^^^ + + New to QToolKit? Check out the Absolute Beginner's Guide. It contains an + introduction to QToolKit's main concepts and links to additional tutorials. + + +++ + + .. button-ref:: user/absolute_beginners + :expand: + :color: secondary + :click-parent: + + To the absolute beginner's guide + + .. grid-item-card:: + :img-top: ../source/_static/index-images/user_guide.svg + + User Guide + ^^^^^^^^^^ + + The user guide provides in-depth information on the + key concepts of QToolKit with useful background information and explanation. + + +++ + + .. button-ref:: user + :expand: + :color: secondary + :click-parent: + + To the user guide + + .. grid-item-card:: + :img-top: ../source/_static/index-images/api.svg + + API Reference + ^^^^^^^^^^^^^ + + The reference guide contains a detailed description of the functions, + modules, and objects included in QToolKit. The reference describes how the + methods work and which parameters can be used. It assumes that you have an + understanding of the key concepts. + + +++ + + .. button-ref:: api + :expand: + :color: secondary + :click-parent: + + To the reference guide + + .. grid-item-card:: + :img-top: ../source/_static/index-images/contributor.svg + + Contributor's Guide + ^^^^^^^^^^^^^^^^^^^ + + Want to add to the codebase? Can help add support to an additional DRM system? + The contributing guidelines will guide you through the + process of improving QToolKit. + + +++ + + .. button-ref:: devindex + :expand: + :color: secondary + :click-parent: + + To the contributor's guide + +.. This is not really the index page, that is found in + _templates/indexcontent.html The toctree content here will be added to the + top of the template header \ No newline at end of file diff --git a/doc/source/license.rst b/doc/source/license.rst new file mode 100644 index 0000000..f36eb5b --- /dev/null +++ b/doc/source/license.rst @@ -0,0 +1,6 @@ +**************** +QToolKit license +**************** + +.. include:: ../../LICENSE + :literal: diff --git a/doc/source/user/basics.rst b/doc/source/user/basics.rst new file mode 100644 index 0000000..67af9c2 --- /dev/null +++ b/doc/source/user/basics.rst @@ -0,0 +1,12 @@ +********************* +QToolKit fundamentals +********************* + +These documents clarify concepts, design decisions, and technical +constraints in QToolKit. This is a great place to understand the +fundamental QToolKit ideas and philosophy. + +.. + .. toctree:: + :maxdepth: 1 + diff --git a/doc/source/user/building.rst b/doc/source/user/building.rst new file mode 100644 index 0000000..93af682 --- /dev/null +++ b/doc/source/user/building.rst @@ -0,0 +1,7 @@ +.. _building-from-source: + +Building from source +==================== + +Get the source from the git repository. +Install it with pip install . \ No newline at end of file diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst new file mode 100644 index 0000000..5155edf --- /dev/null +++ b/doc/source/user/index.rst @@ -0,0 +1,30 @@ +.. _user: + +################### +QToolKit user guide +################### + +This guide is an overview and explains the important features; +details are found in :ref:`reference`. + +.. toctree:: + :caption: Getting started + :maxdepth: 1 + + whatisqtoolkit + install + quickstart + +.. toctree:: + :caption: Advanced usage and interoperability + :maxdepth: 1 + + building + + +.. toctree:: + :hidden: + :caption: Extras + + ../glossary + ../license diff --git a/doc/source/user/install.rst b/doc/source/user/install.rst new file mode 100644 index 0000000..50cd00e --- /dev/null +++ b/doc/source/user/install.rst @@ -0,0 +1,9 @@ +.. _install: + +******************* +Installing QToolKit +******************* + +The only prerequisite for installing QToolKit is Python. + +QToolKit can be installed with 'conda', with 'pip', or from source. \ No newline at end of file diff --git a/doc/source/user/quickstart.rst b/doc/source/user/quickstart.rst new file mode 100644 index 0000000..da9978c --- /dev/null +++ b/doc/source/user/quickstart.rst @@ -0,0 +1,17 @@ +.. _quickstart: + +=================== +QToolKit quickstart +=================== + +Prerequisites +============= +You need python + +The Basics +========== + +Create an easy script + +ADD OTHER TOPICS: +remote submission, resources, ... \ No newline at end of file diff --git a/doc/source/user/whatisqtoolkit.rst b/doc/source/user/whatisqtoolkit.rst new file mode 100644 index 0000000..a302ec4 --- /dev/null +++ b/doc/source/user/whatisqtoolkit.rst @@ -0,0 +1,8 @@ +.. _whatisqtoolkit: + +================= +What is QToolKit? +================= + +QToolKit is ... +TODO: add the features that it has. \ No newline at end of file diff --git a/doc_requirements.txt b/doc_requirements.txt new file mode 100644 index 0000000..53b03cc --- /dev/null +++ b/doc_requirements.txt @@ -0,0 +1,14 @@ +# doxygen required, use apt-get or dnf +sphinx>=4.5.0 +numpydoc==1.4 +pydata-sphinx-theme==0.13.3 +sphinx-design +sphinx-apidoc +ipython!=8.1.0 +scipy +matplotlib +pandas +breathe + +# needed to build release notes +towncrier diff --git a/src/qtoolkit/core/__init__.py b/src/qtoolkit/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/qtoolkit/host/__init__.py b/src/qtoolkit/host/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/qtoolkit/queue/__init__.py b/src/qtoolkit/queue/__init__.py new file mode 100644 index 0000000..e69de29 From b0650d4852cb39c22418749392c813e6b4804e91 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Fri, 14 Apr 2023 17:10:26 +0200 Subject: [PATCH 10/34] Fixed navigation for api. --- doc/source/api/index.rst | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/doc/source/api/index.rst b/doc/source/api/index.rst index daebaf2..484de6a 100644 --- a/doc/source/api/index.rst +++ b/doc/source/api/index.rst @@ -1,15 +1,9 @@ .. _api: -============= +############# API Reference -============= +############# This is the API reference -.. toctree:: - :maxdepth: 1 - - qtoolkit.core - qtoolkit.host - qtoolkit.io - qtoolkit.queue \ No newline at end of file +.. include:: qtoolkit.rst \ No newline at end of file From 9a76d824ee335812e8baa2fa7258d80f062322ce Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Sat, 29 Apr 2023 23:39:55 +0200 Subject: [PATCH 11/34] Updated css styling (Matgenix colors). --- doc/Makefile | 2 +- doc/source/_static/qtoolkit.css | 15 +++++++++++++++ doc/source/index.rst | 12 ++++++------ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/doc/Makefile b/doc/Makefile index 90d6037..3e5f5da 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -69,4 +69,4 @@ html-build: generate @echo @echo "Build finished. The HTML pages are in build/html." api-doc: - sphinx-apidoc -f -o source/api ../src/qtoolkit + sphinx-apidoc -e -f -o source/api ../src/qtoolkit diff --git a/doc/source/_static/qtoolkit.css b/doc/source/_static/qtoolkit.css index 4a77b9c..4561c96 100644 --- a/doc/source/_static/qtoolkit.css +++ b/doc/source/_static/qtoolkit.css @@ -1,5 +1,10 @@ @import url('https://fonts.googleapis.com/css2?family=Lato:ital,wght@0,400;0,700;0,900;1,400;1,700;1,900&family=Open+Sans:ital,wght@0,400;0,600;1,400;1,600&display=swap'); +:root { + --matgenix-color: #46b3c1; + --matgenix-dark-color: #338d99; +} + .navbar-brand img { height: 75px; } @@ -145,3 +150,13 @@ html[data-theme=dark] h1 { html[data-theme=dark] h3 { color: #0a6774; } + +.sd-btn-secondary { + background-color: var(--matgenix-color) !important; + border-color: var(--matgenix-color) !important; +} + +.sd-btn-secondary:hover, .sd-btn-secondary:focus { + background-color: var(--matgenix-dark-color) !important; + border-color: var(--matgenix-dark-color) !important; +} \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst index 39690e7..46f8eef 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -32,17 +32,17 @@ SLURM, PBS, ... Getting Started ^^^^^^^^^^^^^^^ - New to QToolKit? Check out the Absolute Beginner's Guide. It contains an - introduction to QToolKit's main concepts and links to additional tutorials. + If you want to get started quickly, check out our quickstart section. + It contains an introduction to QToolKit's main concepts. +++ - .. button-ref:: user/absolute_beginners + .. button-ref:: user/quickstart :expand: :color: secondary :click-parent: - To the absolute beginner's guide + Quickstart .. grid-item-card:: :img-top: ../source/_static/index-images/user_guide.svg @@ -60,7 +60,7 @@ SLURM, PBS, ... :color: secondary :click-parent: - To the user guide + User Guide .. grid-item-card:: :img-top: ../source/_static/index-images/api.svg @@ -80,7 +80,7 @@ SLURM, PBS, ... :color: secondary :click-parent: - To the reference guide + API Reference .. grid-item-card:: :img-top: ../source/_static/index-images/contributor.svg From 5b1c02ecbccfdeab9a1666409c59a5998608c02c Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Sat, 29 Apr 2023 23:50:19 +0200 Subject: [PATCH 12/34] Added optional deps for docs. --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1432ea9..92453e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,11 @@ tests = [ maintain = [ "git-changelog>=0.6", ] +docs = [ + "sphinx", + "sphinx_design", + "pydata-sphinx-theme", +] strict = [] remote = ["fabric>=3.0.0"] msonable = ["monty>=2022.9.9",] From f1764efb630cbf8e81729dd8ec76842cc04d7e81 Mon Sep 17 00:00:00 2001 From: Guido Petretto Date: Mon, 29 May 2023 09:51:08 +0200 Subject: [PATCH 13/34] modify slurm template --- src/qtoolkit/io/__init__.py | 6 ++++++ src/qtoolkit/io/base.py | 4 +++- src/qtoolkit/io/slurm.py | 6 +++--- src/qtoolkit/manager.py | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/qtoolkit/io/__init__.py b/src/qtoolkit/io/__init__.py index e69de29..a4d31ea 100644 --- a/src/qtoolkit/io/__init__.py +++ b/src/qtoolkit/io/__init__.py @@ -0,0 +1,6 @@ +from qtoolkit.io.base import BaseSchedulerIO +from qtoolkit.io.pbs import PBSIO, PBSState +from qtoolkit.io.shell import ShellIO, ShellState +from qtoolkit.io.slurm import SlurmIO, SlurmState + +scheduler_mapping = {"slurm": SlurmIO, "pbs": PBSIO} diff --git a/src/qtoolkit/io/base.py b/src/qtoolkit/io/base.py index b814409..3c4ee5d 100644 --- a/src/qtoolkit/io/base.py +++ b/src/qtoolkit/io/base.py @@ -111,7 +111,7 @@ class BaseSchedulerIO(QBase): def get_submission_script( self, commands: str | list[str], - options: dict | None = None, + options: dict | QResources | None = None, ) -> str: """ This is roughly what/how it is done in the existing solutions. @@ -205,6 +205,8 @@ def generate_header(self, options: dict | QResources | None) -> str: # resources (nodes, cores, memory, time, [gpus]) # default values for (almost) everything in the object ? + options = options or {} + if isinstance(options, QResources): options = self.check_convert_qresources(options) diff --git a/src/qtoolkit/io/slurm.py b/src/qtoolkit/io/slurm.py index d70bb0c..41b1e99 100644 --- a/src/qtoolkit/io/slurm.py +++ b/src/qtoolkit/io/slurm.py @@ -125,10 +125,10 @@ def qstate(self) -> QState: class SlurmIO(BaseSchedulerIO): header_template: str = """ -#SBATCH --partition=$${queue_name} +#SBATCH --partition=$${partition} #SBATCH --job-name=$${job_name} -#SBATCH --nodes=$${number_of_nodes} -#SBATCH --ntasks=$${number_of_tasks} +#SBATCH --nodes=$${nodes} +#SBATCH --ntasks=$${ntasks} #SBATCH --ntasks-per-node=$${ntasks_per_node} #SBATCH --cpus-per-task=$${cpus_per_task} #SBATCH --mem=$${mem} diff --git a/src/qtoolkit/manager.py b/src/qtoolkit/manager.py index 05cd9e8..80f7a07 100644 --- a/src/qtoolkit/manager.py +++ b/src/qtoolkit/manager.py @@ -45,7 +45,7 @@ def execute_cmd(self, cmd: str, workdir: str | Path | None = None): def get_submission_script( self, commands: str | list[str] | None, - options: dict | QResources = None, + options: dict | QResources | None = None, work_dir: str | Path | None = None, pre_run: str | list[str] | None = None, post_run: str | list[str] | None = None, From 9c30d5fb57c11ee25daf7eea38b9d8428c28172a Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 7 Jun 2023 10:45:12 +0200 Subject: [PATCH 14/34] Adding some tests and documentation. --- src/qtoolkit/io/base.py | 2 +- tests/conftest.py | 14 ++- tests/core/__init__.py | 0 tests/core/test_base.py | 10 +- tests/core/test_data_objects.py | 211 ++++++++++++++++++++++++++++++++ tests/io/__init__.py | 0 tests/io/test_base.py | 107 ++++++++++++++++ 7 files changed, 335 insertions(+), 9 deletions(-) create mode 100644 tests/core/__init__.py create mode 100644 tests/core/test_data_objects.py create mode 100644 tests/io/__init__.py create mode 100644 tests/io/test_base.py diff --git a/src/qtoolkit/io/base.py b/src/qtoolkit/io/base.py index 3c4ee5d..5848478 100644 --- a/src/qtoolkit/io/base.py +++ b/src/qtoolkit/io/base.py @@ -37,7 +37,7 @@ def get_identifiers(self) -> list: return ids -class BaseSchedulerIO(QBase): +class BaseSchedulerIO(QBase, abc.ABC): """Base class for job queues. Attributes diff --git a/tests/conftest.py b/tests/conftest.py index c9305da..372d7a6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -82,15 +82,23 @@ class TestUtils: from monty.serialization import MontyDecoder, MontyEncoder @classmethod - def is_msonable(cls, obj): + def is_msonable(cls, obj, obj_cls=None): if not isinstance(obj, cls.MSONable): return False obj_dict = obj.as_dict() if not obj_dict == obj.__class__.from_dict(obj_dict).as_dict(): return False json_string = cls.json.dumps(obj_dict, cls=cls.MontyEncoder) - obj_dict_from_json = cls.json.loads(json_string, cls=cls.MontyDecoder) - return obj_dict_from_json == obj_dict + obj_from_json = cls.json.loads(json_string, cls=cls.MontyDecoder) + # When the class is defined as an inner class, the MontyDecoder is unable + # to find it automatically. This is only used in the core/test_base tests. + # The next check on the type of the obj_from_json is of course not relevant + # in that specific case. + if obj_cls is not None: + obj_from_json = obj_cls.from_dict(obj_from_json) + if not isinstance(obj_from_json, obj.__class__): + return False + return obj_from_json == obj @classmethod def inkwargs_outref(cls, in_out_ref, inkey, outkey): diff --git a/tests/core/__init__.py b/tests/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/core/test_base.py b/tests/core/test_base.py index 052930e..1ed4e8b 100644 --- a/tests/core/test_base.py +++ b/tests/core/test_base.py @@ -51,7 +51,7 @@ class QClass(qbase.QBase): name: str = "name" qc = QClass() - assert test_utils.is_msonable(qc) + assert test_utils.is_msonable(qc, obj_cls=QClass) def test_not_msonable(self, test_utils, qtk_core_base_mocked_monty_not_found): @dataclass @@ -72,11 +72,11 @@ class SomeEnum(qbase.QEnum): VAL2 = "VAL2" se = SomeEnum("VAL1") - assert test_utils.is_msonable(se) + assert test_utils.is_msonable(se, obj_cls=SomeEnum) assert isinstance(se, enum.Enum) se = SomeEnum.VAL2 - assert test_utils.is_msonable(se) + assert test_utils.is_msonable(se, obj_cls=SomeEnum) assert isinstance(se, enum.Enum) class SomeEnum(qbase.QEnum): @@ -84,11 +84,11 @@ class SomeEnum(qbase.QEnum): VAL2 = 4 se = SomeEnum(3) - assert test_utils.is_msonable(se) + assert test_utils.is_msonable(se, obj_cls=SomeEnum) assert isinstance(se, enum.Enum) se = SomeEnum.VAL2 - assert test_utils.is_msonable(se) + assert test_utils.is_msonable(se, obj_cls=SomeEnum) assert isinstance(se, enum.Enum) def test_not_msonable(self, test_utils, qtk_core_base_mocked_monty_not_found): diff --git a/tests/core/test_data_objects.py b/tests/core/test_data_objects.py new file mode 100644 index 0000000..a680e33 --- /dev/null +++ b/tests/core/test_data_objects.py @@ -0,0 +1,211 @@ +"""Unit tests for the core.data_objects module of QToolKit.""" +import pytest + +from qtoolkit.core.data_objects import ( + CancelResult, + CancelStatus, + ProcessPlacement, + QState, + QSubState, + SubmissionResult, + SubmissionStatus, +) + +try: + import monty +except ModuleNotFoundError: + monty = None + + +class TestSubmissionStatus: + def test_status_list(self): + all_status = [status.value for status in SubmissionStatus] + assert set(all_status) == {"SUCCESSFUL", "FAILED", "JOB_ID_UNKNOWN"} + + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + status = SubmissionStatus("SUCCESSFUL") + assert test_utils.is_msonable(status) + + def test_equal(self): + assert SubmissionStatus("SUCCESSFUL") == SubmissionStatus.SUCCESSFUL + + +class TestSubmissionResult: + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + sr = SubmissionResult( + job_id="abc123", + step_id=1, + exit_code=1, + stdout="mystdout", + stderr="mystderr", + status=SubmissionStatus.FAILED, + ) + assert test_utils.is_msonable(sr) + + def test_equality(self): + sr1 = SubmissionResult( + job_id=None, + step_id=1, + exit_code=1, + stdout="mystdout", + stderr="mystderr", + status=SubmissionStatus.JOB_ID_UNKNOWN, + ) + sr2 = SubmissionResult( + job_id="abc123", + step_id=1, + exit_code=1, + stdout="mystdout", + stderr="mystderr", + status=SubmissionStatus.FAILED, + ) + sr3 = SubmissionResult( + job_id="abc123", + step_id=1, + exit_code=1, + stdout="mystdout", + stderr="mystderr", + status=SubmissionStatus.FAILED, + ) + assert sr2 == sr3 + assert sr1 != sr2 + assert sr1 != sr3 + + +class TestCancelStatus: + def test_status_list(self): + all_status = [status.value for status in CancelStatus] + assert set(all_status) == {"SUCCESSFUL", "FAILED", "JOB_ID_UNKNOWN"} + + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + status = CancelStatus("FAILED") + assert test_utils.is_msonable(status) + + def test_equal(self): + assert CancelStatus("JOB_ID_UNKNOWN") == CancelStatus.JOB_ID_UNKNOWN + assert CancelStatus("JOB_ID_UNKNOWN") != SubmissionStatus.JOB_ID_UNKNOWN + + +class TestCancelResult: + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + cr = CancelResult( + job_id="abc123", + step_id=1, + exit_code=1, + stdout="mystdout", + stderr="mystderr", + status=CancelStatus.FAILED, + ) + assert test_utils.is_msonable(cr) + + def test_equality(self): + cr1 = CancelResult( + job_id="abc123", + step_id=1, + exit_code=0, + stdout="mystdout", + stderr="mystderr", + status=CancelStatus.SUCCESSFUL, + ) + cr2 = CancelResult( + job_id="abc123", + step_id=1, + exit_code=1, + stdout="mystdout", + stderr="mystderr", + status=CancelStatus.FAILED, + ) + cr3 = CancelResult( + job_id="abc123", + step_id=1, + exit_code=1, + stdout="mystdout", + stderr="mystderr", + status=CancelStatus.FAILED, + ) + assert cr2 == cr3 + assert cr1 != cr2 + assert cr1 != cr3 + + +class TestQState: + def test_states_list(self): + all_states = [state.value for state in QState] + assert set(all_states) == { + "UNDETERMINED", + "QUEUED", + "QUEUED_HELD", + "RUNNING", + "SUSPENDED", + "REQUEUED", + "REQUEUED_HELD", + "DONE", + "FAILED", + } + + def test_equality(self): + state1 = QState("QUEUED") + state2 = QState("RUNNING") + state3 = QState.RUNNING + assert state1 != state2 + assert state1 != state3 + assert state2 == state3 + + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + state = QState.DONE + assert test_utils.is_msonable(state) + + +class SomeSubState(QSubState): + STATE_A = "stateA", "STA" + STATE_B = "stateB", "STB", "sb" + STATE_C = "sc" + + def qstate(self) -> QState: + return QState.DONE + + +class TestQSubState: + def test_equality(self): + sst1 = SomeSubState.STATE_B + sst2 = SomeSubState("stateB") + sst3 = SomeSubState("STB") + sst4 = SomeSubState("sb") + assert sst1 == sst2 + assert sst1 == sst3 + assert sst1 == sst4 + + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + sst1 = SomeSubState.STATE_A + assert test_utils.is_msonable(sst1) + sst2 = SomeSubState("sb") + assert test_utils.is_msonable(sst2) + + def test_repr(self): + sst1 = SomeSubState.STATE_B + sst2 = SomeSubState("stateB") + sst3 = SomeSubState("STB") + sst4 = SomeSubState("sb") + assert repr(sst1) == "" + assert repr(sst1) == repr(sst2) + assert repr(sst1) == repr(sst3) + assert repr(sst1) == repr(sst4) + + +class TestProcessPlacement: + def test_process_placement_list(self): + all_process_placements = [ + process_placement.value for process_placement in ProcessPlacement + ] + assert set(all_process_placements) == { + "NO_CONSTRAINTS", + "SCATTERED", + "SAME_NODE", + "EVENLY_DISTRIBUTED", + } diff --git a/tests/io/__init__.py b/tests/io/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/io/test_base.py b/tests/io/test_base.py new file mode 100644 index 0000000..b2f7365 --- /dev/null +++ b/tests/io/test_base.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest +from monty.serialization import loadfn + +from qtoolkit.core.data_objects import CancelResult, QJob, QResources, SubmissionResult +from qtoolkit.io.base import BaseSchedulerIO, QTemplate + +TEST_DIR = Path(__file__).resolve().parents[1] / "test_data" +ref_file = TEST_DIR / "io" / "slurm" / "parse_submit_cmd_inout.yaml" +in_out_ref_list = loadfn(ref_file) + + +def test_qtemplate(): + template_str = """This is a template + with some $$substitutions + another $${tata}te""" + template = QTemplate(template_str) + identifiers = set(template.get_identifiers()) + assert identifiers == {"substitutions", "tata"} + substituted_template = template.safe_substitute({"substitutions": "mysubstitution"}) + assert ( + substituted_template + == """This is a template + with some mysubstitution + another $${tata}te""" + ) + substituted_template = template.safe_substitute({}) + assert ( + substituted_template + == """This is a template + with some $$substitutions + another $${tata}te""" + ) + substituted_template = template.safe_substitute({"tata": "pata"}) + assert ( + substituted_template + == """This is a template + with some $$substitutions + another patate""" + ) + + template_str = """Multi template $$subst + $${subst1}$${subst2} + $${subst3}$${subst3}""" + template = QTemplate(template_str) + identifiers = template.get_identifiers() + assert len(identifiers) == 4 + assert set(identifiers) == {"subst", "subst1", "subst2", "subst3"} + substituted_template = template.safe_substitute({"subst3": "to", "subst": "bla"}) + assert ( + substituted_template + == """Multi template bla + $${subst1}$${subst2} + toto""" + ) + + +def test_base_scheduler(): + class MyScheduler(BaseSchedulerIO): + pass + + with pytest.raises(TypeError): + MyScheduler() + + class MyScheduler(BaseSchedulerIO): + header_template = """#SPECCMD --option1=$${option1} +#SPECCMD --option2=$${option2} +#SPECCMD --option3=$${option3}""" + + SUBMIT_CMD = "mysubmit" + CANCEL_CMD = "mycancel" + + def parse_submit_output(self, exit_code, stdout, stderr) -> SubmissionResult: + pass + + def parse_cancel_output(self, exit_code, stdout, stderr) -> CancelResult: + pass + + def _get_job_cmd(self, job_id: str) -> str: + pass + + def parse_job_output(self, exit_code, stdout, stderr) -> QJob | None: + pass + + def _convert_qresources(self, resources: QResources) -> dict: + pass + + @property + def supported_qresources_keys(self) -> list: + return [] + + def _get_jobs_list_cmd( + self, job_ids: list[str] | None = None, user: str | None = None + ) -> str: + pass + + def parse_jobs_list_output(self, exit_code, stdout, stderr) -> list[QJob]: + pass + + scheduler = MyScheduler() + + header = scheduler.generate_header({"option2": "value_option2"}) + assert header == """#SPECCMD --option2=value_option2""" + scheduler.parse_submit_output(0, "", "") From 5d39bf58b79d6c3fc7164d39802d6884cd45a83f Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 7 Jun 2023 14:51:03 +0200 Subject: [PATCH 15/34] More unit tests for data objects. --- src/qtoolkit/core/data_objects.py | 1 + tests/core/test_data_objects.py | 124 ++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) diff --git a/src/qtoolkit/core/data_objects.py b/src/qtoolkit/core/data_objects.py index 65129ef..9e1c042 100644 --- a/src/qtoolkit/core/data_objects.py +++ b/src/qtoolkit/core/data_objects.py @@ -151,6 +151,7 @@ def __post_init__(self): "plus processes_per_node or only processes" ) raise UnsupportedResourcesError(msg) + self.kwargs = self.kwargs or {} @classmethod def no_constraints(cls, processes, **kwargs): diff --git a/tests/core/test_data_objects.py b/tests/core/test_data_objects.py index a680e33..7466199 100644 --- a/tests/core/test_data_objects.py +++ b/tests/core/test_data_objects.py @@ -5,11 +5,13 @@ CancelResult, CancelStatus, ProcessPlacement, + QResources, QState, QSubState, SubmissionResult, SubmissionStatus, ) +from qtoolkit.core.exceptions import UnsupportedResourcesError try: import monty @@ -209,3 +211,125 @@ def test_process_placement_list(self): "SAME_NODE", "EVENLY_DISTRIBUTED", } + + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + pp1 = ProcessPlacement.SCATTERED + assert test_utils.is_msonable(pp1) + pp2 = ProcessPlacement("SAME_NODE") + assert test_utils.is_msonable(pp2) + + +class TestQResources: + def test_no_process_placement(self): + with pytest.raises( + UnsupportedResourcesError, + match=r"When process_placement is None either define only nodes " + r"plus processes_per_node or only processes", + ): + QResources(processes=8, nodes=2) + + with pytest.raises( + UnsupportedResourcesError, + match=r"When process_placement is None either define only nodes " + r"plus processes_per_node or only processes", + ): + QResources(processes=8, processes_per_node=2) + + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + qr1 = QResources( + queue_name="main", + job_name="myjob", + memory_per_thread=1024, + processes=16, + time_limit=86400, + ) + assert test_utils.is_msonable(qr1) + qr2 = QResources.evenly_distributed(nodes=4, processes_per_node=8) + assert test_utils.is_msonable(qr2) + + def test_no_constraints(self): + qr = QResources.no_constraints(processes=16, queue_name="main") + assert qr.queue_name == "main" + assert qr.process_placement == ProcessPlacement.NO_CONSTRAINTS + + with pytest.raises( + UnsupportedResourcesError, + match=r"nodes and processes_per_node are incompatible with no constraints jobs", + ): + QResources.no_constraints(processes=16, nodes=4) + + with pytest.raises( + UnsupportedResourcesError, + match=r"nodes and processes_per_node are incompatible with no constraints jobs", + ): + QResources.no_constraints(processes=16, processes_per_node=2) + + def test_evenly_distributed(self): + qr = QResources.evenly_distributed( + nodes=4, processes_per_node=2, queue_name="main" + ) + assert qr.queue_name == "main" + assert qr.process_placement == ProcessPlacement.EVENLY_DISTRIBUTED + + with pytest.raises( + UnsupportedResourcesError, + match=r"processes is incompatible with evenly distributed jobs", + ): + QResources.evenly_distributed(nodes=4, processes_per_node=2, processes=12) + + def test_scattered(self): + qr = QResources.scattered(processes=16, queue_name="main") + assert qr.queue_name == "main" + assert qr.process_placement == ProcessPlacement.SCATTERED + + with pytest.raises( + UnsupportedResourcesError, + match=r"nodes and processes_per_node are incompatible with scattered jobs", + ): + QResources.scattered(processes=16, nodes=4) + + with pytest.raises( + UnsupportedResourcesError, + match=r"nodes and processes_per_node are incompatible with scattered jobs", + ): + QResources.scattered(processes=16, processes_per_node=4) + + def test_same_node(self): + qr = QResources.same_node(processes=16, queue_name="main") + assert qr.queue_name == "main" + assert qr.process_placement == ProcessPlacement.SAME_NODE + + with pytest.raises( + UnsupportedResourcesError, + match=r"nodes and processes_per_node are incompatible with same node jobs", + ): + QResources.same_node(processes=16, nodes=4) + + with pytest.raises( + UnsupportedResourcesError, + match=r"nodes and processes_per_node are incompatible with same node jobs", + ): + QResources.same_node(processes=16, processes_per_node=4) + + def test_equality(self): + qr1 = QResources.evenly_distributed( + nodes=4, processes_per_node=4, job_name="myjob" + ) + qr2 = QResources( + nodes=4, + processes_per_node=4, + job_name="myjob", + process_placement=ProcessPlacement.EVENLY_DISTRIBUTED, + ) + qr3 = QResources(nodes=4, processes_per_node=4, job_name="myjob") + qr4 = QResources( + nodes=4, + processes_per_node=4, + job_name="myjob", + process_placement=ProcessPlacement.SAME_NODE, + ) + assert qr1 == qr2 + assert qr1 == qr3 + assert qr1 != qr4 From 498dffafe9e743866f0bca9364d4ef587322a672 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 7 Jun 2023 15:18:45 +0200 Subject: [PATCH 16/34] Unit tests for data objects. --- tests/core/test_data_objects.py | 150 ++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/tests/core/test_data_objects.py b/tests/core/test_data_objects.py index 7466199..8d628b8 100644 --- a/tests/core/test_data_objects.py +++ b/tests/core/test_data_objects.py @@ -5,6 +5,8 @@ CancelResult, CancelStatus, ProcessPlacement, + QJob, + QJobInfo, QResources, QState, QSubState, @@ -333,3 +335,151 @@ def test_equality(self): assert qr1 == qr2 assert qr1 == qr3 assert qr1 != qr4 + + def test_get_processes_distribution(self): + qr = QResources(nodes=4, processes_per_node=2) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [4, None, 2] + qr = QResources(processes=18) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [None, 18, None] + qr = QResources.scattered(processes=12) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [12, 12, 1] + qr = QResources.same_node(processes=12) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [1, 12, 12] + qr = QResources.evenly_distributed(nodes=4, processes_per_node=8) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [4, None, 8] + qr = QResources.no_constraints(processes=14) + proc_distr = qr.get_processes_distribution() + assert proc_distr == [None, 14, None] + + +class TestQJobInfo: + def test_equality(self): + qji1 = QJobInfo( + memory=2000, + memory_per_cpu=500, + nodes=2, + cpus=4, + threads_per_process=2, + time_limit=3600, + ) + qji2 = QJobInfo( + memory=2000, + memory_per_cpu=500, + nodes=2, + cpus=4, + threads_per_process=2, + time_limit=3600, + ) + qji3 = QJobInfo( + memory=4000, + memory_per_cpu=500, + nodes=2, + cpus=4, + threads_per_process=2, + time_limit=3600, + ) + assert qji1 == qji2 + assert qji1 != qji3 + + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + qji = QJobInfo( + memory=2000, + memory_per_cpu=500, + nodes=2, + cpus=4, + threads_per_process=2, + time_limit=3600, + ) + assert test_utils.is_msonable(qji) + + +class TestQJob: + def test_equality(self): + qji1 = QJobInfo( + memory=2000, + memory_per_cpu=500, + nodes=2, + cpus=4, + threads_per_process=2, + time_limit=3600, + ) + qji2 = QJobInfo( + memory=2000, + memory_per_cpu=500, + nodes=2, + cpus=4, + threads_per_process=2, + time_limit=3600, + ) + qji3 = QJobInfo( + memory=4000, + memory_per_cpu=500, + nodes=2, + cpus=4, + threads_per_process=2, + time_limit=3600, + ) + qjob1 = QJob( + name="job1", + job_id="id1", + exit_status=0, + state=QState.DONE, + sub_state=SomeSubState.STATE_A, + info=qji1, + account="myacc", + runtime=2541, + queue_name="mymain", + ) + qjob2 = QJob( + name="job1", + job_id="id1", + exit_status=0, + state=QState.DONE, + sub_state=SomeSubState.STATE_A, + info=qji2, + account="myacc", + runtime=2541, + queue_name="mymain", + ) + qjob3 = QJob( + name="job1", + job_id="id1", + exit_status=0, + state=QState.DONE, + sub_state=SomeSubState.STATE_A, + info=qji3, + account="myacc", + runtime=2541, + queue_name="mymain", + ) + assert qjob1 == qjob2 + assert qjob1 != qjob3 + + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + qji = QJobInfo( + memory=2000, + memory_per_cpu=500, + nodes=2, + cpus=4, + threads_per_process=2, + time_limit=3600, + ) + qjob = QJob( + name="job1", + job_id="id1", + exit_status=0, + state=QState.DONE, + sub_state=SomeSubState.STATE_A, + info=qji, + account="myacc", + runtime=2541, + queue_name="mymain", + ) + assert test_utils.is_msonable(qjob) From 624bb08b54272e4f5e42e78d1e16bc9bab14c888 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 7 Jun 2023 15:21:17 +0200 Subject: [PATCH 17/34] Removed unused core.jobs module --- src/qtoolkit/core/jobs.py | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 src/qtoolkit/core/jobs.py diff --git a/src/qtoolkit/core/jobs.py b/src/qtoolkit/core/jobs.py deleted file mode 100644 index e559c61..0000000 --- a/src/qtoolkit/core/jobs.py +++ /dev/null @@ -1,20 +0,0 @@ -# from __future__ import annotations -# -# from dataclasses import dataclass -# -# from qtoolkit.core.base import QBase -# from qtoolkit.core.data_objects import QState, QSubState, QResources, QJobInfo -# -# -# TODO: this has been moved to data_objects for now. See if it should be here -# for some reason ? -# @dataclass -# class QJob(QBase): -# name: str -# qid: str | None -# exit_status: int | None -# state: QState | None # Standard -# sub_state: QSubState | None -# queue: str | None -# resources: QResources | None -# job_info: QJobInfo | None From b55f16748164e9ac7adb8bef190d353a8c9fc4da Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 7 Jun 2023 15:28:55 +0200 Subject: [PATCH 18/34] Removed unused queue module (directory with __init__.py) --- src/qtoolkit/queue/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/qtoolkit/queue/__init__.py diff --git a/src/qtoolkit/queue/__init__.py b/src/qtoolkit/queue/__init__.py deleted file mode 100644 index e69de29..0000000 From cdff324c0d9ebddfc7ab048aa701106b46615b81 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 7 Jun 2023 15:46:43 +0200 Subject: [PATCH 19/34] Unit tests for base scheduler io. --- src/qtoolkit/io/base.py | 2 +- tests/io/test_base.py | 26 +++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/qtoolkit/io/base.py b/src/qtoolkit/io/base.py index 5848478..b437215 100644 --- a/src/qtoolkit/io/base.py +++ b/src/qtoolkit/io/base.py @@ -243,7 +243,7 @@ def generate_ids_list(self, jobs: list[QJob | int | str] | None) -> list[str]: ids_list = [] for j in jobs: if isinstance(j, QJob): - ids_list.append(j.job_id) + ids_list.append(str(j.job_id)) else: ids_list.append(str(j)) diff --git a/tests/io/test_base.py b/tests/io/test_base.py index b2f7365..e549ef8 100644 --- a/tests/io/test_base.py +++ b/tests/io/test_base.py @@ -104,4 +104,28 @@ def parse_jobs_list_output(self, exit_code, stdout, stderr) -> list[QJob]: header = scheduler.generate_header({"option2": "value_option2"}) assert header == """#SPECCMD --option2=value_option2""" - scheduler.parse_submit_output(0, "", "") + + ids_list = scheduler.generate_ids_list( + [QJob(job_id=4), QJob(job_id="job_id_abc1"), 215, "job12345"] + ) + assert ids_list == ["4", "job_id_abc1", "215", "job12345"] + + submit_cmd = scheduler.get_submit_cmd() + assert submit_cmd == "mysubmit submit.script" + submit_cmd = scheduler.get_submit_cmd(script_file="sub.sh") + assert submit_cmd == "mysubmit sub.sh" + + cancel_cmd = scheduler.get_cancel_cmd(QJob(job_id=5)) + assert cancel_cmd == "mycancel 5" + cancel_cmd = scheduler.get_cancel_cmd(QJob(job_id="abc1")) + assert cancel_cmd == "mycancel abc1" + cancel_cmd = scheduler.get_cancel_cmd("jobid2") + assert cancel_cmd == "mycancel jobid2" + cancel_cmd = scheduler.get_cancel_cmd(632) + assert cancel_cmd == "mycancel 632" + + with pytest.raises( + ValueError, + match=r"The id of the job to be cancelled should be defined. Received: None", + ): + scheduler.get_cancel_cmd(job=None) From bc84aa772ea763615ed161aa1cdcbb0f2fc3b67e Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Wed, 7 Jun 2023 16:02:17 +0200 Subject: [PATCH 20/34] Started tests for shell io. --- tests/conftest.py | 6 +++++- tests/io/test_shell.py | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 tests/io/test_shell.py diff --git a/tests/conftest.py b/tests/conftest.py index 372d7a6..a267056 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,5 @@ +from dataclasses import is_dataclass +from enum import Enum from pathlib import Path import pytest @@ -98,7 +100,9 @@ def is_msonable(cls, obj, obj_cls=None): obj_from_json = obj_cls.from_dict(obj_from_json) if not isinstance(obj_from_json, obj.__class__): return False - return obj_from_json == obj + if is_dataclass(obj) or isinstance(obj, Enum): + return obj_from_json == obj + return obj_from_json.as_dict() == obj.as_dict() @classmethod def inkwargs_outref(cls, in_out_ref, inkey, outkey): diff --git a/tests/io/test_shell.py b/tests/io/test_shell.py new file mode 100644 index 0000000..aedfd18 --- /dev/null +++ b/tests/io/test_shell.py @@ -0,0 +1,24 @@ +import pytest + +try: + import monty +except ModuleNotFoundError: + monty = None + + +from qtoolkit.io.shell import ShellIO + + +class TestShellIO: + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils): + shell_io = ShellIO() + assert test_utils.is_msonable(shell_io) + + def test_get_submit_cmd(self): + shell_io = ShellIO(blocking=True) + submit_cmd = shell_io.get_submit_cmd(script_file="myscript.sh") + assert submit_cmd == "bash myscript.sh > stdout 2> stderr" + shell_io = ShellIO(blocking=False) + submit_cmd = shell_io.get_submit_cmd(script_file="myscript.sh") + assert submit_cmd == "nohup bash myscript.sh > stdout 2> stderr & echo $!" From c2f19ed5f04d7d6445276a88e6bd8830e2af7313 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Thu, 8 Jun 2023 09:37:17 +0200 Subject: [PATCH 21/34] Set sphinx language to english (en) in conf.py --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 8ace3c5..3c58151 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -78,7 +78,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. From a21a80673324bcb8ff1e39cf7e908953210de250 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Thu, 8 Jun 2023 12:50:30 +0200 Subject: [PATCH 22/34] More unit tests for shell IO. --- src/qtoolkit/io/shell.py | 5 +- tests/io/test_shell.py | 146 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 147 insertions(+), 4 deletions(-) diff --git a/src/qtoolkit/io/shell.py b/src/qtoolkit/io/shell.py index d3fc6cd..0a7ab7b 100644 --- a/src/qtoolkit/io/shell.py +++ b/src/qtoolkit/io/shell.py @@ -37,7 +37,7 @@ class ShellState(QSubState): STOPPED = "T" STOPPED_DEBUGGER = "t" PAGING = "W" - DEAD = "D" + DEAD = "X" DEFUNCT = "Z" @property @@ -172,6 +172,9 @@ def _get_jobs_list_cmd( return " ".join(command) def parse_jobs_list_output(self, exit_code, stdout, stderr) -> list[QJob]: + print("exit_code", (type(exit_code), repr(exit_code))) + print("stdout", (type(stdout), repr(stdout))) + print("stderr", (type(stderr), repr(stderr))) if isinstance(stdout, bytes): stdout = stdout.decode() diff --git a/tests/io/test_shell.py b/tests/io/test_shell.py index aedfd18..30017e0 100644 --- a/tests/io/test_shell.py +++ b/tests/io/test_shell.py @@ -6,13 +6,54 @@ monty = None -from qtoolkit.io.shell import ShellIO +from qtoolkit.core.data_objects import ( + CancelResult, + CancelStatus, + QJob, + QState, + SubmissionResult, + SubmissionStatus, +) +from qtoolkit.io.shell import ShellIO, ShellState -class TestShellIO: +@pytest.fixture(scope="module") +def shell_io(): + return ShellIO() + + +class TestShellState: @pytest.mark.skipif(monty is None, reason="monty is not installed") def test_msonable(self, test_utils): - shell_io = ShellIO() + shell_state = ShellState.DEFUNCT + assert test_utils.is_msonable(shell_state) + + def test_states_list(self): + all_states = [state.value for state in ShellState] + assert set(all_states) == {"D", "R", "S", "T", "t", "W", "X", "Z"} + + def test_qstate(self): + shell_state = ShellState.UNINTERRUPTIBLE_SLEEP + assert shell_state.qstate == QState.RUNNING + shell_state = ShellState.RUNNING + assert shell_state.qstate == QState.RUNNING + shell_state = ShellState.INTERRUPTIBLE_SLEEP + assert shell_state.qstate == QState.RUNNING + shell_state = ShellState.STOPPED + assert shell_state.qstate == QState.SUSPENDED + shell_state = ShellState.STOPPED_DEBUGGER + assert shell_state.qstate == QState.SUSPENDED + shell_state = ShellState.PAGING + assert shell_state.qstate == QState.RUNNING + shell_state = ShellState.DEAD + assert shell_state.qstate == QState.DONE + shell_state = ShellState.DEFUNCT + assert shell_state.qstate == QState.DONE + + +class TestShellIO: + @pytest.mark.skipif(monty is None, reason="monty is not installed") + def test_msonable(self, test_utils, shell_io): assert test_utils.is_msonable(shell_io) def test_get_submit_cmd(self): @@ -22,3 +63,102 @@ def test_get_submit_cmd(self): shell_io = ShellIO(blocking=False) submit_cmd = shell_io.get_submit_cmd(script_file="myscript.sh") assert submit_cmd == "nohup bash myscript.sh > stdout 2> stderr & echo $!" + + def test_parse_submit_output(self, shell_io): + sr = shell_io.parse_submit_output(exit_code=0, stdout="13647\n", stderr="") + assert sr == SubmissionResult( + job_id="13647", + step_id=None, + exit_code=0, + stdout="13647\n", + stderr="", + status=SubmissionStatus.SUCCESSFUL, + ) + sr = shell_io.parse_submit_output(exit_code=0, stdout=b"13647\n", stderr=b"") + assert sr == SubmissionResult( + job_id="13647", + step_id=None, + exit_code=0, + stdout="13647\n", + stderr="", + status=SubmissionStatus.SUCCESSFUL, + ) + sr = shell_io.parse_submit_output(exit_code=104, stdout="tata", stderr=b"titi") + assert sr == SubmissionResult( + job_id=None, + step_id=None, + exit_code=104, + stdout="tata", + stderr="titi", + status=SubmissionStatus.FAILED, + ) + sr = shell_io.parse_submit_output(exit_code=0, stdout=b"\n", stderr="") + assert sr == SubmissionResult( + job_id=None, + step_id=None, + exit_code=0, + stdout="\n", + stderr="", + status=SubmissionStatus.JOB_ID_UNKNOWN, + ) + + def test_parse_cancel_output(self, shell_io): + cr = shell_io.parse_cancel_output(exit_code=0, stdout="", stderr="") + assert cr == CancelResult( + job_id=None, + step_id=None, + exit_code=0, + stdout="", + stderr="", + status=CancelStatus.SUCCESSFUL, + ) + cr = shell_io.parse_cancel_output( + exit_code=1, + stdout=b"", + stderr=b"/bin/sh: line 1: kill: (14020) - No such process\n", + ) + assert cr == CancelResult( + job_id=None, + step_id=None, + exit_code=1, + stdout="", + stderr="/bin/sh: line 1: kill: (14020) - No such process\n", + status=CancelStatus.FAILED, + ) + + def test_get_job_cmd(self, shell_io): + get_job_cmd = shell_io.get_job_cmd(123) + assert get_job_cmd == "ps -o pid,user,etimes,state,comm -p 123" + get_job_cmd = shell_io.get_job_cmd(456) + assert get_job_cmd == "ps -o pid,user,etimes,state,comm -p 456" + get_job_cmd = shell_io.get_job_cmd(QJob(job_id="789")) + assert get_job_cmd == "ps -o pid,user,etimes,state,comm -p 789" + + def test_get_jobs_list_cmd(self, shell_io): + get_jobs_list_cmd = shell_io.get_jobs_list_cmd( + jobs=[QJob(job_id=125), 126, "127"], user=None + ) + assert get_jobs_list_cmd == "ps -o pid,user,etimes,state,comm -p 125,126,127" + + def test_parse_jobs_list_output(self, shell_io): + joblist = shell_io.parse_jobs_list_output( + exit_code=0, + stdout=" PID USER ELAPSED S COMMAND\n 18092 davidwa+ 465 S bash\n 18112 davidwa+ 461 S bash\n", + stderr="", + ) + assert joblist == [ + QJob( + job_id="18092", + runtime=465, + name="bash", + state=QState.RUNNING, + sub_state=ShellState.INTERRUPTIBLE_SLEEP, + ), + QJob( + job_id="18112", + runtime=461, + name="bash", + state=QState.RUNNING, + sub_state=ShellState.INTERRUPTIBLE_SLEEP, + ), + ] From d1646b532ba9cbec3db1e9e35bbd6dac33cbd163 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Thu, 8 Jun 2023 15:38:23 +0200 Subject: [PATCH 23/34] More tests for shell io + few docstrings additions. --- src/qtoolkit/io/base.py | 2 +- src/qtoolkit/io/shell.py | 27 +++++++++++++++++++++++---- tests/io/test_shell.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/qtoolkit/io/base.py b/src/qtoolkit/io/base.py index b437215..f442f35 100644 --- a/src/qtoolkit/io/base.py +++ b/src/qtoolkit/io/base.py @@ -311,7 +311,7 @@ def check_convert_qresources(self, resources: QResources) -> dict: unsupported_options = not_none.difference(self.supported_qresources_keys) if unsupported_options: - msg = f"Keys not supported: {', '.join(unsupported_options)}" + msg = f"Keys not supported: {', '.join(sorted(unsupported_options))}" raise UnsupportedResourcesError(msg) return self._convert_qresources(resources) diff --git a/src/qtoolkit/io/shell.py b/src/qtoolkit/io/shell.py index 0a7ab7b..f605482 100644 --- a/src/qtoolkit/io/shell.py +++ b/src/qtoolkit/io/shell.py @@ -65,6 +65,17 @@ class ShellIO(BaseSchedulerIO): CANCEL_CMD: str | None = "kill -9" def __init__(self, blocking=False, stdout_path="stdout", stderr_path="stderr"): + """Construct the ShellIO object. + + Parameters + ---------- + blocking: bool + Whether the execution should be blocking. + stdout_path: str or Path + Path to the standard output file. + stderr_path: str or Path + Path to the standard error file. + """ self.blocking = blocking self.stdout_path = stdout_path self.stderr_path = stderr_path @@ -75,7 +86,8 @@ def get_submit_cmd(self, script_file: str | Path | None = "submit.script") -> st Parameters ---------- - script_file: (str) path of the script file to use. + script_file: str or Path + Path of the script file to use. """ script_file = script_file or "" @@ -172,10 +184,17 @@ def _get_jobs_list_cmd( return " ".join(command) def parse_jobs_list_output(self, exit_code, stdout, stderr) -> list[QJob]: - print("exit_code", (type(exit_code), repr(exit_code))) - print("stdout", (type(stdout), repr(stdout))) - print("stderr", (type(stderr), repr(stderr))) + """Parse the output of the ps command to list jobs. + Parameters + ---------- + exit_code : int + Exit code of the ps command. + stdout : str + Standard output of the ps command. + stderr : str + Standard error of the ps command. + """ if isinstance(stdout, bytes): stdout = stdout.decode() if isinstance(stderr, bytes): diff --git a/tests/io/test_shell.py b/tests/io/test_shell.py index 30017e0..32de498 100644 --- a/tests/io/test_shell.py +++ b/tests/io/test_shell.py @@ -10,10 +10,16 @@ CancelResult, CancelStatus, QJob, + QResources, QState, SubmissionResult, SubmissionStatus, ) +from qtoolkit.core.exceptions import ( + CommandFailedError, + OutputParsingError, + UnsupportedResourcesError, +) from qtoolkit.io.shell import ShellIO, ShellState @@ -162,3 +168,27 @@ def test_parse_jobs_list_output(self, shell_io): sub_state=ShellState.INTERRUPTIBLE_SLEEP, ), ] + with pytest.raises( + CommandFailedError, match=r"command ps failed: string in stderr" + ): + shell_io.parse_jobs_list_output( + exit_code=1, + stdout=b"", + stderr=b"string in stderr", + ) + with pytest.raises( + OutputParsingError, match=r"Unknown job state K for job id 18112" + ): + shell_io.parse_jobs_list_output( + exit_code=0, + stdout=b" PID USER ELAPSED S COMMAND\n 18092 davidwa+ 465 S bash\n 18112 davidwa+ 461 K bash\n", + stderr=b"", + ) + + def test_check_convert_qresources(self, shell_io): + qr = QResources(processes=1) + with pytest.raises( + UnsupportedResourcesError, + match=r"Keys not supported: kwargs, process_placement, processes", + ): + shell_io.check_convert_qresources(qr) From 469f2c30205ed56735bf59dc38addaf9216658dc Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Thu, 8 Jun 2023 15:43:21 +0200 Subject: [PATCH 24/34] Renamed QBase to QTKObject and QEnum to QTKEnum. --- src/qtoolkit/core/base.py | 4 ++-- src/qtoolkit/core/data_objects.py | 24 ++++++++++++------------ src/qtoolkit/host/base.py | 6 +++--- src/qtoolkit/io/base.py | 4 ++-- src/qtoolkit/io/slurm.py | 2 +- src/qtoolkit/manager.py | 4 ++-- tests/core/test_base.py | 14 +++++++------- 7 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/qtoolkit/core/base.py b/src/qtoolkit/core/base.py index 35ec24e..1e63040 100644 --- a/src/qtoolkit/core/base.py +++ b/src/qtoolkit/core/base.py @@ -11,9 +11,9 @@ enum_superclses = (Enum,) # type: ignore -class QBase(supercls): # type: ignore +class QTKObject(supercls): # type: ignore pass -class QEnum(*enum_superclses): # type: ignore +class QTKEnum(*enum_superclses): # type: ignore pass diff --git a/src/qtoolkit/core/data_objects.py b/src/qtoolkit/core/data_objects.py index 9e1c042..ac4bc40 100644 --- a/src/qtoolkit/core/data_objects.py +++ b/src/qtoolkit/core/data_objects.py @@ -5,18 +5,18 @@ from datetime import timedelta from pathlib import Path -from qtoolkit.core.base import QBase, QEnum +from qtoolkit.core.base import QTKEnum, QTKObject from qtoolkit.core.exceptions import UnsupportedResourcesError -class SubmissionStatus(QEnum): +class SubmissionStatus(QTKEnum): SUCCESSFUL = "SUCCESSFUL" FAILED = "FAILED" JOB_ID_UNKNOWN = "JOB_ID_UNKNOWN" @dataclass -class SubmissionResult(QBase): +class SubmissionResult(QTKObject): job_id: int | str | None = None step_id: int | None = None exit_code: int | None = None @@ -25,14 +25,14 @@ class SubmissionResult(QBase): status: SubmissionStatus | None = None -class CancelStatus(QEnum): +class CancelStatus(QTKEnum): SUCCESSFUL = "SUCCESSFUL" FAILED = "FAILED" JOB_ID_UNKNOWN = "JOB_ID_UNKNOWN" @dataclass -class CancelResult(QBase): +class CancelResult(QTKObject): job_id: int | str | None = None step_id: int | None = None exit_code: int | None = None @@ -41,7 +41,7 @@ class CancelResult(QBase): status: CancelStatus | None = None -class QState(QEnum): +class QState(QTKEnum): """Enumeration of possible ("standardized") job states. These "standardized" states are based on the drmaa specification. @@ -64,7 +64,7 @@ class QState(QEnum): FAILED = "FAILED" -class QSubState(QEnum): +class QSubState(QTKEnum): """QSubState class defined without any enum values so it can be subclassed. These sub-states should be the actual job states in a given queuing system @@ -93,7 +93,7 @@ def qstate(self) -> QState: raise NotImplementedError -class ProcessPlacement(QEnum): +class ProcessPlacement(QTKEnum): NO_CONSTRAINTS = "NO_CONSTRAINTS" SCATTERED = "SCATTERED" SAME_NODE = "SAME_NODE" @@ -101,7 +101,7 @@ class ProcessPlacement(QEnum): @dataclass -class QResources(QBase): +class QResources(QTKObject): """Data defining resources for a given job (submitted or to be submitted). Attributes @@ -142,7 +142,7 @@ class QResources(QBase): def __post_init__(self): if self.process_placement is None: if self.processes and not self.processes_per_node and not self.nodes: - self.process_placement = ProcessPlacement.NO_CONSTRAINTS # type: ignore # due to QEnum + self.process_placement = ProcessPlacement.NO_CONSTRAINTS # type: ignore # due to QTKEnum elif self.nodes and self.processes_per_node and not self.processes: self.process_placement = ProcessPlacement.EVENLY_DISTRIBUTED else: @@ -239,7 +239,7 @@ def get_processes_distribution(self) -> list: @dataclass -class QJobInfo(QBase): +class QJobInfo(QTKObject): memory: int | None = None # in Kb memory_per_cpu: int | None = None # in Kb nodes: int | None = None @@ -249,7 +249,7 @@ class QJobInfo(QBase): @dataclass -class QJob(QBase): +class QJob(QTKObject): name: str | None = None job_id: str | None = None exit_status: int | None = None diff --git a/src/qtoolkit/host/base.py b/src/qtoolkit/host/base.py index 12218d6..fb29816 100644 --- a/src/qtoolkit/host/base.py +++ b/src/qtoolkit/host/base.py @@ -4,15 +4,15 @@ from dataclasses import dataclass from pathlib import Path -from qtoolkit.core.base import QBase +from qtoolkit.core.base import QTKObject @dataclass -class HostConfig(QBase): +class HostConfig(QTKObject): root_dir: str | Path -class BaseHost(QBase): +class BaseHost(QTKObject): """Base Host class.""" # def __init__(self, config, user): diff --git a/src/qtoolkit/io/base.py b/src/qtoolkit/io/base.py index f442f35..c9fd0d4 100644 --- a/src/qtoolkit/io/base.py +++ b/src/qtoolkit/io/base.py @@ -6,7 +6,7 @@ from pathlib import Path from string import Template -from qtoolkit.core.base import QBase +from qtoolkit.core.base import QTKObject from qtoolkit.core.data_objects import CancelResult, QJob, QResources, SubmissionResult from qtoolkit.core.exceptions import UnsupportedResourcesError @@ -37,7 +37,7 @@ def get_identifiers(self) -> list: return ids -class BaseSchedulerIO(QBase, abc.ABC): +class BaseSchedulerIO(QTKObject, abc.ABC): """Base class for job queues. Attributes diff --git a/src/qtoolkit/io/slurm.py b/src/qtoolkit/io/slurm.py index 41b1e99..8d81ad7 100644 --- a/src/qtoolkit/io/slurm.py +++ b/src/qtoolkit/io/slurm.py @@ -103,7 +103,7 @@ class SlurmState(QSubState): @property def qstate(self) -> QState: - # the type:ignore is required due to the dynamic class creation of QEnum + # the type:ignore is required due to the dynamic class creation of QTKEnum return _STATUS_MAPPING[self] # type: ignore diff --git a/src/qtoolkit/manager.py b/src/qtoolkit/manager.py index 80f7a07..ed50b77 100644 --- a/src/qtoolkit/manager.py +++ b/src/qtoolkit/manager.py @@ -2,14 +2,14 @@ from pathlib import Path -from qtoolkit.core.base import QBase +from qtoolkit.core.base import QTKObject from qtoolkit.core.data_objects import CancelResult, QJob, QResources, SubmissionResult from qtoolkit.host.base import BaseHost from qtoolkit.host.local import LocalHost from qtoolkit.io.base import BaseSchedulerIO -class QueueManager(QBase): +class QueueManager(QTKObject): """Base class for job queues. Attributes diff --git a/tests/core/test_base.py b/tests/core/test_base.py index 1ed4e8b..722d8a1 100644 --- a/tests/core/test_base.py +++ b/tests/core/test_base.py @@ -16,7 +16,7 @@ def qtk_core_base_mocked_monty_not_found(mocker): # Note: # Here we use importlib to dynamically import the qtoolkit.core.base module. - # We want to test the QBase and QEnum super classes with monty present or not. + # We want to test the QTKObject and QTKEnum super classes with monty present or not. # This is done by mocking the import. We then need to use importlib to reload # the qtoolkit.core.base module when we want to change the behaviour of the # the monty.json import inside the qtoolkit.core.base module (i.e. mocking the @@ -47,7 +47,7 @@ def test_msonable(self, test_utils): import qtoolkit.core.base as qbase @dataclass - class QClass(qbase.QBase): + class QClass(qbase.QTKObject): name: str = "name" qc = QClass() @@ -55,7 +55,7 @@ class QClass(qbase.QBase): def test_not_msonable(self, test_utils, qtk_core_base_mocked_monty_not_found): @dataclass - class QClass(qtk_core_base_mocked_monty_not_found.QBase): + class QClass(qtk_core_base_mocked_monty_not_found.QTKObject): name: str = "name" qc = QClass() @@ -67,7 +67,7 @@ class TestQEnum: def test_msonable(self, test_utils): import qtoolkit.core.base as qbase - class SomeEnum(qbase.QEnum): + class SomeEnum(qbase.QTKEnum): VAL1 = "VAL1" VAL2 = "VAL2" @@ -79,7 +79,7 @@ class SomeEnum(qbase.QEnum): assert test_utils.is_msonable(se, obj_cls=SomeEnum) assert isinstance(se, enum.Enum) - class SomeEnum(qbase.QEnum): + class SomeEnum(qbase.QTKEnum): VAL1 = 3 VAL2 = 4 @@ -92,7 +92,7 @@ class SomeEnum(qbase.QEnum): assert isinstance(se, enum.Enum) def test_not_msonable(self, test_utils, qtk_core_base_mocked_monty_not_found): - class SomeEnum(qtk_core_base_mocked_monty_not_found.QEnum): + class SomeEnum(qtk_core_base_mocked_monty_not_found.QTKEnum): VAL1 = "VAL1" VAL2 = "VAL2" @@ -104,7 +104,7 @@ class SomeEnum(qtk_core_base_mocked_monty_not_found.QEnum): assert not test_utils.is_msonable(se) assert isinstance(se, enum.Enum) - class SomeEnum(qtk_core_base_mocked_monty_not_found.QEnum): + class SomeEnum(qtk_core_base_mocked_monty_not_found.QTKEnum): VAL1 = 3 VAL2 = 4 From c2791e6e1121e4cc7906e13cbf3a8970775560f3 Mon Sep 17 00:00:00 2001 From: Guido Petretto Date: Wed, 28 Jun 2023 13:55:04 +0200 Subject: [PATCH 25/34] update shell io --- src/qtoolkit/io/__init__.py | 2 +- src/qtoolkit/io/shell.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/qtoolkit/io/__init__.py b/src/qtoolkit/io/__init__.py index a4d31ea..cc9080d 100644 --- a/src/qtoolkit/io/__init__.py +++ b/src/qtoolkit/io/__init__.py @@ -3,4 +3,4 @@ from qtoolkit.io.shell import ShellIO, ShellState from qtoolkit.io.slurm import SlurmIO, SlurmState -scheduler_mapping = {"slurm": SlurmIO, "pbs": PBSIO} +scheduler_mapping = {"slurm": SlurmIO, "pbs": PBSIO, "shell": ShellIO} diff --git a/src/qtoolkit/io/shell.py b/src/qtoolkit/io/shell.py index d3fc6cd..727bc39 100644 --- a/src/qtoolkit/io/shell.py +++ b/src/qtoolkit/io/shell.py @@ -59,6 +59,10 @@ def qstate(self) -> QState: class ShellIO(BaseSchedulerIO): header_template: str = """ +exec > $${qout_path} +exec 2> $${qerr_path} + +echo $${job_name} $${qverbatim} """ @@ -178,8 +182,10 @@ def parse_jobs_list_output(self, exit_code, stdout, stderr) -> list[QJob]: if isinstance(stderr, bytes): stderr = stderr.decode() - if exit_code != 0: - msg = f"command ps failed: {stderr}" + # if asking only for pid that are not running the exit code is != 0, + # so check also on stderr for failing + if exit_code != 0 and stderr.strip(): + msg = f"command ps failed: stdout: {stdout}. stderr: {stderr}" raise CommandFailedError(msg) jobs_list = [] From f0a9e77e952fa6c15e228bad466e28157c942901 Mon Sep 17 00:00:00 2001 From: Guido Petretto Date: Wed, 28 Jun 2023 17:24:41 +0200 Subject: [PATCH 26/34] fix remote host --- src/qtoolkit/host/remote.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/qtoolkit/host/remote.py b/src/qtoolkit/host/remote.py index 362e043..9fc6080 100644 --- a/src/qtoolkit/host/remote.py +++ b/src/qtoolkit/host/remote.py @@ -161,6 +161,10 @@ def execute(self, command: str | list[str], workdir: str | Path | None = None): exit_code : int Exit code of the command. """ + + if isinstance(command, (list, tuple)): + command = " ".join(command) + # TODO: check here if we use the context manager. What happens if we provide the # connection from outside (not through a config) and we want to keep it alive ? From f6bf66e798c2480c4893ba03bed4f9310caef64a Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 14 Jul 2023 13:39:36 +0200 Subject: [PATCH 27/34] Fix name for `ntasks` in slurm template --- src/qtoolkit/io/slurm.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/qtoolkit/io/slurm.py b/src/qtoolkit/io/slurm.py index 8d81ad7..8f35f4c 100644 --- a/src/qtoolkit/io/slurm.py +++ b/src/qtoolkit/io/slurm.py @@ -128,7 +128,7 @@ class SlurmIO(BaseSchedulerIO): #SBATCH --partition=$${partition} #SBATCH --job-name=$${job_name} #SBATCH --nodes=$${nodes} -#SBATCH --ntasks=$${ntasks} +#SBATCH --ntasks=$${number_of_tasks} #SBATCH --ntasks-per-node=$${ntasks_per_node} #SBATCH --cpus-per-task=$${cpus_per_task} #SBATCH --mem=$${mem} @@ -342,7 +342,6 @@ def _parse_scontrol_cmd_output(self, stdout): def _get_jobs_list_cmd( self, job_ids: list[str] | None = None, user: str | None = None ) -> str: - if user and job_ids: raise ValueError("Cannot query by user and job(s) in SLURM") @@ -394,7 +393,6 @@ def parse_jobs_list_output(self, exit_code, stdout, stderr) -> list[QJob]: jobs_list = [] for data in jobdata_raw: if len(data) != num_fields: - msg = f"Wrong number of fields. Found {len(jobdata_raw)}, expected {num_fields}" # TODO should this raise or just continue? and should there be # a logging of the errors? From a3c7c00e324e2c63ffd1845e78401658f5285a0d Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 14 Jul 2023 13:49:29 +0200 Subject: [PATCH 28/34] Use `ntasks` as the internal variable name --- src/qtoolkit/io/slurm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/qtoolkit/io/slurm.py b/src/qtoolkit/io/slurm.py index 8f35f4c..7e8fc2b 100644 --- a/src/qtoolkit/io/slurm.py +++ b/src/qtoolkit/io/slurm.py @@ -128,7 +128,7 @@ class SlurmIO(BaseSchedulerIO): #SBATCH --partition=$${partition} #SBATCH --job-name=$${job_name} #SBATCH --nodes=$${nodes} -#SBATCH --ntasks=$${number_of_tasks} +#SBATCH --ntasks=$${ntasks} #SBATCH --ntasks-per-node=$${ntasks_per_node} #SBATCH --cpus-per-task=$${cpus_per_task} #SBATCH --mem=$${mem} @@ -555,11 +555,11 @@ def _convert_qresources(self, resources: QResources) -> dict: nodes, processes, processes_per_node = resources.get_processes_distribution() if processes: - header_dict["number_of_tasks"] = processes + header_dict["ntasks"] = processes if processes_per_node: header_dict["ntasks_per_node"] = processes_per_node if nodes: - header_dict["number_of_nodes"] = nodes + header_dict["nodes"] = nodes if resources.threads_per_process: header_dict["cpus_per_task"] = resources.threads_per_process From c03f3d3b1f9eb27f116a324bb13a98c15a93806c Mon Sep 17 00:00:00 2001 From: Guido Petretto Date: Mon, 28 Aug 2023 16:00:19 +0200 Subject: [PATCH 29/34] add --exclusive to Slurm template --- .gitignore | 3 +++ src/qtoolkit/io/slurm.py | 1 + 2 files changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 373876b..3d67b95 100644 --- a/.gitignore +++ b/.gitignore @@ -154,3 +154,6 @@ cython_debug/ # PyCharm .idea/ + +# macOS +.DS_store diff --git a/src/qtoolkit/io/slurm.py b/src/qtoolkit/io/slurm.py index 7e8fc2b..a21e4d1 100644 --- a/src/qtoolkit/io/slurm.py +++ b/src/qtoolkit/io/slurm.py @@ -150,6 +150,7 @@ class SlurmIO(BaseSchedulerIO): #SBATCH --qos=$${qos} #SBATCH --priority=$${priority} #SBATCH --array=$${array} +#SBATCH --exclusive=$${exclusive} $${qverbatim}""" SUBMIT_CMD: str | None = "sbatch" From 48842348e887264079940edb4eec9d8f7bd14acd Mon Sep 17 00:00:00 2001 From: Guido Petretto Date: Mon, 28 Aug 2023 16:01:47 +0200 Subject: [PATCH 30/34] use etime instead of etimes for compatibility in ShellIO --- src/qtoolkit/io/shell.py | 37 +++++++++++++++++++++++++++++++++++-- tests/io/test_shell.py | 14 +++++++++++++- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/qtoolkit/io/shell.py b/src/qtoolkit/io/shell.py index e55f08e..75bee06 100644 --- a/src/qtoolkit/io/shell.py +++ b/src/qtoolkit/io/shell.py @@ -174,9 +174,10 @@ def _get_jobs_list_cmd( ) raise ValueError(msg) + # use etime instead of etimes for compatibility command = [ "ps", - "-o pid,user,etimes,state,comm", + "-o pid,user,etime,state,comm", ] if user: @@ -220,7 +221,7 @@ def parse_jobs_list_output(self, exit_code, stdout, stderr) -> list[QJob]: qjob = QJob() qjob.job_id = data[0] qjob.username = data[1] - qjob.runtime = int(data[2]) + qjob.runtime = self._convert_str_to_time(data[2]) qjob.name = data[4] try: @@ -251,3 +252,35 @@ def supported_qresources_keys(self) -> list: does not pass an unsupported value, expecting to have an effect. """ return [] + + @staticmethod + def _convert_str_to_time(time_str: str | None) -> int | None: + """ + Convert a string in the format used in etime [[DD-]hh:]mm:ss to a + number of seconds. + """ + + if not time_str: + return None + + time_split = time_str.split(":") + + days = hours = 0 + + try: + if "-" in time_split[0]: + split_day = time_split[0].split("-") + days = int(split_day[0]) + time_split = [split_day[1]] + time_split[1:] + + if len(time_split) == 3: + hours, minutes, seconds = (int(v) for v in time_split) + elif len(time_split) == 2: + minutes, seconds = (int(v) for v in time_split) + else: + raise OutputParsingError() + + except ValueError: + raise OutputParsingError() + + return days * 86400 + hours * 3600 + minutes * 60 + seconds diff --git a/tests/io/test_shell.py b/tests/io/test_shell.py index 32de498..92195b6 100644 --- a/tests/io/test_shell.py +++ b/tests/io/test_shell.py @@ -169,7 +169,7 @@ def test_parse_jobs_list_output(self, shell_io): ), ] with pytest.raises( - CommandFailedError, match=r"command ps failed: string in stderr" + CommandFailedError, match=r"command ps failed: .* string in stderr" ): shell_io.parse_jobs_list_output( exit_code=1, @@ -192,3 +192,15 @@ def test_check_convert_qresources(self, shell_io): match=r"Keys not supported: kwargs, process_placement, processes", ): shell_io.check_convert_qresources(qr) + + def test_header(self, shell_io): + # check that the required elements are properly handled in header template + options = { + "qout_path": "/some/file", + "qerr_path": "/some/file", + "job_name": "NAME", + } + header = shell_io.generate_header(options) + assert "exec > /some/file" in header + assert "exec 2> /some/file" in header + assert "NAME" in header From 838183b117f5a91a6d5ab667b14d0748f0893695 Mon Sep 17 00:00:00 2001 From: Fabian Peschel Date: Wed, 20 Sep 2023 14:07:41 +0200 Subject: [PATCH 31/34] fix bug --- src/qtoolkit/io/shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/qtoolkit/io/shell.py b/src/qtoolkit/io/shell.py index 75bee06..8da03fc 100644 --- a/src/qtoolkit/io/shell.py +++ b/src/qtoolkit/io/shell.py @@ -225,7 +225,7 @@ def parse_jobs_list_output(self, exit_code, stdout, stderr) -> list[QJob]: qjob.name = data[4] try: - shell_job_state = ShellState(data[3]) + shell_job_state = ShellState(data[3][0]) except ValueError: msg = f"Unknown job state {data[3]} for job id {qjob.job_id}" raise OutputParsingError(msg) From 179579e7ca7d8e7edc568ffdbd95a5ad180ec6c4 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Thu, 5 Oct 2023 14:00:28 +0200 Subject: [PATCH 32/34] Use ssh for copier git --- .copier-answers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 2d4a831..8dbe8bd 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,6 +1,6 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY _commit: 7a70b18 -_src_path: gh:Matgenix/Matgenix-copier-template +_src_path: git@github.com:Matgenix/Matgenix-copier-template author_email: david.waroquiers@matgenix.com author_fullname: David Waroquiers author_username: davidwaroquiers From c48e3897d3ca801745b08526c5b5c610b3f91650 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 6 Oct 2023 20:35:59 +0100 Subject: [PATCH 33/34] Update repo from copier template and tweak README --- .copier-answers.yml | 3 +- .github/workflows/release.yml | 89 +++++++++++++++++++++++++++++++++++ .github/workflows/testing.yml | 42 ++++++++--------- .pre-commit-config.yaml | 3 +- CONTRIBUTING.md | 5 +- INSTALL.md | 20 ++++++++ README.md | 38 ++++----------- pyproject.toml | 7 ++- 8 files changed, 150 insertions(+), 57 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 INSTALL.md diff --git a/.copier-answers.yml b/.copier-answers.yml index 8dbe8bd..156eb3e 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: 7a70b18 +_commit: d0e6447 _src_path: git@github.com:Matgenix/Matgenix-copier-template author_email: david.waroquiers@matgenix.com author_fullname: David Waroquiers @@ -15,3 +15,4 @@ repository_namespace: matgenix repository_provider: https://github.com short_description: QToolKit is a python wrapper interfacing with job queues (e.g. PBS, SLURM, ...). + diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..1e56097 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,89 @@ +name: Publish and Deploy + +on: + release: + types: + - published + +env: + PUBLISH_UPDATE_BRANCH: main + GIT_USER_NAME: Matgenix + GIT_USER_EMAIL: "dev@matgenix.com" + +jobs: + + publish: + name: Publish package + runs-on: ubuntu-latest + if: github.repository == 'matgenix/qtoolkit' && startsWith(github.ref, 'refs/tags/v') + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Python dependencies + run: | + python -m pip install -U pip + pip install -U setuptools wheel + pip install -e .[all] + + - name: Update changelog + uses: CharMixer/auto-changelog-action@v1 + with: + token: ${{ secrets.RELEASE_PAT_BOT }} + release_branch: ${{ env.PUBLISH_UPDATE_BRANCH }} + exclude_labels: "duplicate,question,invalid,wontfix,dependency_updates,skip_changelog" + + - name: Update '${{ env.PUBLISH_UPDATE_BRANCH }}' + uses: CasperWA/push-protected@v2 + with: + token: ${{ secrets.RELEASE_PAT_BOT }} + branch: ${{ env.PUBLISH_UPDATE_BRANCH }} + unprotect_reviews: true + sleep: 15 + force: true + tags: true + + - name: Get tagged versions + run: echo "PREVIOUS_VERSION=$(git tag -l --sort -version:refname | sed -n 2p)" >> $GITHUB_ENV + + - name: Create release-specific changelog + uses: CharMixer/auto-changelog-action@v1 + with: + token: ${{ secrets.RELEASE_PAT_BOT }} + release_branch: ${{ env.PUBLISH_UPDATE_BRANCH }} + since_tag: "${{ env.PREVIOUS_VERSION }}" + output: "release_changelog.md" + exclude_labels: "duplicate,question,invalid,wontfix,dependency_updates,skip_changelog" + + - name: Append changelog to release body + run: | + gh api /repos/${{ github.repository }}/releases/${{ github.event.release.id }} --jq '.body' > release_body.md + cat release_changelog.md >> release_body.md + gh api /repos/${{ github.repository }}/releases/${{ github.event.release.id }} -X PATCH -F body='@release_body.md' + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_PAT_BOT }} + + - name: Build source distribution + run: python -m build + + - name: Publish package to Test PyPI first + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_PASSWORD }} + repository-url: https://test.pypi.org/legacy/ + + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index d35a5a5..4523ad9 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: '3.8' + python-version: '3.10' cache: pip cache-dependency-path: pyproject.toml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v3 @@ -49,22 +49,22 @@ jobs: - name: Test run: pytest --cov=qtoolkit --cov-report=xml -# docs: -# runs-on: ubuntu-latest -# -# steps: -# - uses: actions/checkout@v3 -# -# - uses: actions/setup-python@v4 -# with: -# python-version: '3.11' -# cache: pip -# cache-dependency-path: pyproject.toml -# -# - name: Install dependencies -# run: | -# python -m pip install --upgrade pip -# pip install .[strict,docs] -# -# - name: Build -# run: jupyter-book build docs --path-output docs_build + docs: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install .[strict,docs] + + - name: Build + run: jupyter-book build docs --path-output docs_build diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 91c5c57..41dca2c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,7 @@ repos: - id: fix-encoding-pragma args: [--remove] - id: end-of-file-fixer + exclude: .copier-answers.yml - id: trailing-whitespace - repo: https://github.com/myint/autoflake rev: v2.0.0 @@ -67,4 +68,4 @@ repos: rev: v3.3.1 hooks: - id: pyupgrade - args: [--py38-plus] + args: [--py39-plus] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 544eb78..592e896 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,5 @@ # Contributing to QToolKit -Full name is Queue Tool Kit - We love your input! We want to make contributing to as easy and transparent as possible, whether it's: @@ -64,6 +62,9 @@ We have a few tips for writing good PRs that are accepted into the main repo: to python](https://docs.python-guide.org/writing/tests) for some good resources on writing good tests. - Understand your contributions will fall under the same license as this repo. +- This project uses `pre-commit` for uniform linting across many developers. You can install + it through the extra dev dependencies with `pip install -e .[dev]` and then run `pre-commit install` + to activate it for you local repository. When you submit your PR, our CI service will automatically run your tests. We welcome good discussion on the best ways to write your code, and the comments diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000..2b48ecd --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,20 @@ +# Installation + +Clone this repository and then install with `pip` in the virtual environment of your choice. + +``` +git clone git@https://github.com:matgenix/qtoolkit +cd qtoolkit +pip install . +``` + +## Development installation + +You can use + +``` +pip install -e .[dev,tests] +``` + +to perform an editable installation with additional development and test dependencies. +You can then activate `pre-commit` in your local repository with `pre-commit install`. diff --git a/README.md b/README.md index c53fbc8..f98191f 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,20 @@ # QToolKit -[![tests](https://img.shields.io/github/actions/workflow/status/matgenix/qtoolkit/testing.yml?branch=main&label=tests)](https://github.com/matgenix/qtoolkit/actions?query=workflow%3Atesting) +[![tests](https://img.shields.io/github/actions/workflow/status/matgenix/qtoolkit/testing.yml?branch=main&label=tests)](https://github.com/matgenix/qtoolkit/actions/workflows/testing.yml) [![code coverage](https://img.shields.io/codecov/c/gh/matgenix/qtoolkit)](https://codecov.io/gh/matgenix/qtoolkit) [![pypi version](https://img.shields.io/pypi/v/qtoolkit?color=blue)](https://pypi.org/project/qtoolkit) ![supported python versions](https://img.shields.io/pypi/pyversions/qtoolkit) -** [Full Documentation][docs] ** +**[Full Documentation][docs]** -QToolKit is a python software for ... Features of QToolKit include - -- Feature A -- Feature B -- ... - -## Quick start - -How to get started. - -## Installation - -How to install. - -## User guide - -How to use the software. +> **Warning**: +> :construction: This repository is still under construction. :construction: ## Need help? Ask questions about QToolKit on the [QToolKit support forum][help-forum]. If you've found an issue with QToolKit, please submit a bug report on [GitHub Issues][issues]. -## Reference - -Full reference - ## What’s new? Track changes to qtoolkit through the [changelog][changelog]. @@ -60,11 +41,12 @@ QToolKit is developed and maintained by Matgenix SRL. A full list of all contributors can be found [here][contributors]. -[help-forum]: https://github.com/materialsproject/atomate2/issues -[issues]: https://github.com/materialsproject/atomate2/issues -[installation]: https://matgenix.github.io/qtoolkit/user/install.html +[help-forum]: https://https://github.com/matgenix/qtoolkit/issues +[issues]: https://https://github.com/matgenix/qtoolkit/issues +[installation]: https://https://github.com/matgenix/qtoolkit/blob/main/INSTALL.md [contributing]: https://github.com/matgenix/qtoolkit/blob/main/CONTRIBUTING.md [codeofconduct]: https://github.com/matgenix/qtoolkit/blob/main/CODE_OF_CONDUCT.md -[contributors]: https://matgenix.github.io/qtoolkit/about/contributors.html -[license]: https://raw.githubusercontent.com/matgenix//main/LICENSE +[changelog]: https://https://github.com/matgenix/qtoolkit/blob/main/CHANGELOG.md +[contributors]: https://matgenix.github.io/qtoolkit/graphs/contributors +[license]: https://raw.githubusercontent.com/matgenix/qtoolkit/main/LICENSE [docs]: https://matgenix.github.io/qtoolkit/ diff --git a/pyproject.toml b/pyproject.toml index 92453e6..817bded 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,6 @@ authors = [{ name = "David Waroquiers", email = "david.waroquiers@matgenix.com" dynamic = ["version"] classifiers = [ "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -24,8 +23,8 @@ classifiers = [ "Topic :: Other/Nonlisted Topic", "Topic :: Scientific/Engineering", ] -requires-python = ">=3.8" -dependencies = [] +requires-python = ">=3.9" +dependencies =[] [project.optional-dependencies] dev = [ @@ -73,7 +72,7 @@ max-line-length = 88 max-doc-length = 88 select = "C, E, F, W, B" extend-ignore = "E203, W503, E501, F401, RST21" -min-python-version = "3.8.0" +min-python-version = "3.9.0" docstring-convention = "numpy" rst-roles = "class, func, ref, obj" From eeb12122bbb715c7ed519981ab52b6c9ea4f16ad Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 6 Oct 2023 21:15:25 +0100 Subject: [PATCH 34/34] Use separate token for test PyPI --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1e56097..69ce5c8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -79,7 +79,7 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ - password: ${{ secrets.PYPI_PASSWORD }} + password: ${{ secrets.TEST_PYPI_PASSWORD }} repository-url: https://test.pypi.org/legacy/ - name: Publish package to PyPI