Merge branch 'main' into get_daemon_client(profile_name)

.docker/tests/test_aiida.py

@@ -1,4 +1,5 @@
 import json
+import re
 
 import pytest
 from packaging.version import parse
@@ -32,8 +33,10 @@ def test_verdi_status(aiida_exec, container_user):
     assert '✔ broker:' in output
     assert 'Daemon is running' in output
 
-    # check that we have suppressed the warnings
-    assert 'Warning' not in output
+    # Check that we have suppressed the warnings coming from using an install from repo and newer RabbitMQ version.
+    # Make sure to match only lines that start with ``Warning:`` because otherwise deprecation warnings from other
+    # packages that we cannot control may fail the test.
+    assert not re.match('^Warning:.*', output)
 
 
 def test_computer_setup_success(aiida_exec, container_user):

docs/source/installation/guide_complete.rst

@@ -130,7 +130,7 @@ Although it is possible to run AiiDA without a daemon it does provide significan
         .. important::
 
             The ``aiida-core.services`` package ensures that RabbitMQ is installed in the conda environment.
-            However, it is not a _service_, in the sense that it is not automatically started, but has to be started manually.
+            However, it is not a *service*, in the sense that it is not automatically started, but has to be started manually.
 
             .. code-block:: console
 
@@ -254,7 +254,7 @@ Common options
 
 The exact options available for the ``verdi profile setup`` command depend on the selected storage plugin, but there are a number of common options and functionality:
 
-* ``--profile``: The name of the profile.
+* ``--profile-name``: The name of the profile.
 * ``--set-as-default``: Whether the new profile should be defined as the new default.
 * ``--email``: Email for the default user that is created.
 * ``--first-name``: First name for the default user that is created.
@@ -276,6 +276,10 @@ The exact options available for the ``verdi profile setup`` command depend on th
 ``core.sqlite_dos``
 -------------------
 
+.. tip::
+
+    The ``verdi presto`` command provides a fully automated way to set up a profile with the ``core.sqlite_dos`` storage plugin if no configuration is required.
+
 This storage plugin uses `SQLite <https://sqlite.org/>`_ and the `disk-objectstore <https://disk-objectstore.readthedocs.io/en/latest/>`_ to store data.
 The ``disk-objectstore`` is a Python package that is automatically installed as a dependency when installing ``aiida-core``, which was covered in the :ref:`Python package installation section <installation:guide-complete:python-package>`.
 The installation instructions for SQLite depend on your system; please visit the `SQLite website <https://www.sqlite.org/download.html>`_ for details.
@@ -296,20 +300,23 @@ The options specific to the ``core.sqlite_dos`` storage plugin are:
 ``core.psql_dos``
 -----------------
 
-This storage plugin uses `PostgreSQL <https://www.postgresql.org/>`_ and the `disk-objectstore <https://disk-objectstore.readthedocs.io/en/latest/>`_ to store data.
-The ``disk-objectstore`` is a Python package that is automatically installed as a dependency when installing ``aiida-core``, which was covered in the :ref:`Python package installation section <installation:guide-complete:python-package>`.
-The storage plugin can connect to a PostgreSQL instance running on the localhost or on a server that can be reached over the internet.
-Instructions for installing PostgreSQL is beyond the scope of this guide.
-
 .. tip::
 
     The creation of the PostgreSQL user and database as explained below is implemented in an automated way in the ``verdi presto`` command.
-    Instead of performing the steps below manually and running ``verdi profile setup core.psql_dos`` manually, it is possible to run:
+    Instead of performing the steps below manually and running ``verdi profile setup core.psql_dos``, it is possible to run:
 
     .. code-block::
 
         verdi presto --use-postgres
 
+    The ``verdi presto`` command also automatically tries to configure RabbitMQ as the broker if it is running locally.
+    Therefore, if the command succeeds in connecting to both PostgreSQL and RabbitMQ, ``verdi presto --use-postgres`` provides a fully automated way to create a profile suitable for production workloads.
+
+This storage plugin uses `PostgreSQL <https://www.postgresql.org/>`_ and the `disk-objectstore <https://disk-objectstore.readthedocs.io/en/latest/>`_ to store data.
+The ``disk-objectstore`` is a Python package that is automatically installed as a dependency when installing ``aiida-core``, which was covered in the :ref:`Python package installation section <installation:guide-complete:python-package>`.
+The storage plugin can connect to a PostgreSQL instance running on the localhost or on a server that can be reached over the internet.
+Instructions for installing PostgreSQL is beyond the scope of this guide.
+
 Before creating a profile, a database (and optionally a custom database user) has to be created.
 First, connect to PostgreSQL using ``psql``, the `native command line client for PostgreSQL <https://www.postgresql.org/docs/current/app-psql.html>`_:
 

docs/source/installation/guide_quick.rst

@@ -4,17 +4,18 @@
 Quick installation guide
 ========================
 
+.. warning::
+
+    Not all AiiDA functionality is supported by the quick installation.
+    Please refer to the :ref:`section below <installation:guide-quick:limitations>` for more information and see the :ref:`complete installation guide <installation:guide-complete>` for instructions to set up a feature-complete and performant installation.
+
+
 First, install the ``aiida-core`` Python package:
 
 .. code-block:: console
 
     pip install aiida-core
 
-.. attention::
-
-    AiiDA requires a recent version of Python.
-    Please refer to the `Python Package Index <https://pypi.org/project/aiida-core/>`_ for the minimum required version.
-
 Next, set up a profile where all data is stored:
 
 .. code-block:: console
@@ -40,46 +41,67 @@ If none of the lines show a red cross, indicating a problem, the installation wa
 
     If you encountered any issues, please refer to the :ref:`troubleshooting section <installation:troubleshooting>`.
 
-.. warning::
-
-    Not all AiiDA functionality is supported by the quick installation.
-    Please refer to the :ref:`section below <installation:guide-quick:limitations>` for more information.
-
 
 .. _installation:guide-quick:limitations:
 
 Quick install limitations
 =========================
 
-Functionality
--------------
+A setup that is ideal for production work requires the PostgreSQL and RabbitMQ services.
+By default, ``verdi presto`` creates a profile that allows running AiiDA without these:
+
+* **Database**: The PostgreSQL database that is used to store queryable data, is replaced by SQLite.
+* **Broker**: The RabbitMQ message broker that allows communication with and between processes is disabled.
+
+The following matrix shows the possible combinations of the database and broker options and their use cases:
+
++----------------------+----------------------------------------------------+-------------------------------------------------------------+
+|                      | **SQLite database**                                | **PostgreSQL database**                                     |
++======================+====================================================+=============================================================+
+| **No broker**        | Quick start with AiiDA                             | [*not really relevant for any usecase*]                     |
++----------------------+----------------------------------------------------+-------------------------------------------------------------+
+| **RabbitMQ**         | Production (low-throughput; beta, has limitations) | Production (maximum performance, ideal for high-throughput) |
++----------------------+----------------------------------------------------+-------------------------------------------------------------+
+
+The sections below provide details on the use of the PostgreSQL and RabbitMQ services and the limitations when running AiiDA without them.
+
+.. _installation:guide-quick:limitations:rabbitmq:
+
+RabbitMQ
+--------
 
 Part of AiiDA's functionality requires a `message broker <https://en.wikipedia.org/wiki/Message_broker>`_, with the default implementation using `RabbitMQ <https://www.rabbitmq.com/>`_.
-The message broker is used to allow communication with the :ref:`daemon <topics:daemon>`.
-Since RabbitMQ is a separate service and is not always trivial to install, the quick installation guide sets up a profile that does not require it.
-As a result, the daemon cannot be started and processes cannot be submitted to it but can only be run locally.
+The message broker is used to allow communication with processes and the :ref:`daemon <topics:daemon>` as well as between themselves.
+Since RabbitMQ is a separate service and is not always trivial to install, the quick installation guide allows setting up a profile that does not require it.
+However, as a result the profile:
+
+* is not suitable for high-throughput workloads (a polling-based mechanism is used rather than an event-based one)
+* cannot run the daemon (no ``verdi daemon start/stop``) and therefore processes cannot be submitted to the daemon (i.e., one can only use ``run()`` instead of ``submit()`` to launch calculations and workflows)
+* cannot play, pause, kill processes
 
 .. note::
     The ``verdi presto`` command automatically checks if RabbitMQ is running on the localhost.
-    If it can successfully connect, it configures the profile with the message broker and therefore the daemon functionality will be available.
+    If it can successfully connect, it configures the profile with the message broker and therefore the limitations listed above do not apply.
 
 .. tip::
-    The connection parameters of RabbitMQ can be (re)configured after the profile is set up with ``verdi profile configure-rabbitmq``.
-    This can be useful when the RabbitMQ setup is different from the default that AiiDA checks for and the automatic configuration of ``verdi presto`` failed.
+    A profile created by ``verdi presto`` can easily start using RabbitMQ as the broker at a later stage.
+    Once a RabbitMQ service is available (see :ref:`install RabbitMQ <installation:guide-complete:rabbitmq>` for instruction to install it) and run ``verdi profile configure-rabbitmq`` to configure its use for the profile.
 
+.. _installation:guide-quick:limitations:postgresql:
 
-Performance
------------
+PostgreSQL
+----------
 
-The quick installation guide by default creates a profile that uses `SQLite <https://www.sqlite.org/>`_ for the database.
-Since SQLite does not require running a service, it is easy to install and use on essentially any system.
-However, for certain use cases it is not going to be the most performant solution.
-AiiDA also supports `PostgreSQL <https://www.postgresql.org/>`_ which is often going to be more performant compared to SQLite.
+AiiDA stores (part of) the data of the provenance graph in a database and the `PostgreSQL <https://www.postgresql.org/>`_ service provides great performance for use-cases that require high-throughput.
+Since PostgreSQL is a separate service and is not always trivial to install, the quick installation guide allows setting up a profile that uses the serverless `SQLite <https://www.sqlite.org/>`_ instead.
+However, as a result the profile:
+
+* is not suitable for high-throughput workloads (concurrent writes from multiple processes to the database are serialized)
+* does not support the ``has_key`` and ``contains`` operators in the ``QueryBuilder``
+* does not support the ``get_creation_statistics`` method of the ``QueryBuilder``
 
 .. tip::
     If a PostgreSQL service is available, run ``verdi presto --use-postgres`` to set up a profile that uses PostgreSQL instead of SQLite.
     The command tries to connect to the service and automatically create a user account and database to use for the new profile.
     AiiDA provides defaults that work for most setups where PostgreSQL is installed on the localhost.
     Should this fail, the connection parameters can be customized using the ``--postgres-hostname``, ``--postgres-port``, ``--postgres-username``, ``--postgres-password`` options.
-
-Please refer to the :ref:`complete installation guide <installation:guide-complete>` for instructions to set up a feature-complete and performant installation.

pyproject.toml

@@ -174,6 +174,7 @@ requires-python = '>=3.9'
 [project.entry-points.'aiida.transports']
 'core.local' = 'aiida.transports.plugins.local:LocalTransport'
 'core.ssh' = 'aiida.transports.plugins.ssh:SshTransport'
+'core.ssh_auto' = 'aiida.transports.plugins.ssh_auto:SshAutoTransport'
 
 [project.entry-points.'aiida.workflows']
 'core.arithmetic.add_multiply' = 'aiida.workflows.arithmetic.add_multiply:add_multiply'

src/aiida/__init__.py

@@ -27,7 +27,7 @@
     'For further information please visit http://www.aiida.net/. All rights reserved.'
 )
 __license__ = 'MIT license, see LICENSE.txt file.'
-__version__ = '2.6.1'
+__version__ = '2.6.1.post0'
 __authors__ = 'The AiiDA team.'
 __paper__ = (
     'S. P. Huber et al., "AiiDA 1.0, a scalable computational infrastructure for automated reproducible workflows and '

src/aiida/cmdline/commands/cmd_computer.py

@@ -605,7 +605,7 @@ def computer_test(user, print_traceback, computer):
             message += '\n  Use the `--print-traceback` option to see the full traceback.'
 
         echo.echo(message)
-        echo.echo_warning(f'{1} out of {num_tests} tests failed')
+        echo.echo_warning('1 out of 1 tests failed')
 
 
 @verdi_computer.command('delete')

src/aiida/cmdline/commands/cmd_presto.py

@@ -25,9 +25,17 @@
 
 
 def get_default_presto_profile_name():
+    from aiida.common.exceptions import ConfigurationError
     from aiida.manage import get_config
 
-    profile_names = get_config().profile_names
+    try:
+        profile_names = get_config().profile_names
+    except ConfigurationError:
+        # This can happen when tab-completing in an environment that did not create the configuration folder yet.
+        # It would have been possible to just call ``get_config(create=True)`` to create the config directory, but this
+        # should not be done during tab-completion just to generate a default value.
+        return DEFAULT_PROFILE_NAME_PREFIX
+
     indices = []
 
     for profile_name in profile_names:
@@ -170,7 +178,7 @@ def verdi_presto(
     created profile uses the new PostgreSQL database instead of SQLite.
     """
     from aiida.brokers.rabbitmq.defaults import detect_rabbitmq_config
-    from aiida.common import exceptions
+    from aiida.common import docs, exceptions
     from aiida.manage.configuration import create_profile, load_profile
     from aiida.orm import Computer
 
@@ -209,6 +217,7 @@ def verdi_presto(
         broker_config = detect_rabbitmq_config()
     except ConnectionError as exception:
         echo.echo_report(f'RabbitMQ server not found ({exception}): configuring the profile without a broker.')
+        echo.echo_report(f'See {docs.URL_NO_BROKER} for details on the limitations of running without a broker.')
     else:
         echo.echo_report('RabbitMQ server detected: configuring the profile with a broker.')
         broker_backend = 'core.rabbitmq'

src/aiida/cmdline/commands/cmd_process.py

@@ -101,6 +101,7 @@ def process_list(
 
     from aiida.cmdline.commands.cmd_daemon import execute_client_command
     from aiida.cmdline.utils.common import print_last_process_state_change
+    from aiida.common.docs import URL_NO_BROKER
     from aiida.common.exceptions import ConfigurationError
     from aiida.engine.daemon.client import get_daemon_client
     from aiida.orm import ProcessNode, QueryBuilder
@@ -137,7 +138,7 @@ def process_list(
     try:
         client = get_daemon_client()
     except ConfigurationError:
-        echo.echo_warning('This profile does not have a broker and so it has no daemon.')
+        echo.echo_warning(f'This profile does not have a broker and so it has no daemon. See {URL_NO_BROKER}')
         return
 
     if not client.is_daemon_running:

src/aiida/cmdline/commands/cmd_profile.py

@@ -55,6 +55,7 @@ def command_create_profile(
     :param kwargs: Arguments to initialise instance of the selected storage implementation.
     """
     from aiida.brokers.rabbitmq.defaults import detect_rabbitmq_config
+    from aiida.common import docs
     from aiida.plugins.entry_point import get_entry_point_from_class
 
     if not storage_cls.read_only and email is None:
@@ -79,6 +80,7 @@ def command_create_profile(
     else:
         echo.echo_report('Creating profile without RabbitMQ.')
         echo.echo_report('It can be configured at a later point in time with `verdi profile configure-rabbitmq`.')
+        echo.echo_report(f'See {docs.URL_NO_BROKER} for details on the limitations of running without a broker.')
 
     try:
         profile = create_profile(

src/aiida/cmdline/commands/cmd_status.py

@@ -58,6 +58,7 @@ class ServiceStatus(enum.IntEnum):
 def verdi_status(print_traceback, no_rmq):
     """Print status of AiiDA services."""
     from aiida import __version__
+    from aiida.common.docs import URL_NO_BROKER
     from aiida.common.exceptions import ConfigurationError
     from aiida.engine.daemon.client import DaemonException, DaemonNotRunningException
     from aiida.manage.configuration.settings import AIIDA_CONFIG_FOLDER
@@ -141,7 +142,7 @@ def verdi_status(print_traceback, no_rmq):
         print_status(
             ServiceStatus.WARNING,
             'broker',
-            'No broker defined for this profile: certain functionality not available.',
+            f'No broker defined for this profile: certain functionality not available. See {URL_NO_BROKER}',
         )
 
     # Getting the daemon status
@@ -151,7 +152,7 @@ def verdi_status(print_traceback, no_rmq):
         print_status(
             ServiceStatus.WARNING,
             'daemon',
-            'No broker defined for this profile: daemon is not available.',
+            'No broker defined for this profile: daemon is not available. See {URL_NO_BROKER}',
         )
     except DaemonNotRunningException as exception:
         print_status(ServiceStatus.WARNING, 'daemon', str(exception))