From 09ac91654648adb684a58d5d2d7b1c11a503dae8 Mon Sep 17 00:00:00 2001 From: Leopold Talirz Date: Tue, 20 Oct 2020 22:08:45 +0200 Subject: [PATCH] Docs: update REST API wsgi scripts (#4488) The wsgi scripts for deploying the AiiDA REST in production were outdated and are updated. The how-to on deploying your own REST API server is significantly streamlined and now includes the wsgi files as well as the examplary apache virtualhost configuration. Co-authored-by: Giovanni Pizzi --- aiida/restapi/api.py | 5 +- aiida/restapi/run_api.py | 7 +- .../howto/include/snippets/aiida-rest.conf | 32 +++++++ .../include/snippets/myprofile-rest.wsgi | 9 ++ docs/source/howto/share_data.rst | 85 +++++------------- docs/wsgi/__init__.py | 9 -- docs/wsgi/app1/__init__.py | 9 -- docs/wsgi/app1/config.py | 88 ------------------ docs/wsgi/app1/rest.wsgi | 20 ----- docs/wsgi/app2/__init__.py | 9 -- docs/wsgi/app2/config.py | 89 ------------------- docs/wsgi/app2/rest.wsgi | 20 ----- docs/wsgi/many.conf | 39 -------- docs/wsgi/one.conf | 28 ------ tests/restapi/test_routes.py | 10 +-- 15 files changed, 71 insertions(+), 388 deletions(-) create mode 100644 docs/source/howto/include/snippets/aiida-rest.conf create mode 100755 docs/source/howto/include/snippets/myprofile-rest.wsgi delete mode 100644 docs/wsgi/__init__.py delete mode 100644 docs/wsgi/app1/__init__.py delete mode 100644 docs/wsgi/app1/config.py delete mode 100755 docs/wsgi/app1/rest.wsgi delete mode 100644 docs/wsgi/app2/__init__.py delete mode 100644 docs/wsgi/app2/config.py delete mode 100755 docs/wsgi/app2/rest.wsgi delete mode 100644 docs/wsgi/many.conf delete mode 100644 docs/wsgi/one.conf diff --git a/aiida/restapi/api.py b/aiida/restapi/api.py index bdba0ca2d4..7b2661efcb 100644 --- a/aiida/restapi/api.py +++ b/aiida/restapi/api.py @@ -8,9 +8,8 @@ # For further information please visit http://www.aiida.net # ########################################################################### """ -Implementation of RESTful API for materialscloud.org based on flask + -flask_restful module -For the time being the API returns the parsed valid endpoints upon GET request +Implementation of RESTful API for AiiDA based on flask and flask_restful. + Author: Snehal P. Waychal and Fernando Gargiulo @ Theos, EPFL """ diff --git a/aiida/restapi/run_api.py b/aiida/restapi/run_api.py index c9ba63fc82..b551d12769 100755 --- a/aiida/restapi/run_api.py +++ b/aiida/restapi/run_api.py @@ -76,9 +76,10 @@ def configure_api(flask_app=api_classes.App, flask_api=api_classes.AiidaApi, **k :type flask_app: :py:class:`flask.Flask` :param flask_api: flask_restful API class to be used to wrap the app :type flask_api: :py:class:`flask_restful.Api` - :param config: directory containing the config.py file used to configure the RESTapi - :param catch_internal_server: If true, catch and print all inter server errors - :param wsgi_profile: use WSGI profiler middleware for finding bottlenecks in web application + :param config: directory containing the config.py configuration file + :param catch_internal_server: If true, catch and print internal server errors with full python traceback. + Useful during app development. + :param wsgi_profile: use WSGI profiler middleware for finding bottlenecks in the web application :returns: Flask RESTful API :rtype: :py:class:`flask_restful.Api` diff --git a/docs/source/howto/include/snippets/aiida-rest.conf b/docs/source/howto/include/snippets/aiida-rest.conf new file mode 100644 index 0000000000..8b4d102ac3 --- /dev/null +++ b/docs/source/howto/include/snippets/aiida-rest.conf @@ -0,0 +1,32 @@ +# Apache virtual host configuration file for AiiDA REST API +# Copy to /etc/apache2/sites-enabled/aiida-rest.conf + + + + LogLevel debug + + # Let the app do authorization + WSGIPassAuthorization On + + # Require privileges on the wsgi directory + + Require all granted + + + # BEGIN SECTION for "myprofile" AiiDA profile + # Use 5 threads and "aiida" virtual python environment + WSGIDaemonProcess rest-myprofile \ + user=ubuntu group=ubuntu \ + threads=5 \ + python-home=/home/ubuntu/.virtualenvs/aiida \ + display-name=aiida-rest-myprofile + + # REST API will be served on /myprofile/api/v4 + WSGIScriptAlias /myprofile /home/ubuntu/wsgi/myprofile-rest.wsgi + + WSGIProcessGroup myprofile + + # END SECTION for "myprofile" AiiDA profile + + + diff --git a/docs/source/howto/include/snippets/myprofile-rest.wsgi b/docs/source/howto/include/snippets/myprofile-rest.wsgi new file mode 100755 index 0000000000..c886292320 --- /dev/null +++ b/docs/source/howto/include/snippets/myprofile-rest.wsgi @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +# wsgi script for AiiDA profile 'myprofile' +from aiida.restapi.run_api import configure_api +from aiida.manage.configuration import load_profile + +load_profile('myprofile') + +api = configure_api() +application = api.app diff --git a/docs/source/howto/share_data.rst b/docs/source/howto/share_data.rst index b033d65986..bb71eafe22 100644 --- a/docs/source/howto/share_data.rst +++ b/docs/source/howto/share_data.rst @@ -145,9 +145,13 @@ Like all ``verdi`` commands, you can select a different AiiDA profile via the `` REST API version history: - * ``aiida-core`` >= 1.0.0b6: ``v4`` - * ``aiida-core`` >= 1.0.0b3, <1.0.0b6: ``v3`` - * ``aiida-core`` <1.0.0b3: ``v2`` + +Version history +--------------- + + * ``aiida-core`` >= 1.0.0b6: ``v4``. Simplified endpoints; only ``/nodes``, ``/processes``, ``/calcjobs``, ``/groups``, ``/computers`` and ``/servers`` remain. + * ``aiida-core`` >= 1.0.0b3, <1.0.0b6: ``v3``. Development version, never shipped with a stable release. + * ``aiida-core`` <1.0.0b3: ``v2``. First API version, with new endpoints added step by step. .. _how-to:share:serve:query: @@ -171,77 +175,34 @@ Deploying a REST API server ^^^^^^^^^^^^^^^^^^^^^^^^^^^ The ``verdi restapi`` command runs the REST API through the ``werkzeug`` python-based HTTP server. -In order to deploy production instances of the REST API for serving your data to others, we recommend using a fully fledged web server, such as `Apache `_ or `NGINX `_. +In order to deploy production instances of the REST API for serving your data to others, we recommend using a fully fledged web server, such as `Apache `_ or `NGINX `_, which then runs the REST API python application through the `web server gateway interface (WSGI) `_. .. note:: - One Apache/NGINX server can host multiple APIs, e.g. connecting to different AiiDA profiles. - -In the following, we assume you have a working installation of Apache with the ``mod_wsgi`` `WSGI module `_ enabled. - -The goal of the example is to hookup the APIs ``django`` and ``sqlalchemy`` pointing to two AiiDA profiles, called for simplicity ``django`` and ``sqlalchemy``. - -All the relevant files are enclosed under the path ``/docs/wsgi/`` starting from the AiiDA source code path. -In each of the folders ``app1/`` and ``app2/``, there is a file named ``rest.wsgi`` containing a python script that instantiates and configures a python web app called ``application``, according to the rules of ``mod_wsgi``. -For how the script is written, the object ``application`` is configured through the file ``config.py`` contained in the same folder. -Indeed, in ``app1/config.py`` the variable ``aiida-profile`` is set to ``"django"``, whereas in ``app2/config.py`` its value is ``"sqlalchemy"``. - -The path where you put the ``.wsgi`` file as well as its name are irrelevant as long as they are correctly referred to in the Apache configuration file, as shown later on. -Similarly, you can place ``config.py`` in a custom path, provided you change the variable ``config_file_path`` in the ``wsgi file`` accordingly. - -In ``rest.wsgi`` the only options you might want to change is ``catch_internal_server``. -When set to ``True``, it lets the exceptions thrown during the execution of the app propagate all the way through until they reach the logger of Apache. -Especially when the app is not entirely stable yet, one would like to read the full python error traceback in the Apache error log. + One Apache/NGINX server can host multiple instances of the REST APIs, e.g. serving data from different AiiDA profiles. -Finally, you need to setup the Apache site through a proper configuration file. -We provide two template files: ``one.conf`` or ``many.conf``. -The first file tells Apache to bundle both apps in a unique Apache daemon process. -Apache usually creates multiple processes dynamically and with this configuration each process will handle both apps. +A ``myprofile-rest.wsgi`` script for an AiiDA profile ``myprofile`` would look like this: -The script ``many.conf``, instead, defines two different process groups, one for each app. -So the processes created dynamically by Apache will always be handling one app each. -The minimal number of Apache daemon processes equals the number of apps, contrarily to the first architecture, where one process is enough to handle two or even a larger number of apps. +.. literalinclude:: include/snippets/myprofile-rest.wsgi -Let us call the two apps for this example ``django`` and ``sqlalchemy``, matching with the chosen AiiDA profiles. -In both ``one.conf`` and ``many.conf``, the important directives that should be updated if one changes the paths or names of the apps are: +.. note:: See the documentation of :py:func:`~aiida.restapi.run_api.configure_api` for all available configuration options. - - ``WSGIProcessGroup`` to define the process groups for later reference. - In ``one.conf`` this directive appears only once to define the generic group ``profiles``, as there is only one kind of process handling both apps. - In ``many.conf`` this directive appears once per app and is embedded into a "Location" tag, e.g.:: +In the following, we explain how to run this wsgi application using Apache on Ubuntu. - - WSGIProcessGroup sqlalchemy - + #. Install and enable the ``mod_wsgi`` `WSGI module `_ module: - - ``WSGIDaemonProcess`` to define the path to the AiiDA virtual environment. - This appears once per app in both configurations. - - - ``WSGIScriptAlias`` to define the absolute path of the ``.wsgi`` file of each app. - - - The ```` tag mainly used to grant Apache access to the files used by each app, e.g.:: - - /aiida/restapi/wsgi/app1"> - Require all granted - - -The latest step is to move either ``one.conf`` or ``many.conf`` into the Apache configuration folder and restart the Apache server. -In Ubuntu, this is usually done with the commands: + .. code-block:: console -.. code-block:: bash + $ sudo apt install libapache2-mod-wsgi-py3 + $ sudo a2enmod wsgi - cp .conf /etc/apache2/sites-enabled/000-default.conf - sudo service apache2 restart + #. Place the WSGI script in a folder on your server, for example ``/home/ubuntu/wsgi/myprofile-rest.wsgi``. -We believe the two basic architectures we have just explained can be successfully applied in many different deployment scenarios. -Nevertheless, we suggest users who need finer tuning of the deployment setup to look into to the official documentation of `Apache `_ and, more importantly, `WSGI `_. + #. Configure apache to run the WSGI application using a virtual host configuration similar to: -The URLs of the requests handled by Apache must start with one of the paths specified in the directives ``WSGIScriptAlias``. -These paths identify uniquely each app and allow Apache to route the requests to their correct apps. -Examples of well-formed URLs are: + .. literalinclude:: include/snippets/aiida-rest.conf -.. code-block:: bash + Place this ``aiida-rest.conf`` file in ``/etc/apache2/sites-enabled`` - curl http://localhost/django/api/v4/computers -X GET - curl http://localhost/sqlalchemy/api/v4/computers -X GET + #. Restart apache: ``sudo service apache2 restart``. -The first (second) request will be handled by the app ``django`` (``sqlalchemy``), and will serve results fetched from the AiiDA profile ``django`` (``sqlalchemy``). -Notice that we have not specified any port in the URLs since Apache listens conventionally to port 80, where any request lacking the port is automatically redirected (port 443 for HTTPS). +You should now be able to reach your REST API at ``localhost/myprofile/api/v4`` (Port 80). diff --git a/docs/wsgi/__init__.py b/docs/wsgi/__init__.py deleted file mode 100644 index 2776a55f97..0000000000 --- a/docs/wsgi/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -########################################################################### -# Copyright (c), The AiiDA team. All rights reserved. # -# This file is part of the AiiDA code. # -# # -# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # -# For further information on the license, see the LICENSE.txt file # -# For further information please visit http://www.aiida.net # -########################################################################### diff --git a/docs/wsgi/app1/__init__.py b/docs/wsgi/app1/__init__.py deleted file mode 100644 index 2776a55f97..0000000000 --- a/docs/wsgi/app1/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -########################################################################### -# Copyright (c), The AiiDA team. All rights reserved. # -# This file is part of the AiiDA code. # -# # -# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # -# For further information on the license, see the LICENSE.txt file # -# For further information please visit http://www.aiida.net # -########################################################################### diff --git a/docs/wsgi/app1/config.py b/docs/wsgi/app1/config.py deleted file mode 100644 index f3be6bfc8a..0000000000 --- a/docs/wsgi/app1/config.py +++ /dev/null @@ -1,88 +0,0 @@ -# -*- coding: utf-8 -*- -########################################################################### -# Copyright (c), The AiiDA team. All rights reserved. # -# This file is part of the AiiDA code. # -# # -# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # -# For further information on the license, see the LICENSE.txt file # -# For further information please visit http://www.aiida.net # -########################################################################### -import os -import json - -## Pagination defaults -LIMIT_DEFAULT = 400 -PERPAGE_DEFAULT = 20 - -##Version prefix for all the URLs -PREFIX='/api/v3' - - -""" -Flask app configs. - -DEBUG: True/False. enables debug mode N.B. -!!!For production run use ALWAYS False!!! - -PROPAGATE_EXCEPTIONS: True/False serve REST exceptions to the client (and not a -generic 500: Internal Server Error exception) - -""" -APP_CONFIG = { - 'DEBUG': False, - 'PROPAGATE_EXCEPTIONS': True, - } - - -""" -JSON serialization config. Leave this dictionary empty if default Flask -serializer is desired. - -Here is a list a all supported fields. If a field is not present in the -dictionary its value is assumed to be 'default'. - -DATETIME_FORMAT: allowed values are 'asinput' and 'default'. - -""" -SERIALIZER_CONFIG = {'datetime_format': 'default'} - -""" -Caching configuration - -memcached: backend caching system -""" -cache_config={'CACHE_TYPE': 'memcached'} -CACHING_TIMEOUTS = { #Caching TIMEOUTS (in seconds) - 'nodes': 10, - 'users': 10, - 'calculations': 10, - 'computers': 10, - 'datas': 10, - 'groups': 10, - 'codes': 10, -} - -""" -Schema customization (if file schema_custom.json is present in this same folder) -""" -#TODO add more verbose description -schema_custom_config = os.path.join(os.path.split(__file__)[0], 'schema_custom.json') -try: - with open(schema_custom_config) as fin: - custom_schema = json.load(fin) -except IOError: - custom_schema = {} - -# IO tree -MAX_TREE_DEPTH = 5 - -""" -Aiida profile used by the REST api when no profile is specified (ex. by ---aiida-profile flag). -This has to be one of the profiles registered in .aiida/config.json - -In case you want to use the default stored in -.aiida/config.json, set this varibale to "default" - -""" -default_aiida_profile = 'django' diff --git a/docs/wsgi/app1/rest.wsgi b/docs/wsgi/app1/rest.wsgi deleted file mode 100755 index dd68c112e9..0000000000 --- a/docs/wsgi/app1/rest.wsgi +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -import sys -import os -from aiida.restapi.api import App, AiidaApi -from aiida.restapi.run_api import run_api - -AIIDA_DIR = os.path.join( - os.path.dirname( - os.path.abspath(__file__) - ), - os.pardir, os.pardir, os.pardir, 'aiida/') - -sys.path = [AIIDA_DIR] + sys.path - -config_file_path = os.path.dirname(os.path.abspath(__file__)) - -(application, api) = run_api(App, AiidaApi, - '--config-dir', config_file_path, - hookup=False, - catch_internal_server=False) diff --git a/docs/wsgi/app2/__init__.py b/docs/wsgi/app2/__init__.py deleted file mode 100644 index 2776a55f97..0000000000 --- a/docs/wsgi/app2/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -########################################################################### -# Copyright (c), The AiiDA team. All rights reserved. # -# This file is part of the AiiDA code. # -# # -# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # -# For further information on the license, see the LICENSE.txt file # -# For further information please visit http://www.aiida.net # -########################################################################### diff --git a/docs/wsgi/app2/config.py b/docs/wsgi/app2/config.py deleted file mode 100644 index a4dbfa2228..0000000000 --- a/docs/wsgi/app2/config.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*- coding: utf-8 -*- -########################################################################### -# Copyright (c), The AiiDA team. All rights reserved. # -# This file is part of the AiiDA code. # -# # -# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # -# For further information on the license, see the LICENSE.txt file # -# For further information please visit http://www.aiida.net # -########################################################################### -## Pagination defaults -LIMIT_DEFAULT = 400 -PERPAGE_DEFAULT = 20 - -##Version prefix for all the URLs (in most cases, you need to omit trailing -# slash) -PREFIX='/api/v3' - - -""" -Flask app configs. - -DEBUG: True/False. enables debug mode N.B. -!!!For production run use ALWAYS False!!! - -PROPAGATE_EXCEPTIONS: True/False serve REST exceptions to the client (and not a -generic 500: Internal Server Error exception) - -""" -APP_CONFIG = { - 'DEBUG': False, - 'PROPAGATE_EXCEPTIONS': True, - } - - -""" -JSON serialization config. Leave this dictionary empty if default Flask -serializer is desired. - -Here is a list a all supported fields. If a field is not present in the -dictionary its value is assumed to be 'default'. - -DATETIME_FORMAT: allowed values are 'asinput' and 'default'. - -""" -SERIALIZER_CONFIG = {'datetime_format': 'default'} - -""" -Caching configuration - -memcached: backend caching system -""" -cache_config={'CACHE_TYPE': 'memcached'} -CACHING_TIMEOUTS = { #Caching TIMEOUTS (in seconds) - 'nodes': 10, - 'users': 10, - 'calculations': 10, - 'computers': 10, - 'datas': 10, - 'groups': 10, - 'codes': 10, -} - -""" -Schema customization (if file schema_custom.json is present in this same folder) -""" -#TODO add more verbose description -import os -import json - -schema_custom_config = os.path.join(os.path.split(__file__)[0], 'schema_custom.json') -try: - with open(schema_custom_config) as fin: - custom_schema = json.load(fin) -except IOError: - custom_schema = {} - -# IO tree -MAX_TREE_DEPTH = 5 - -""" -Aiida profile used by the REST api when no profile is specified (ex. by ---aiida-profile flag). -This has to be one of the profiles registered in .aiida/config.json - -In case you want to use the default stored in -.aiida/config.json, set this varibale to "default" - -""" -default_aiida_profile = 'sqlalchemy' diff --git a/docs/wsgi/app2/rest.wsgi b/docs/wsgi/app2/rest.wsgi deleted file mode 100755 index dd68c112e9..0000000000 --- a/docs/wsgi/app2/rest.wsgi +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -import sys -import os -from aiida.restapi.api import App, AiidaApi -from aiida.restapi.run_api import run_api - -AIIDA_DIR = os.path.join( - os.path.dirname( - os.path.abspath(__file__) - ), - os.pardir, os.pardir, os.pardir, 'aiida/') - -sys.path = [AIIDA_DIR] + sys.path - -config_file_path = os.path.dirname(os.path.abspath(__file__)) - -(application, api) = run_api(App, AiidaApi, - '--config-dir', config_file_path, - hookup=False, - catch_internal_server=False) diff --git a/docs/wsgi/many.conf b/docs/wsgi/many.conf deleted file mode 100644 index d75c5390f1..0000000000 --- a/docs/wsgi/many.conf +++ /dev/null @@ -1,39 +0,0 @@ -# To use this configuration copy this file to /etc/apache2/sites-enabled/000-default.conf - - - ServerAdmin webmaster@localhost - -# Django app - WSGIDaemonProcess django python-home= - WSGIScriptAlias /django /docs/wsgi/app1/rest.wsgi - -# SQLAlchemy App - WSGIDaemonProcess sqlalchemy python-home=/ - WSGIScriptAlias /sqlalchemy /docs/wsgi/app2/rest.wsgi - -# Assigning apps to different Process groups - - WSGIProcessGroup django - - - - WSGIProcessGroup sqlalchemy - - -# Have this option On if authorization is done by the app rather than Apache - WSGIPassAuthorization On - -# Require privileges on the folders of both apps - /docs/wsgi/app1"> - Require all granted - - /docs/wsgi/app2"> - Require all granted - - -# Apache log and debug confs - LogLevel debug - ErrorLog ${APACHE_LOG_DIR}/error.log - CustomLog ${APACHE_LOG_DIR}/access.log combined - - diff --git a/docs/wsgi/one.conf b/docs/wsgi/one.conf deleted file mode 100644 index 265605d560..0000000000 --- a/docs/wsgi/one.conf +++ /dev/null @@ -1,28 +0,0 @@ -# To use this configuration copy this file to /etc/apache2/sites-enabled/000-default.conf - - - ServerAdmin webmaster@localhost - -# A unique process type and group for all apps - WSGIProcessGroup profiles - WSGIDaemonProcess profiles python-home= - WSGIScriptAlias /django /docs/wsgi/app1/rest.wsgi - WSGIScriptAlias /sqlalchemy /docs/wsgi/app2/rest.wsgi - -# Have this option On if authorization is done by the app rather than Apache - WSGIPassAuthorization On - -# Require privileges on the folders of both apps - /docs/wsgi/app1"> - Require all granted - - /docs/wsgi/app2"> - Require all granted - - -# Apache log and debug confs - LogLevel debug - ErrorLog ${APACHE_LOG_DIR}/error.log - CustomLog ${APACHE_LOG_DIR}/access.log combined - - diff --git a/tests/restapi/test_routes.py b/tests/restapi/test_routes.py index 2fed4bfd64..8a7c5a43d0 100644 --- a/tests/restapi/test_routes.py +++ b/tests/restapi/test_routes.py @@ -32,19 +32,11 @@ class RESTApiTestCase(AiidaTestCase): @classmethod def setUpClass(cls, *args, **kwargs): # pylint: disable=too-many-locals, too-many-statements """ - Basides the standard setup we need to add few more objects in the - database to be able to explore different requests/filters/orderings etc. + Add objects to the database for different requests/filters/orderings etc. """ - # call parent setUpClass method super().setUpClass() - # connect the app and the api - # Init the api by connecting it the the app (N.B. respect the following - # order, api.__init__) - kwargs = dict(PREFIX=cls._url_prefix, PERPAGE_DEFAULT=cls._PERPAGE_DEFAULT, LIMIT_DEFAULT=cls._LIMIT_DEFAULT) - api = configure_api(catch_internal_server=True) - cls.app = api.app cls.app.config['TESTING'] = True