Merge branch 'main' into related-data-mouseover

HEPData · Oct 16, 2023 · 5d8a8b2 · 5d8a8b2
2 parents 5ec84e1 + 697ccbb
commit 5d8a8b2
Show file tree

Hide file tree

Showing 20 changed files with 72 additions and 39 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -53,12 +53,10 @@ jobs:
     strategy:
 
       matrix:
-        postgres-version: [ 12, 13 ]
+        postgres-version: [ 14 ]
         os-version: [ '2.7.0' ]
-        python-version: [ '3.7', '3.8', '3.9' ]
+        python-version: [ '3.8', '3.9' ]
         exclude:
-          - postgres-version: ${{ github.event.act && 13 }}
-          - python-version: ${{ github.event.act && '3.7' }}
           - python-version: ${{ github.event.act && '3.8' }}
 
     # Service containers to run with `runner-job`
@@ -159,24 +157,24 @@ jobs:
         py.test -vv tests/*_test.py
     - name: Setup Sauce Connect
       uses: saucelabs/sauce-connect-action@v2
-      if: startsWith(matrix.postgres-version, '12') && startsWith(matrix.python-version, '3.9')
+      if: startsWith(matrix.python-version, '3.9')
       with:
         username: ${{ secrets.SAUCE_USERNAME }}
         accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
         region: eu-central
         tunnelName: ${{ github.run_id }}
-        scVersion: 4.9.0
+        scVersion: 4.9.1
         verbose: true
     - name: Run end-to-end tests
-      if: startsWith(matrix.postgres-version, '12') && startsWith(matrix.python-version, '3.9')
+      if: startsWith(matrix.python-version, '3.9')
       env:
         SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
         SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
         COVERAGE_FILE: '.coverage_e2e'
       run: |
         if [[ -n ${{ secrets.SAUCE_USERNAME }} && -n ${{ secrets.SAUCE_ACCESS_KEY}} ]]; then py.test -vv tests/e2e; fi
     - name: Run coveralls
-      if: ${{ startsWith(matrix.postgres-version, '12') && startsWith(matrix.python-version, '3.9') && !env.ACT }}
+      if: ${{ startsWith(matrix.python-version, '3.9') && !env.ACT }}
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         COVERALLS_SERVICE_NAME: github

diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
@@ -92,7 +92,7 @@ Before you submit a pull request, check that it meets these guidelines:
 1. The pull request should include tests and must not decrease test coverage.
 2. If the pull request adds functionality, the docs should be updated. Put
    your new functionality into a function with a docstring.
-3. The pull request should work for Python 3.7, 3.8 and 3.9. Check
+3. The pull request should work for Python 3.8 and 3.9. Check
    https://github.com/HEPData/hepdata/actions?query=event%3Apull_request
    and make sure that the tests pass.  Sometimes there are temporary failures,
    for example, due to unavailability of an external service or the test infrastructure.

diff --git a/Dockerfile b/Dockerfile
@@ -43,8 +43,8 @@ RUN hepdata collect -v  && \
 
 RUN bash -c "echo $APP_ENVIRONMENT"
 
-RUN bash -c "set -x; [[ ${APP_ENVIRONMENT:-prod} = local-web ]] && (cd /usr/local/var && wget https://saucelabs.com/downloads/sc-4.9.0-${SAUCE_OS:-linux}.tar.gz && \
-  tar -xvf sc-4.9.0-${SAUCE_OS:-linux}.tar.gz) || echo 'Not installing SC on prod or worker build'"
+RUN bash -c "set -x; [[ ${APP_ENVIRONMENT:-prod} = local-web ]] && (cd /usr/local/var && wget https://saucelabs.com/downloads/sc-4.9.1-${SAUCE_OS:-linux}.tar.gz && \
+  tar -xvf sc-4.9.1-${SAUCE_OS:-linux}.tar.gz) || echo 'Not installing SC on prod or worker build'"
 
 WORKDIR /code
 

diff --git a/INSTALL.rst b/INSTALL.rst
@@ -29,12 +29,12 @@ Running services locally
 Prerequisites
 =============
 
-HEPData runs with Python 3.7, 3.8 or 3.9. It also uses several services, which you will need to install before running HEPData.
+HEPData runs with Python 3.8 or 3.9. It also uses several services, which you will need to install before running HEPData.
 
 These services can be installed using the relevant package manager for your system,
 for example, using ``yum`` or ``apt-get`` for Linux or ``brew`` for macOS:
 
- * `PostgreSQL <http://www.postgresql.org/>`_ (version 12) database server
+ * `PostgreSQL <http://www.postgresql.org/>`_ (version 14) database server
  * `Redis <http://redis.io/>`_ for caching
  * `OpenSearch <https://opensearch.org/>`_ (version 2.7.0) for indexing and information retrieval. See below for further instructions.
  * `Node.js <https://nodejs.org>`_ (version 18) JavaScript run-time environment and its package manager `npm <https://www.npmjs.com/>`_.
@@ -84,7 +84,7 @@ Installation
 
 Python
 ------
-The HEPData code is only compatible with Python 3.7, 3.8 or 3.9 (not Python 2 or other 3.x versions).  We recommend Python 3.9.
+The HEPData code is only compatible with Python 3.8 or 3.9 (not Python 2 or other 3.x versions).  We recommend Python 3.9.
 
 First install all requirements in a Python virtual environment.
 (Use `virtualenv <https://virtualenv.pypa.io/en/stable/installation.html>`_ or
@@ -181,7 +181,7 @@ PostgreSQL
 
 See `YUM Installation <https://wiki.postgresql.org/wiki/YUM_Installation>`_ and
 `First steps <https://wiki.postgresql.org/wiki/First_steps>`_.  On Linux you might need ``sudo su - postgres`` before
-executing the steps below.  On macOS you can install with ``brew install postgresql@12``.
+executing the steps below.  On macOS you can install with ``brew install postgresql@14``.
 
 .. code-block:: console
 
@@ -224,9 +224,9 @@ Set email confirmation for the test user within the database.
    UPDATE 1
 
 If you're having problems with access permissions to the database (on Linux), a simple solution is to edit the
-PostgreSQL Client Authentication Configuration File (e.g. ``/var/lib/pgsql/12/data/pg_hba.conf``) to
+PostgreSQL Client Authentication Configuration File (e.g. ``/var/lib/pgsql/14/data/pg_hba.conf``) to
 ``trust`` local and IPv4/IPv6 connections (instead of ``peer`` or ``ident``), then restart the PostgreSQL
-server (e.g. ``sudo systemctl restart postgresql-12``).
+server (e.g. ``sudo systemctl restart postgresql-14``).
 
 Recreate the OpenSearch index
 -----------------------------
@@ -370,7 +370,7 @@ To run the tests:
 
 .. code-block:: console
 
-   $ docker-compose exec web bash -c "/usr/local/var/sc-4.9.0-${SAUCE_OS:-linux}/bin/sc -u $SAUCE_USERNAME -k $SAUCE_ACCESS_KEY --region eu-central & ./run-tests.sh"
+   $ docker-compose exec web bash -c "/usr/local/var/sc-4.9.1-${SAUCE_OS:-linux}/bin/sc -u $SAUCE_USERNAME -k $SAUCE_ACCESS_KEY --region eu-central & ./run-tests.sh"
 
 .. _docker-compose-tips:
 

diff --git a/docs/modules/records.rst b/docs/modules/records.rst
@@ -15,6 +15,7 @@ contains the JavaScript code that renders the tables and plots in a web browser
    hepdata.modules.records.subscribers.api
    hepdata.modules.records.subscribers.models
    hepdata.modules.records.subscribers.rest
+   hepdata.modules.records.utils.analyses
    hepdata.modules.records.utils.common
    hepdata.modules.records.utils.data_processing_utils
    hepdata.modules.records.utils.doi_minter
@@ -60,6 +61,11 @@ hepdata.modules.records.subscribers.rest
 
 .. automodule:: hepdata.modules.records.subscribers.rest
 
+hepdata.modules.records.utils.analyses
+--------------------------------------
+
+.. automodule:: hepdata.modules.records.utils.analyses
+
 hepdata.modules.records.utils.common
 ------------------------------------
 

diff --git a/hepdata/cli.py b/hepdata/cli.py
@@ -223,9 +223,10 @@ def do_unload(records_to_unload):
 
 @utils.command()
 @with_appcontext
-def find_and_add_record_analyses():
-    """Finds analyses such as Rivet and adds them to records."""
-    update_analyses()
+@click.option('--endpoint', '-e', type=str, help='Specific endpoint to update (e.g. "rivet" or "MadAnalysis"). Omit for all.')
+def find_and_add_record_analyses(endpoint):
+    """Finds analyses such as Rivet and MadAnalysis 5 and adds them to records."""
+    update_analyses(endpoint)
 
 
 @utils.command()

diff --git a/hepdata/config.py b/hepdata/config.py
@@ -313,11 +313,14 @@ def _(x):
         'endpoint_url': 'http://rivet.hepforge.org/analyses.json',
         'url_template': 'http://rivet.hepforge.org/analyses/{0}'
     },
+    'MadAnalysis': {
+        'endpoint_url': 'https://madanalysis.irmp.ucl.ac.be/raw-attachment/wiki/MA5SandBox/analyses.json',
+        'url_template': 'https://doi.org/{0}'
+    },
     #'ufo': {},
     #'xfitter': {},
     #'applgrid': {},
     #'fastnlo': {},
-    #'madanalysis': {},
 }
 
 HISTFACTORY_FILE_TYPE = 'HistFactory'

diff --git a/hepdata/ext/opensearch/document_enhancers.py b/hepdata/ext/opensearch/document_enhancers.py
@@ -91,7 +91,8 @@ def add_shortened_authors(doc):
 
 def add_analyses(doc):
     """
-    TODO: Generalise for badges other than Rivet
+    Add analyses links such as Rivet, MadAnalysis 5 and HistFactory to the index.
+
     :param doc:
     :return:
     """

diff --git a/hepdata/modules/records/assets/js/hepdata_common.js b/hepdata/modules/records/assets/js/hepdata_common.js
@@ -44,7 +44,7 @@ HEPDATA.file_type_to_details = {
   "bitbucket": {"icon": "bitbucket", "description": "Bitbucket Repository"},
   "fastnlo": {"icon": "area-chart", "description": "fastNLO Analysis"},
   "rivet": {"icon": "area-chart", "description": "Rivet Analysis"},
-  "madanalysis": {"icon": "area-chart", "description": "MadAnalysis5 Analysis"},
+  "madanalysis": {"icon": "area-chart", "description": "MadAnalysis 5 Analysis"},
   "xfitter": {"icon": "area-chart", "description": "xFitter Analysis"},
   "applgrid": {"icon": "area-chart", "description": "APPLgrid Analysis"},
   "ufo": {"icon": "rocket", "description": "Universal Feynrules Output (UFO)"},

diff --git a/hepdata/modules/records/templates/hepdata_records/components/resources-widget.html b/hepdata/modules/records/templates/hepdata_records/components/resources-widget.html
@@ -39,7 +39,7 @@ <h4>Add Resource for <span id="selected_resource_item">Submission</span></h4>
                                                 <select name="analysisType" id="analysisType" class="form-control">
                                                     <option disabled>Analyses</option>
                                                     <option value="applgrid">APPLgrid</option>
-                                                    <option value="madanalysis">MadAnalysis5</option>
+                                                    <option value="MadAnalysis">MadAnalysis 5</option>
                                                     <option value="rivet">Rivet</option>
                                                     <option value="fastnlo">fastNLO</option>
                                                     <option value="ufo">Universal Feynrules Output (UFO)</option>

diff --git a/hepdata/modules/records/templates/hepdata_records/publication_record.html b/hepdata/modules/records/templates/hepdata_records/publication_record.html
@@ -153,7 +153,7 @@
                             <ul>
                               {% for analysis in ctx.record.analyses %}
                                 <li><div class="analysis"><a href="{{ analysis.analysis }}" target="_blank"><span
-                                  class="fa fa-line-chart"></span> {{ analysis.type }}{% if analysis.type != config.HISTFACTORY_FILE_TYPE %} Analysis{% endif %}</a></div></li>
+                                  class="fa fa-line-chart"></span> {{ analysis.type }}{% if analysis.type == 'rivet' %} Analysis{% endif %}</a></div></li>
                               {% endfor %}
                             </ul>
                         </div>

diff --git a/hepdata/modules/records/templates/hepdata_records/related_record.html b/hepdata/modules/records/templates/hepdata_records/related_record.html
@@ -142,7 +142,7 @@
                         <ul>
                           {% for analysis in ctx.record.analyses %}
                             <li><div class="analysis"><a href="{{ analysis.analysis }}" target="_blank"><span
-                              class="fa fa-line-chart"></span> {{ analysis.type }}{% if analysis.type != config.HISTFACTORY_FILE_TYPE %} Analysis{% endif %}</a></div></li>
+                              class="fa fa-line-chart"></span> {{ analysis.type }}{% if analysis.type == 'rivet' %} Analysis{% endif %}</a></div></li>
                           {% endfor %}
                         </ul>
                     </div>

diff --git a/hepdata/modules/records/utils/analyses.py b/hepdata/modules/records/utils/analyses.py
@@ -38,11 +38,18 @@
 
 
 @shared_task
-def update_analyses():
-    """Update (Rivet) analyses and remove outdated resources."""
+def update_analyses(endpoint=None):
+    """
+    Update (Rivet and MadAnalysis 5) analyses and remove outdated resources.
+
+    :param endpoint: either "Rivet" or "MadAnalysis" or None (default) for both
+    """
     endpoints = current_app.config["ANALYSES_ENDPOINTS"]
     for analysis_endpoint in endpoints:
 
+        if endpoint and endpoint != analysis_endpoint:
+            continue
+
         if "endpoint_url" in endpoints[analysis_endpoint]:
 
             log.info("Updating analyses from {0}...".format(analysis_endpoint))

diff --git a/hepdata/modules/search/templates/hepdata_search/modals/search_help.html b/hepdata/modules/search/templates/hepdata_search/modals/search_help.html
@@ -183,6 +183,13 @@ <h4>Other useful searches</h4>
                                          (Rivet analysis)
                                      </span>
                                 </li>
+                                <li>
+                                    <a href='/search?q=analysis:MadAnalysis'
+                                       target="_new">analysis:MadAnalysis</a>
+                                    <span class="text-muted">
+                                       (MadAnalysis 5 analysis)
+                                    </span>
+                                </li>
                                 <li>
                                     <a href='/search?q=analysis:HistFactory'
                                        target="_new">analysis:HistFactory</a>

diff --git a/hepdata/modules/search/templates/hepdata_search/search_results.html b/hepdata/modules/search/templates/hepdata_search/search_results.html
@@ -101,7 +101,7 @@ <h4 class="record-header">
                                             <div class="analysis">
                                                 <a href="{{ analysis.analysis }}" target="_blank"><span
                                                         class="fa fa-line-chart"></span>
-                                                    {{ analysis.type }}{% if analysis.type != config.HISTFACTORY_FILE_TYPE %} Analysis{% endif %}
+                                                    {{ analysis.type }}{% if analysis.type == 'rivet' %} Analysis{% endif %}
                                                   </a>
                                             </div>
                                         {% endfor %}

diff --git a/hepdata/resilient_requests.py b/hepdata/resilient_requests.py
@@ -30,7 +30,7 @@ def send(self, request, **kwargs):
 retry_strategy = Retry(total=TOTAL_RETRIES,
                        backoff_factor=4,
                        status_forcelist=[429, 500, 502, 503, 504],
-                       method_whitelist=["GET", "POST"])
+                       allowed_methods=["GET", "POST"])
 adapter = TimeoutHTTPAdapter(max_retries=retry_strategy)
 
 

diff --git a/hepdata/version.py b/hepdata/version.py
@@ -28,4 +28,4 @@
 and parsed by ``setup.py``.
 """
 
-__version__ = "0.9.4dev20230914"
+__version__ = "0.9.4dev20231010"
diff --git a/requirements.txt b/requirements.txt
@@ -1,5 +1,5 @@
 beautifulsoup4==4.12.2
-bleach==6.0.0
+bleach==6.1.0
 datacite==1.1.3
 gunicorn==21.2.0
 hepdata-converter-ws-client==0.2.2
@@ -14,9 +14,9 @@ invenio-db[postgresql]==1.1.5
 invenio-logging[sentry-sdk]==1.3.2
 invenio-oauthclient==3.5.0
 invenio-pidstore==1.3.1
-invenio-records==2.2.1
+invenio-records==2.2.2
 invenio-search[opensearch2]==2.2.0
-invenio-theme==2.5.4
+invenio-theme==2.5.5
 invenio-userprofiles==2.3.0
 python-twitter-v2==0.8.1
 responses==0.23.1

diff --git a/setup.py b/setup.py
@@ -181,9 +181,10 @@ def run_tests(self):
         'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
         'Topic :: Software Development :: Libraries :: Python Modules',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Development Status :: Production',
     ],
     cmdclass={'test': PyTest},
-    python_requires='>=3.7, <3.10',
+    python_requires='>=3.8, <3.10',
 )
diff --git a/tests/records_test.py b/tests/records_test.py
@@ -1029,13 +1029,22 @@ def test_update_analyses(app):
     assert analysis_resources[0].file_location == 'http://rivet.hepforge.org/analyses#ATLAS_2012_I1203852'
 
     # Call update_analyses(): should add new resource and delete existing one
-    update_analyses()
+    update_analyses('rivet')
     analysis_resources = DataResource.query.filter_by(file_type='rivet').all()
     assert len(analysis_resources) == 1
     assert analysis_resources[0].file_location == 'http://rivet.hepforge.org/analyses/ATLAS_2012_I1203852'
 
     # Call update_analyses() again: should be no further changes (but covers more lines of code)
-    update_analyses()
+    update_analyses('rivet')
     analysis_resources = DataResource.query.filter_by(file_type='rivet').all()
     assert len(analysis_resources) == 1
     assert analysis_resources[0].file_location == 'http://rivet.hepforge.org/analyses/ATLAS_2012_I1203852'
+
+    # Import a record that has an associated MadAnalysis 5 analysis
+    import_records(['ins1811596'], synchronous=True)
+    analysis_resources = DataResource.query.filter_by(file_type='MadAnalysis').all()
+    assert len(analysis_resources) == 0
+    update_analyses('MadAnalysis')
+    analysis_resources = DataResource.query.filter_by(file_type='MadAnalysis').all()
+    assert len(analysis_resources) == 1
+    assert analysis_resources[0].file_location == 'https://doi.org/10.14428/DVN/I2CZWU'