Merge pull request #53 from 20DM/yoda2_writer

YODA2 writer

.github/workflows/ci.yml

@@ -3,7 +3,7 @@ name: Continuous Integration
 on:
   push:
   pull_request:
-    branches: [ master ]
+    branches: [ main ]
   release:
     types: [ published ]
 
@@ -14,22 +14,21 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
       with:
-        python-version: 3.8
+        python-version: '3.10'
     - name: Install pip dependencies
       run: |
         pip install --upgrade pip
-        pip install --ignore-installed coveralls
+        pip install coveralls
     - name: Run tests
       env:
-        DOCKER_IMAGE: hepdata/hepdata-converter:0.2.0
+        DOCKER_IMAGE: hepdata/hepdata-converter:latest
       run: |
         export CURRENT_PATH=`pwd`
-        docker run -v $CURRENT_PATH:$CURRENT_PATH $DOCKER_IMAGE /bin/bash -c "cd $CURRENT_PATH && coverage run -m unittest discover hepdata_converter/testsuite 'test_*'"
-        docker run -v $CURRENT_PATH:$CURRENT_PATH $DOCKER_IMAGE /bin/bash -c "cd $CURRENT_PATH && python3 setup.py install && hepdata-converter -v"
+        docker run -v $CURRENT_PATH:$CURRENT_PATH $DOCKER_IMAGE /bin/bash -c "cd $CURRENT_PATH && pip install -e '.[tests]' && hepdata-converter -v && coverage run -m unittest discover hepdata_converter/testsuite 'test_*'"
     - name: Run coveralls
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -42,14 +41,14 @@ jobs:
     if: github.event_name == 'release'
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Build PyPI package
       env:
-        DOCKER_IMAGE: hepdata/hepdata-converter:0.2.0
+        DOCKER_IMAGE: hepdata/hepdata-converter:latest
       run: |
         export CURRENT_PATH=`pwd`
-        docker run -v $CURRENT_PATH:$CURRENT_PATH $DOCKER_IMAGE /bin/bash -c "cd $CURRENT_PATH && pip3 install wheel"
-        docker run -v $CURRENT_PATH:$CURRENT_PATH $DOCKER_IMAGE /bin/bash -c "cd $CURRENT_PATH && python3 setup.py sdist bdist_wheel"
+        docker run -v $CURRENT_PATH:$CURRENT_PATH $DOCKER_IMAGE /bin/bash -c "cd $CURRENT_PATH && pip install wheel"
+        docker run -v $CURRENT_PATH:$CURRENT_PATH $DOCKER_IMAGE /bin/bash -c "cd $CURRENT_PATH && python setup.py sdist bdist_wheel"
     - name: Publish distribution to PyPI
       uses: pypa/gh-action-pypi-publish@master
       with:

.gitignore

@@ -58,4 +58,7 @@ README.html
 target/
 
 .idea
-MANIFEST
+MANIFEST
+
+# venv
+venv/

.readthedocs.yml

@@ -5,18 +5,27 @@
 # Required
 version: 2
 
-# Build documentation in the docs/ directory with Sphinx
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+
+# Build documentation in the "docs/" directory with Sphinx
 sphinx:
   configuration: docs/conf.py
+  # Fail on all warnings to avoid broken references
+  fail_on_warning: true
 
 # Optionally build your docs in additional formats such as PDF and ePub
 formats: all
 
-# Optionally set the version of Python and requirements required to build your docs
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
 python:
-  version: 3.7
   install:
-    - method: pip
-      path: .
-      extra_requirements:
-        - docs
+  - method: pip
+    path: .
+    extra_requirements:
+    - docs

README.rst

@@ -3,16 +3,16 @@ HEPData Converter
 =================
 
 
-.. image:: https://github.com/HEPData/hepdata-converter/workflows/Continuous%20Integration/badge.svg?branch=master
-   :target: https://github.com/HEPData/hepdata-converter/actions?query=branch%3Amaster
+.. image:: https://github.com/HEPData/hepdata-converter/actions/workflows/ci.yml/badge.svg?branch=main
+   :target: https://github.com/HEPData/hepdata-converter/actions?query=branch%3Amain
    :alt: GitHub Actions Build Status
 
-.. image:: https://coveralls.io/repos/github/HEPData/hepdata-converter/badge.svg?branch=master
-   :target: https://coveralls.io/github/HEPData/hepdata-converter?branch=master
+.. image:: https://coveralls.io/repos/github/HEPData/hepdata-converter/badge.svg?branch=main
+   :target: https://coveralls.io/github/HEPData/hepdata-converter?branch=main
    :alt: Coveralls Status
 
 .. image:: https://img.shields.io/github/license/HEPData/hepdata-converter.svg
-   :target: https://github.com/HEPData/hepdata-converter/blob/master/LICENSE.txt
+   :target: https://github.com/HEPData/hepdata-converter/blob/main/LICENSE.txt
    :alt: License
 
 .. image:: https://img.shields.io/github/release/hepdata/hepdata-converter.svg?maxAge=2592000
@@ -34,11 +34,11 @@ HEPData Converter
 
 This Python 3 library provides support for converting:
 
-* Old HepData input format (`sample <https://github.com/HEPData/hepdata-submission/blob/master/examples/oldhepdata/sample.oldhepdata>`_) to `YAML <https://github.com/HEPData/hepdata-submission>`_
+* Old HepData input format (`sample <https://github.com/HEPData/hepdata-submission/blob/main/examples/oldhepdata/sample.oldhepdata>`_) to `YAML <https://github.com/HEPData/hepdata-submission>`_
 * `YAML <https://github.com/HEPData/hepdata-submission>`_ to:
 
-  * `ROOT <https://root.cern.ch>`_ (tested with v6.22/02)
-  * `YODA <https://yoda.hepforge.org>`_ (tested with v1.8.3)
+  * `ROOT <https://root.cern.ch>`_ (tested with v6.28/04)
+  * `YODA <https://yoda.hepforge.org>`_ (tested with v2.0.0alpha)
   * `CSV <https://en.wikipedia.org/wiki/Comma-separated_values>`_
 
-Documentation: http://hepdata-converter.readthedocs.io
+Documentation: https://hepdata-converter.readthedocs.io

docs/Makefile

@@ -2,7 +2,7 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
+SPHINXOPTS    = -W --keep-going
 SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = _build

docs/conf.py

@@ -74,7 +74,7 @@ def __getattr__(cls, name):
 master_doc = 'index'
 
 # General information about the project.
-project = u''
+project = u'HEPData Converter'
 copyright = u'2016, CERN'
 author = u'CERN'
 
@@ -98,7 +98,7 @@ def __getattr__(cls, name):
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -167,7 +167,7 @@ def __getattr__(cls, name):
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
-#html_title = None
+html_title = f"{project} v{version} documentation"
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
 #html_short_title = None
@@ -340,4 +340,4 @@ def __getattr__(cls, name):
 
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'https://docs.python.org/2/': None}
+intersphinx_mapping = {'python': ('https://docs.python.org/3/', None)}

docs/installation.rst

@@ -33,12 +33,13 @@ alternative output formats via the web interface.
 
 To install this package locally, you first need to install `YODA <https://yoda.hepforge.org>`_ and
 `ROOT <https://root.cern.ch>`_ (including `PyROOT <https://root.cern/manual/python/>`_).  Check that you can
-``import yoda`` and ``import ROOT`` from Python.  You might want to install into a dedicated virtual
-environment:
+``import yoda`` and ``import ROOT`` from Python.  You might want to install into a dedicated `virtual environment
+<https://docs.python.org/3/tutorial/venv.html>`_:
 
 .. code-block:: console
 
-    $ mkvirtualenv hepdata-converter
+    $ python3 -m venv hepdata-converter
+    $ source hepdata-converter/bin/activate
     (hepdata-converter)$ pip install hepdata-converter
 
 This will install the latest released version from `PyPI <https://pypi.python.org/pypi/hepdata-converter>`_.
@@ -52,17 +53,30 @@ Developers might want to instead install the project directly from
 
 .. code-block:: console
 
-    $ workon hepdata-converter
-    (hepdata-converter)$ git clone https://github.com/HEPData/hepdata-converter
-    (hepdata-converter)$ cd hepdata-converter
-    (hepdata-converter)$ pip install -e .
+    $ git clone https://github.com/HEPData/hepdata-converter
+    $ cd hepdata-converter
+    $ python3 -m venv venv
+    $ source venv/bin/activate
+    (venv)$ pip install -e '.[tests]'
 
 Developers can then run the tests with the following command:
 
 .. code:: bash
 
     python -m unittest discover hepdata_converter/testsuite 'test_*'
 
+The documentation can be built locally in the virtual environment using Sphinx:
+
+.. code:: console
+
+    (venv)$ pip install -e '.[docs]'
+    (venv)$ cd docs
+    (venv)$ make html
+
+Then view the output by opening a web browser at ``_build/html/index.html``.
+Developers should also check that they can successfully build other formats using ``make latexpdf`` and ``make epub``.
+All three formats will be built by *Read the Docs* for the ``main`` branch on GitHub.
+
 
 Docker
 ------
@@ -80,9 +94,9 @@ The ``hepdata-converter`` package can be installed inside the Docker container:
 
 .. code-block:: console
 
-    root@617be04cbab5:/# pip3 install --ignore-installed hepdata-converter
+    root@617be04cbab5:/# pip install hepdata-converter
     root@617be04cbab5:/# hepdata-converter -h
-    root@617be04cbab5:/# python3 -c 'import hepdata_converter'
+    root@617be04cbab5:/# python -c 'import hepdata_converter'
 
 Note that the Docker container will be automatically removed when it exits (if running with the ``--rm`` option).  The
 Python module or CLI can then be used as described in :doc:`Usage <usage>`.  Input and output files can be moved
@@ -105,7 +119,7 @@ mount the current directory of the local filesystem when running the Docker cont
     $ git clone https://github.com/HEPData/hepdata-converter
     $ cd hepdata-converter
     $ docker run -v $PWD:$PWD -w $PWD --rm -it hepdata/hepdata-converter /bin/bash
-    root@2c22e88402d2:/hepdata-converter# python3 -m unittest discover hepdata_converter/testsuite 'test_*'
-    root@2c22e88402d2:/hepdata-converter# pip3 install -e .
+    root@2c22e88402d2:/hepdata-converter# pip install -e '.[tests]'
     root@2c22e88402d2:/hepdata-converter# hepdata-converter -h
-    root@2c22e88402d2:/hepdata-converter# python3 -c 'import hepdata_converter'
+    root@2c22e88402d2:/hepdata-converter# python -c 'import hepdata_converter'
+    root@2c22e88402d2:/hepdata-converter# python -m unittest discover hepdata_converter/testsuite 'test_*'

docs/usage.rst

@@ -26,7 +26,7 @@ Usage
 =====
 
 The library exposes a single function ``convert`` which enables conversion from different input formats
-(``oldhepdata``, ``yaml``) to different output formats (``csv``, ``root``, ``yaml``, ``yoda``), by using a simple in-memory
+(``oldhepdata``, ``yaml``) to different output formats (``csv``, ``root``, ``yaml``, ``yoda``, ``yoda1``), by using a simple in-memory
 intermediary format.
 
 

hepdata_converter/__init__.py

@@ -9,7 +9,7 @@
 
 def convert(input, output=None, options={}):
     """Converts a supported ``input_format`` (*oldhepdata*, *yaml*)
-    to a supported ``output_format`` (*csv*, *root*, *yaml*, *yoda*).
+    to a supported ``output_format`` (*csv*, *root*, *yaml*, *yoda*, *yoda1*).
 
     :param input: location of input file for *oldhepdata* format or input directory for *yaml* format
     :param output: location of output directory to which converted files will be written

hepdata_converter/testsuite/test_yoda1writer.py

@@ -0,0 +1,34 @@
+# -*- encoding: utf-8 -*-
+import os
+import hepdata_converter
+from hepdata_converter.testsuite import insert_data_as_file, insert_path, insert_paths
+from hepdata_converter.testsuite.test_writer import WriterTestSuite
+
+__author__ = 'Michał Szostak'
+
+
+class YODA1WriterTestSuite(WriterTestSuite):
+    @insert_path('yaml_full')
+    @insert_data_as_file('yoda1/full.yoda')
+    def test_simple_parse(self, yaml_simple_path, yoda_template):
+        output_file_path = os.path.join(self.current_tmp, 'datafile.yoda')
+        hepdata_converter.convert(yaml_simple_path, output_file_path,
+                                  options={'output_format': 'yoda1',
+                                           'validator_schema_version': '0.1.0',
+                                           'hepdata_doi': '10.17182/hepdata.62535.v1',
+                                           'rivet_analysis_name': 'ATLAS_2012_I1203852'})
+
+        self.assertNotEqual(os.stat(output_file_path).st_size, 0, 'output yoda file is empty')
+        with open(output_file_path, 'r') as f:
+            self.assertMultiLineAlmostEqual(f, yoda_template)
+
+    @insert_paths('yaml/ins1283183', 'yaml/ins1397637', 'yaml/ins699647', 'yaml/ins1413748')
+    def test_parse_all(self, test_submissions):
+
+        for idx, test_submission in enumerate(test_submissions):
+            output_file_path = os.path.join(self.current_tmp, 'data-{}.yoda'.format(idx))
+
+            hepdata_converter.convert(test_submission, output_file_path,
+                                      options={'output_format': 'yoda1'})
+
+            self.assertNotEqual(os.stat(output_file_path).st_size, 0, 'output yoda file is empty')

hepdata_converter/testsuite/test_yodawriter.py

@@ -32,3 +32,47 @@ def test_parse_all(self, test_submissions):
                                       options={'output_format': 'yoda'})
 
             self.assertNotEqual(os.stat(output_file_path).st_size, 0, 'output yoda file is empty')
+
+    @insert_path('yaml_full')
+    @insert_data_as_file('yoda/full_match.yoda')
+    def test_parse_pattern_match(self, yaml_simple_path, yoda_template):
+        output_file_path = os.path.join(self.current_tmp, 'datafile.yoda')
+        hepdata_converter.convert(yaml_simple_path, output_file_path,
+                                  options={'output_format': 'yoda',
+                                           'validator_schema_version': '0.1.0',
+                                           'hepdata_doi': '10.17182/hepdata.62535.v1',
+                                           'rivet_analysis_name': 'ATLAS_2012_I1203852',
+                                           'rivet_ref_match': 'd01'})
+
+        self.assertNotEqual(os.stat(output_file_path).st_size, 0, 'output yoda file is empty')
+        with open(output_file_path, 'r') as f:
+            self.assertMultiLineAlmostEqual(f, yoda_template)
+
+    @insert_path('yaml_full')
+    @insert_data_as_file('yoda/full_unmatch.yoda')
+    def test_parse_pattern_unmatch(self, yaml_simple_path, yoda_template):
+        output_file_path = os.path.join(self.current_tmp, 'datafile.yoda')
+        hepdata_converter.convert(yaml_simple_path, output_file_path,
+                                  options={'output_format': 'yoda',
+                                           'validator_schema_version': '0.1.0',
+                                           'hepdata_doi': '10.17182/hepdata.62535.v1',
+                                           'rivet_analysis_name': 'ATLAS_2012_I1203852',
+                                           'rivet_ref_unmatch': 'd07'})
+
+        self.assertNotEqual(os.stat(output_file_path).st_size, 0, 'output yoda file is empty')
+        with open(output_file_path, 'r') as f:
+            self.assertMultiLineAlmostEqual(f, yoda_template)
+
+    @insert_path('yaml_no_independent')
+    @insert_data_as_file('yoda/no_independent.yoda')
+    def test_parse_no_dependent(self, yaml_simple_path, yoda_template):
+        output_file_path = os.path.join(self.current_tmp, 'datafile.yoda')
+        hepdata_converter.convert(yaml_simple_path, output_file_path,
+                                  options={'output_format': 'yoda',
+                                           'validator_schema_version': '0.1.0',
+                                           'rivet_analysis_name': 'ATLAS_2021_I1887997'})
+
+        self.assertNotEqual(os.stat(output_file_path).st_size, 0, 'output yoda file is empty')
+        with open(output_file_path, 'r') as f:
+            self.assertMultiLineAlmostEqual(f, yoda_template)
+