Merge pull request #10 from HEPData/filter-files

Filter files before sending to converter

.github/workflows/ci.yml

@@ -10,16 +10,13 @@ jobs:
   test:
 
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [ '2.7', '3.7' ]
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
+    - name: Set up Python 3.6
       uses: actions/setup-python@v2
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: 3.6
     - name: Install pip dependencies
       run: |
         pip install --upgrade pip setuptools
@@ -33,9 +30,9 @@ jobs:
       run: |
         coverage run -m unittest discover hepdata_converter_ws_client/testsuite 'test_*'
     - name: Run coveralls
-      if: startsWith(matrix.python-version, '3')
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        COVERALLS_SERVICE_NAME: github
       run: |
         coveralls
 

hepdata_converter_ws_client/__init__.py

@@ -79,7 +79,15 @@ def convert(url, input, output=None, options={}, id=None, extract=True, timeout=
         assert os.path.exists(input)
 
         with tarfile.open(mode='w:gz', fileobj=input_stream) as tar:
-            tar.add(input, arcname=archive_name)
+            if os.path.isdir(input):
+                with os.scandir(input) as it:
+                    for entry in it:
+                        if entry.is_file():
+                            if os.path.splitext(entry.name)[1] in ['.yaml', '.json']:
+                                tar.add(entry.path, arcname=os.path.join(archive_name, entry.name))
+            else:
+                tar.add(input, arcname=archive_name)
+
     elif hasattr(input, 'read'):
         with tarfile.open(mode='w:gz', fileobj=input_stream) as tar:
             info = tarfile.TarInfo(archive_name)

hepdata_converter_ws_client/testsuite/__init__.py

@@ -56,6 +56,24 @@ def _inner(*args, **kwargs):
         return _inner
 
 
+class insert_data_as_extracted_dir(object):
+    def __init__(self, *sample_file_name):
+        self.sample_file_name = _parse_path_arguments(sample_file_name)
+        self.temp_path = tempfile.gettempdir()
+
+    def __call__(self, function):
+        def _inner(*args, **kwargs):
+            args = list(args)
+            with tempfile.TemporaryDirectory() as temp_dir:
+                shutil.unpack_archive(construct_testdata_path(self.sample_file_name), temp_dir)
+                # Assume zips consist of a single directory
+                unpacked_dir = os.path.join(temp_dir, os.listdir(temp_dir)[0])
+                args.append(unpacked_dir)
+                function(*args, **kwargs)
+
+        return _inner
+
+
 class insert_path(object):
     def __init__(self, *sample_file_name):
         self.sample_file_name = _parse_path_arguments(sample_file_name)

hepdata_converter_ws_client/testsuite/test_convert.py

@@ -4,7 +4,7 @@
 import os
 import requests
 
-from hepdata_converter_ws_client.testsuite import insert_path, insert_data_as_binary_file, TMPDirMixin, ExtendedTestCase
+from hepdata_converter_ws_client.testsuite import insert_path, insert_data_as_binary_file, insert_data_as_extracted_dir, TMPDirMixin, ExtendedTestCase
 from hepdata_converter_ws_client import ARCHIVE_NAME
 import hepdata_converter_ws_client
 
@@ -37,6 +37,15 @@ def test_convert_fileobj(self, oldhepdata_file, oldhepdata_yaml_path):
 
         self.assertDirsEqual(oldhepdata_yaml_path, path)
 
+    @insert_data_as_extracted_dir('testsubmission/TestHEPSubmission.zip')
+    @insert_path('testsubmission/yaml')
+    def test_convert_zip_with_resources(self, testsubmission_file, testsubmission_yaml_path):
+        # test zip
+        path = os.path.join(self.current_tmp, 'yaml')
+        hepdata_converter_ws_client.convert(self.get_server_url(), testsubmission_file, path,
+                                            options={'input_format': 'yaml'})
+        self.assertDirsEqual(testsubmission_yaml_path, path)
+
     @insert_path('oldhepdata/invalidsample.oldhepdata')
     def test_convert_invalid(self, oldhepdata_path):
         # test paths

hepdata_converter_ws_client/testsuite/testdata/testsubmission/yaml/data1.yaml

@@ -0,0 +1,37 @@
+dependent_variables:
+- header: {name: SIG(fiducial), units: FB}
+  qualifiers:
+  - {name: RE, value: P P --> Z0 < LEPTON+ LEPTON- > Z0 < LEPTON+ LEPTON- > X}
+  - {name: SQRT(S), units: GEV, value: 7000}
+  values:
+  - errors:
+    - asymerror: {minus: -3.0, plus: 3.3}
+      label: stat
+    - asymerror: {minus: -1.0, plus: 1.2}
+      label: sys
+    - {label: 'sys,lumi', symerror: 1.0}
+    value: 25.4
+- header: {name: SIG(fiducial), units: FB}
+  qualifiers:
+  - {name: RE, value: P P --> Z0 < LEPTON+ LEPTON- > Z0* < LEPTON+ LEPTON- > X}
+  - {name: SQRT(S), units: GEV, value: 7000}
+  values:
+  - errors:
+    - asymerror: {minus: -3.5, plus: 3.8}
+      label: stat
+    - asymerror: {minus: -1.5, plus: 1.7}
+      label: sys
+    - {label: 'sys,lumi', symerror: 1.2}
+    value: 29.8
+- header: {name: SIG(fiducial), units: FB}
+  qualifiers:
+  - {name: RE, value: P P --> Z0 < LEPTON+ LEPTON- > Z0 < NU NUBAR > X}
+  - {name: SQRT(S), units: GEV, value: 7000}
+  values:
+  - errors:
+    - asymerror: {minus: -2.9, plus: 3.1}
+      label: stat
+    - {label: sys, symerror: 1.7}
+    - {label: 'sys,lumi', symerror: 0.5}
+    value: 12.7
+independent_variables: []

hepdata_converter_ws_client/testsuite/testdata/testsubmission/yaml/data2.yaml

@@ -0,0 +1,13 @@
+dependent_variables:
+- header: {name: SIG(total), units: FB}
+  qualifiers:
+  - {name: RE, value: P P --> Z0 Z0 X}
+  - {name: SQRT(S), units: GEV, value: 7000}
+  values:
+  - errors:
+    - {label: stat, symerror: 0.7}
+    - asymerror: {minus: -0.3, plus: 0.4}
+      label: sys
+    - {label: 'sys,lumi', symerror: 0.3}
+    value: 6.7
+independent_variables: []

hepdata_converter_ws_client/testsuite/testdata/testsubmission/yaml/data3.yaml

@@ -0,0 +1,33 @@
+dependent_variables:
+- header: {name: 10**6 * 1/SIG(fiducial) * D(SIG(fiducial))/DPT, units: GEV**-1}
+  qualifiers:
+  - {name: RE, value: P P --> Z0 < LEPTON+ LEPTON- > Z0 < LEPTON+ LEPTON- > X}
+  - {name: SQRT(S), units: GEV, value: 7000}
+  values:
+  - errors:
+    - {label: stat, symerror: 1100}
+    - {label: 'sys,detector', symerror: 79}
+    - {label: 'sys,background', symerror: 15}
+    value: 7000
+  - errors:
+    - {label: stat, symerror: 1600}
+    - {label: 'sys,detector', symerror: 75}
+    - {label: 'sys,background', symerror: 15}
+    value: 9800
+  - errors:
+    - {label: stat, symerror: 490}
+    - {label: 'sys,detector', symerror: 41}
+    - {label: 'sys,background', symerror: 2}
+    value: 1600
+  - errors:
+    - {label: stat, symerror: 60}
+    - {label: 'sys,detector', symerror: 2}
+    - {label: 'sys,background', symerror: 0}
+    value: 80
+independent_variables:
+- header: {name: Leading dilepton PT, units: GEV}
+  values:
+  - {high: 60, low: 0}
+  - {high: 100, low: 60}
+  - {high: 200, low: 100}
+  - {high: 600, low: 200}

hepdata_converter_ws_client/testsuite/testdata/testsubmission/yaml/data4.yaml

@@ -0,0 +1,27 @@
+dependent_variables:
+- header: {name: 10**6 * 1/SIG(fiducial) * D(SIG(fiducial))/DPT, units: GEV**-1}
+  qualifiers:
+  - {name: RE, value: P P --> Z0 < LEPTON+ LEPTON- > Z0 < NU NUBAR > X}
+  - {name: SQRT(S), units: GEV, value: 7000}
+  values:
+  - errors:
+    - {label: stat, symerror: 3340}
+    - {label: 'sys,detector', symerror: 80}
+    - {label: 'sys,background', symerror: 740}
+    value: 9930
+  - errors:
+    - {label: stat, symerror: 3210}
+    - {label: 'sys,detector', symerror: 200}
+    - {label: 'sys,background', symerror: 260}
+    value: 8280
+  - errors:
+    - {label: stat, symerror: 1490}
+    - {label: 'sys,detector', symerror: 120}
+    - {label: 'sys,background', symerror: 390}
+    value: 3900
+independent_variables:
+- header: {name: Leading dilepton PT, units: GEV}
+  values:
+  - {high: 90, low: 50}
+  - {high: 130, low: 90}
+  - {high: 200, low: 130}

hepdata_converter_ws_client/testsuite/testdata/testsubmission/yaml/data5.yaml

@@ -0,0 +1,33 @@
+dependent_variables:
+- header: {name: '10**6 * 1/SIG(fiducial) * D(SIG(fiducial))/DDELTA(PHI(LEPTON+,LEPTON-))'}
+  qualifiers:
+  - {name: RE, value: P P --> Z0 < LEPTON+ LEPTON- > Z0 < LEPTON+ LEPTON- > X}
+  - {name: SQRT(S), units: GEV, value: 7000}
+  values:
+  - errors:
+    - {label: stat, symerror: 69000}
+    - {label: 'sys,detector', symerror: 6600}
+    - {label: 'sys,background', symerror: 10}
+    value: 130000
+  - errors:
+    - {label: stat, symerror: 100000}
+    - {label: 'sys,detector', symerror: 9000}
+    - {label: 'sys,background', symerror: 400}
+    value: 280000
+  - errors:
+    - {label: stat, symerror: 80000}
+    - {label: 'sys,detector', symerror: 10000}
+    - {label: 'sys,background', symerror: 300}
+    value: 260000
+  - errors:
+    - {label: stat, symerror: 50000}
+    - {label: 'sys,detector', symerror: 2000}
+    - {label: 'sys,background', symerror: 300}
+    value: 420000
+independent_variables:
+- header: {name: 'Leading dilepton DELTA(PHI(LEPTON+, LEPTON-))', units: GEV}
+  values:
+  - {high: 0.5, low: 0.0}
+  - {high: 1.0, low: 0.5}
+  - {high: 1.7, low: 1.0}
+  - {high: 3.14159, low: 1.7}

hepdata_converter_ws_client/testsuite/testdata/testsubmission/yaml/data6.yaml

@@ -0,0 +1,27 @@
+dependent_variables:
+- header: {name: '10**6 * 1/SIG(fiducial) * D(SIG(fiducial))/DDELTA(PHI(LEPTON+,LEPTON-))'}
+  qualifiers:
+  - {name: RE, value: P P --> Z0 < LEPTON+ LEPTON- > Z0 < NU NUBAR > X}
+  - {name: SQRT(S), units: GEV, value: 7000}
+  values:
+  - errors:
+    - {label: stat, symerror: 158000}
+    - {label: 'sys,detector', symerror: 4000}
+    - {label: 'sys,background', symerror: 19000}
+    value: 346000
+  - errors:
+    - {label: stat, symerror: 91000}
+    - {label: 'sys,detector', symerror: 4000}
+    - {label: 'sys,background', symerror: 71000}
+    value: 569000
+  - errors:
+    - {label: stat, symerror: 70000}
+    - {label: 'sys,detector', symerror: 2000}
+    - {label: 'sys,background', symerror: 58000}
+    value: 100000
+independent_variables:
+- header: {name: 'Leading dilepton DELTA(PHI(LEPTON+,LEPTON-))', units: GEV}
+  values:
+  - {high: 0.5, low: 0.0}
+  - {high: 1.7, low: 0.5}
+  - {high: 3.14159, low: 1.7}

hepdata_converter_ws_client/testsuite/testdata/testsubmission/yaml/data7.yaml

@@ -0,0 +1,33 @@
+dependent_variables:
+- header: {name: 10**6 * 1/SIG(fiducial) * D(SIG(fiducial))/DM(ZZ)}
+  qualifiers:
+  - {name: RE, value: P P --> Z0 < LEPTON+ LEPTON- > Z0 < LEPTON+ LEPTON- > X}
+  - {name: SQRT(S), units: GeV, value: 7000}
+  values:
+  - errors:
+    - {label: stat, symerror: 300}
+    - {label: 'sys,detector', symerror: 40}
+    - {label: 'sys,background', symerror: 2}
+    value: 2200
+  - errors:
+    - {label: stat, symerror: 1000}
+    - {label: 'sys,detector', symerror: 100}
+    - {label: 'sys,background', symerror: 5}
+    value: 4500
+  - errors:
+    - {label: stat, symerror: 400}
+    - {label: 'sys,detector', symerror: 20}
+    - {label: 'sys,background', symerror: 2}
+    value: 1000
+  - errors:
+    - {label: stat, symerror: 100}
+    - {label: 'sys,detector', symerror: 10}
+    - {label: 'sys,background', symerror: 1}
+    value: 280
+independent_variables:
+- header: {name: M(ZZ), units: GEV}
+  values:
+  - {high: 240, low: 0}
+  - {high: 300, low: 240}
+  - {high: 400, low: 300}
+  - {high: 800, low: 400}

hepdata_converter_ws_client/testsuite/testdata/testsubmission/yaml/data8.yaml

@@ -0,0 +1,27 @@
+dependent_variables:
+- header: {name: 10**6 * 1/SIG(fiducial) * D(SIG(fiducial))/DMT(ZZ), units: GEV**-1}
+  qualifiers:
+  - {name: RE, value: P P --> Z0 < LEPTON+ LEPTON- > Z0 < NU NUBAR > X}
+  - {name: SQRT(S), units: GEV, value: 7000}
+  values:
+  - errors:
+    - {label: stat, symerror: 4400}
+    - {label: 'sys,detector', symerror: 300}
+    - {label: 'sys,background', symerror: 1900}
+    value: 10500
+  - errors:
+    - {label: stat, symerror: 2630}
+    - {label: 'sys,detector', symerror: 230}
+    - {label: 'sys,background', symerror: 280}
+    value: 6320
+  - errors:
+    - {label: stat, symerror: 1210}
+    - {label: 'sys,detector', symerror: 60}
+    - {label: 'sys,background', symerror: 480}
+    value: 3680
+independent_variables:
+- header: {name: MT(ZZ), units: GEV}
+  values:
+  - {high: 250, low: 220}
+  - {high: 300, low: 250}
+  - {high: 400, low: 300}