Merge pull request #19 from ResearchObject/setup-poetry

Set up package with poetry

.github/workflows/lint.yml

@@ -21,17 +21,19 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.10"
+      - name: Install poetry
+        run: |
+          sudo apt install -y pipx
+          pipx install poetry
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+          poetry install
       - name: Lint with flake8
         run: |
           # stop the build if there are Python syntax errors or names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          poetry run flake8 ./src/rocrate_inveniordm --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+          poetry run flake8 ./src/rocrate_inveniordm --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
       - name: Lint with black
         run: |
-          black --check --diff .
+          poetry run black --check --diff .

.github/workflows/test.yml

@@ -25,11 +25,13 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Install poetry
+      run: |
+        sudo apt install -y pipx
+        pipx install poetry
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-        if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+        poetry install
     - name: Set up credentials
       run: |
         echo "INVENIORDM_BASE_URL=\"https://sandbox.zenodo.org/\"" >> .env
@@ -39,10 +41,10 @@ jobs:
         ZENODO_SANDBOX_API_KEY : ${{secrets.ZENODO_SANDBOX_API_KEY}}
     - name: Lint with mypy
       run: |
-        mypy . --ignore-missing-imports
+        poetry run mypy . --ignore-missing-imports
     - name: Test with pytest
       run: |
-        pytest
+        poetry run pytest
     - uses: actions/upload-artifact@v4
       with:
         name: test-outputs-${{ matrix.os }}-python${{ matrix.python-version }}

.gitignore

@@ -13,3 +13,9 @@ mapping/__pycache__
 
 # test outputs
 test/output
+.coverage
+/datacite-out.json
+/test-ro-crate.zip
+
+# build outputs
+dist/

pyproject.toml

@@ -0,0 +1,47 @@
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+# Poetry
+
+[tool.poetry]
+name = "rocrate-inveniordm"
+version = "2.0.0"
+description = "Upload RO-Crates to InvenioRDM and automatically fill the metadata. Supports any InvenioRDM instance, including Zenodo."
+authors = [
+    "Philipp Beer",
+    "Milan Szente",
+    "Eli Chadwick <[email protected]>",
+]
+maintainers = ["Eli Chadwick <[email protected]>"]
+license = "MIT"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.8.1"
+requests = "^2.31.0"
+python-dateutil = "2.9.0"
+python-iso639 = "^2023.6.15"
+pytz = "^2024.1"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.2.2"
+pytest-cov = "^5.0.0"
+pytest-dotenv = "^0.5.2"
+black = "^24.4.2"
+flake8 = "^7.1.0"
+mypy = "^1.10.1"
+types-requests = "^2.32.0.20240622"
+types-pytz = "^2024.1.0.20240417"
+types-python-dateutil = "^2.9.0.20240316"
+
+[tool.poetry.scripts]
+rocrate_inveniordm = "rocrate_inveniordm.deposit:main"
+
+# Testing tools
+
+## Pytest
+
+[tool.pytest.ini_options]
+pythonpath = "src"
+addopts = "--cov=rocrate_inveniordm"

deposit.py → src/rocrate_inveniordm/deposit.py

@@ -14,8 +14,8 @@
 import shutil
 import sys
 
-import mapping.converter as converter
-import upload.uploader as uploader
+import rocrate_inveniordm.mapping.converter as converter
+import rocrate_inveniordm.upload.uploader as uploader
 
 
 def main():
@@ -164,7 +164,7 @@ def deposit(
 
     # Upload and publish files, depending on no_upload and publish options
     if no_upload:
-        print(f"Created datacite-out.json, skipping upload.")
+        print("Created datacite-out.json, skipping upload.")
         return None
     else:
         record_id = uploader.deposit(data_cite_metadata, all_files, publish=publish)

mapping/converter.py → src/rocrate_inveniordm/mapping/converter.py

@@ -1,8 +1,10 @@
 import json
 import sys
+from importlib import resources
+import rocrate_inveniordm.mapping as mapping
 
-import mapping.condition_functions as cf
-import mapping.processing_functions as pf
+import rocrate_inveniordm.mapping.condition_functions as cf
+import rocrate_inveniordm.mapping.processing_functions as pf
 
 
 def main():
@@ -39,15 +41,22 @@ def get_arrays_from_from_values(input_list):
     return output_list
 
 
+def load_mapping_json() -> dict:
+    mapping_file = resources.files(mapping) / "mapping.json"
+    with mapping_file.open("r") as f:
+        mapping_json = json.load(f)
+    return mapping_json
+
+
 def convert(rc, metadata_only=False):
     """
     Convert a RO-Crate to a DataCite object
 
     :param rc: The RO-Crate
     :param metadata_only: Whether it is a metadata-only DataCite
     """
-    mapping_file = open("mapping/mapping.json")
-    m = json.load(mapping_file)
+
+    m = load_mapping_json()
 
     dc = setup_dc()
     if metadata_only:

upload/uploader.py → src/rocrate_inveniordm/upload/uploader.py

@@ -11,7 +11,7 @@
 import sys
 
 import requests
-import upload.credentials as credentials
+import rocrate_inveniordm.upload.credentials as credentials
 
 api_url = credentials.get_repository_base_url()
 

test/integration/test_integration.py

@@ -34,7 +34,7 @@ def test_created_datacite_files(crate_name):
     # note - check_output raises CalledProcessError if exit code is non-zero
     # --no-upload prevents upload and generates DataCite files only
     log = subprocess.check_output(
-        f"python deposit.py {crate_path} --no-upload", shell=True, text=True
+        f"rocrate_inveniordm {crate_path} --no-upload", shell=True, text=True
     )
     # preserve data files and log in TEST_OUTPUT_FOLDER
     shutil.copyfile(
@@ -67,7 +67,7 @@ def test_created_invenio_records(crate_name):
     # Act
     # note - check_output raises CalledProcessError if exit code is non-zero
     log = subprocess.check_output(
-        f"python deposit.py {crate_path}", shell=True, text=True
+        f"rocrate_inveniordm {crate_path}", shell=True, text=True
     )
     with open(
         os.path.join(TEST_OUTPUT_FOLDER, f"log-upload-{crate_name}.txt"),
@@ -122,7 +122,7 @@ def test_cli__zip():
     # Act
     # note - check_output raises CalledProcessError if exit code is non-zero
     log = subprocess.check_output(
-        f"python deposit.py {crate_path} -z", shell=True, text=True
+        f"rocrate_inveniordm {crate_path} -z", shell=True, text=True
     )
     match = re.search(expected_log_pattern_2, log, flags=re.MULTILINE)
     record_id = match.group("id")
@@ -159,7 +159,7 @@ def test_cli__datacite():
     # Act
     # note - check_output raises CalledProcessError if exit code is non-zero
     log = subprocess.check_output(
-        f"python deposit.py {crate_path} -d {compare_path}", shell=True, text=True
+        f"rocrate_inveniordm {crate_path} -d {compare_path}", shell=True, text=True
     )
     match = re.search(expected_log_pattern_2, log, flags=re.MULTILINE)
     record_id = match.group("id")
@@ -186,7 +186,7 @@ def test_cli__omit_roc_files():
     # Act
     # note - check_output raises CalledProcessError if exit code is non-zero
     log = subprocess.check_output(
-        f"python deposit.py {crate_path} -o", shell=True, text=True
+        f"rocrate_inveniordm {crate_path} -o", shell=True, text=True
     )
     match = re.search(expected_log_pattern, log, flags=re.MULTILINE)
     record_id = match.group("id")
@@ -215,7 +215,7 @@ def test_cli__publish():
     # Act
     # note - check_output raises CalledProcessError if exit code is non-zero
     log = subprocess.check_output(
-        f"python deposit.py {crate_path} -p", shell=True, text=True
+        f"rocrate_inveniordm {crate_path} -p", shell=True, text=True
     )
     match = re.search(expected_log_pattern, log, flags=re.MULTILINE)
     record_id = match.group("id")

test/unit/test_conditions.py

@@ -3,7 +3,7 @@
 import pytest
 import pytz
 
-import mapping.condition_functions as cf
+import rocrate_inveniordm.mapping.condition_functions as cf
 
 
 def test_is_uri__true():

test/unit/test_converter.py

@@ -1,8 +1,18 @@
+import json
 import re
 
 import pytest
 
-import mapping.converter as converter
+import rocrate_inveniordm.mapping.converter as converter
+
+
+def test_load_mapping_json():
+    with open("src/rocrate_inveniordm/mapping/mapping.json") as f:
+        expected_json = json.load(f)
+
+    result_json = converter.load_mapping_json()
+
+    assert result_json == expected_json
 
 
 def test_check_condition__true():

test/unit/test_credentials.py

@@ -2,7 +2,7 @@
 import pytest
 from unittest import mock
 
-import upload.credentials as credentials
+import rocrate_inveniordm.upload.credentials as credentials
 
 
 @mock.patch.dict(os.environ, {"INVENIORDM_API_KEY": ""})

test/unit/test_processing.py

@@ -1,4 +1,4 @@
-import mapping.processing_functions as pf
+import rocrate_inveniordm.mapping.processing_functions as pf
 
 
 def test_dateProcessing__date_only():

test/unit/test_publisher.py

@@ -7,7 +7,11 @@
 
 import pytest
 
-from mapping.converter import apply_mapping, get_mapping_paths, setup_dc
+from rocrate_inveniordm.mapping.converter import (
+    apply_mapping,
+    get_mapping_paths,
+    setup_dc,
+)
 
 publisher_string = "Test University"
 publisher_entity = {

test/unit/test_utils.py

@@ -2,7 +2,8 @@
 import pytest
 from requests.exceptions import HTTPError
 
-from deposit import deposit
+from rocrate_inveniordm.deposit import deposit
+import rocrate_inveniordm.upload.credentials as credentials
 from test.unit.utils import (
     add_entity_to_template,
     fetch_inveniordm_record,
@@ -12,7 +13,6 @@
     load_template_rc,
     set_field_in_template_rde,
 )
-import upload.credentials as credentials
 
 TEST_DATA_FOLDER = "test/data"
 TEST_OUTPUT_FOLDER = "test/output"

test/unit/utils.py

@@ -1,7 +1,8 @@
 import json
 import requests
 
-import upload.credentials as credentials
+import rocrate_inveniordm.mapping.converter as converter
+import rocrate_inveniordm.upload.credentials as credentials
 
 
 def load_template_rc(file="test/data/template-crate.json"):
@@ -25,9 +26,8 @@ def add_entity_to_template(entity, rc):
     return rc
 
 
-def get_single_mapping(mapping_class, rule, mapping_file_name="mapping/mapping.json"):
-    mapping_file = open(mapping_file_name)
-    m = json.load(mapping_file)
+def get_single_mapping(mapping_class, rule):
+    m = converter.load_mapping_json()
 
     if not mapping_class.endswith("_mapping"):
         mapping_class += "_mapping"
@@ -37,9 +37,8 @@ def get_single_mapping(mapping_class, rule, mapping_file_name="mapping/mapping.j
     return rule
 
 
-def get_mapping_class(mapping_class, mapping_file_name="mapping/mapping.json"):
-    mapping_file = open(mapping_file_name)
-    m = json.load(mapping_file)
+def get_mapping_class(mapping_class):
+    m = converter.load_mapping_json()
 
     if not mapping_class.endswith("_mapping"):
         mapping_class += "_mapping"