Drop Python <= 3.8 support and add 3.13 (#588)

* Drop Python <= 3.8 support

* Add py 3.13 in CI tests

* Use conda-incubator to set up conda env

* Ensure correct activation throughout the tests

* Migrate to importlib in script2rst

* Migrate to importlib in resources

* Migrate to packaging.version

.github/workflows/test.yml

@@ -20,54 +20,48 @@ on:
 jobs:
   build-linux:
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -el {0}
     strategy:
       max-parallel: 10
       fail-fast: false
       matrix:
         python-version:
-          - "3.8"
           - "3.9"
           - "3.10"
           - "3.11"
           - "3.12"
+          - "3.13"
         conda-env:
           - "core"
           - "full"
     name: Python ${{ matrix.python-version }} and ${{ matrix.conda-env }} dependencies
     steps:
     - name: Checkout
       uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+    - uses: conda-incubator/setup-miniconda@v3
       with:
-        python-version:  ${{ matrix.python-version }}
-    - name: Add conda to system path
-      run: |
-        # $CONDA is an environment variable pointing to the root of the miniconda directory
-        echo $CONDA/bin >> $GITHUB_PATH
-        conda --version
-        conda info
-    - name: Make sure conda is updated
-      run: |
-        conda update conda
-        conda --version
+        auto-update-conda: true
+        auto-activate-base: false
+        conda-remove-defaults: true
+        channels: conda-forge
+        activate-environment: ptypy_env
+        python-version: ${{ matrix.python-version }}
     - name: Install ${{ matrix.conda-env }} dependencies
       run: |
         # replace python version in dependencies
         sed -i 's/python/python=${{ matrix.python-version }}/' dependencies_${{ matrix.conda-env }}.yml
         if [ ${{ matrix.conda-env }} == 'full' ] && [ ${{ matrix.python-version }} == '3.12' ]; then
           sed -i '/- pyfftw/d' dependencies_${{ matrix.conda-env }}.yml
         fi
-        # if [ ${{ matrix.conda-env }} == 'full' ] && [ ${{ matrix.python-version }} == '3.8' ]; then
-        #   sed -i '/- mpi4py/d' dependencies_${{ matrix.conda-env }}.yml
-        # fi
         # if [ ${{ matrix.conda-env }} == 'full' ] && [ ${{ matrix.python-version }} == '3.9' ]; then
         #   sed -i '/- mpi4py/d' dependencies_${{ matrix.conda-env }}.yml
         # fi
         conda install --solver=classic mpich
-        conda env update --file dependencies_${{ matrix.conda-env }}.yml --name base
+        conda env update --file dependencies_${{ matrix.conda-env }}.yml --name ptypy_env
         conda install --solver=classic flake8 pytest pytest-cov
-        conda list	
+        conda list
     - name: Prepare ptypy
       run: |
         # Install ptypy

dependencies_dev.yml

@@ -10,6 +10,7 @@ dependencies:
   - pyzmq
   - pep8
   - mpi4py
+  - packaging
   - pillow
   - pyfftw
   - pip

dependencies_full.yml

@@ -9,6 +9,7 @@ dependencies:
   - h5py
   - pyzmq
   - mpi4py
+  - packaging
   - pillow
   - pyfftw
   - pyyaml

doc/script2rst.py

@@ -1,5 +1,6 @@
 import sys
 import io
+from importlib.resources import files
 import contextlib
 import os
 
@@ -9,13 +10,12 @@
            'simupod.py',
            'ownengine.py',
            'subclassptyscan.py']
+_ptypy_dir = files('ptypy')
 
 if len(sys.argv) == 1:
-    import pkg_resources
-
     for script in scripts:
-        scr = pkg_resources.resource_filename('ptypy', tutorial_dir+script)
-        if not os.path.exists(scr):
+        scr = _ptypy_dir / (tutorial_dir + script)
+        if not scr.exists():
             print('Using backup tutorial for %s' % script)
             scr = '../tutorial/'+script
         #subprocess.call(['python',sys.argv[0]+' '+scr]) # doesn't work
@@ -50,13 +50,13 @@ def stdoutIO(stdout=None):
 frst.write("""
 .. note::
    This tutorial was generated from the python source
-   :file:`[ptypy_root]/tutorial/%(fname)s` using :file:`ptypy/doc/%(this)s`. 
+   :file:`[ptypy_root]/tutorial/%(fname)s` using :file:`ptypy/doc/%(this)s`.
    You are encouraged to modify the parameters and rerun the tutorial with::
-   
+
      $ python [ptypy_root]/tutorial/%(fname)s
 
 """ % {'fname': os.path.split(script_name)[-1], 'this': sys.argv[0]})
- 
+
 was_comment = True
 
 while True:
@@ -86,7 +86,7 @@ def stdoutIO(stdout=None):
             frst.write('   '+line2[1:].strip()+'\n')
         frst.write('\n')
         continue
-        
+
     if line.startswith('"""'):
         frst.write('.. parsed-literal::\n\n')
         while True:
@@ -95,11 +95,11 @@ def stdoutIO(stdout=None):
                 break
             frst.write('   ' + line2)
         continue
-    
+
     decorator = False
     indent = False
     for key in indent_keys:
-        if line.startswith(key): 
+        if line.startswith(key):
             indent = True
             break
 
@@ -125,12 +125,12 @@ def stdoutIO(stdout=None):
             pt = fpy.tell()
         exec(func+'\n')
         continue
-        
+
     wline = line.strip()
     if not wline:
         frst.write('\n')
         continue
-    
+
     with stdoutIO() as sout:
         exec(wline)
         out = sout.getvalue()
@@ -150,15 +150,15 @@ def stdoutIO(stdout=None):
             if was_comment:
                 wline = '\n::\n\n'+wline
                 was_comment = False
-                
+
             frst.write(wline+'\n')
-        
+
         #print out
         if out.strip():
             print(out)
             for l in out.split('\n'):
                 frst.write(' '*3+l+'\n')
             out = ''
 
-    
+
 

ptypy/accelerate/cuda_cupy/multi_gpu.py

@@ -10,12 +10,12 @@
   - OpenMPI in a conda install needs to have the environment variable
   --> if cuda support isn't enabled, the application simply crashes with a seg fault
 
-2) For NCCL peer-to-peer transfers, the EXCLUSIVE compute mode cannot be used. 
+2) For NCCL peer-to-peer transfers, the EXCLUSIVE compute mode cannot be used.
    It should be in DEFAULT mode.
 
 """
 
-from pkg_resources import parse_version
+from packaging.version import parse
 import numpy as np
 import cupy as cp
 from ptypy.utils import parallel
@@ -44,7 +44,7 @@
 have_cuda_mpi = (mpi4py is not None) and \
     "OMPI_MCA_opal_cuda_support" in os.environ and \
     os.environ["OMPI_MCA_opal_cuda_support"] == "true" and \
-    parse_version(parse_version(mpi4py.__version__).base_version) >= parse_version("3.1.0") and \
+    parse(parse(mpi4py.__version__).base_version) >= parse("3.1.0") and \
     not ('PTYPY_USE_MPI' in os.environ)
 
 
@@ -114,7 +114,7 @@ def allReduceSum(self, arr):
 
         count, datatype = self.__get_NCCL_count_dtype(arr)
 
-        self.com.allReduce(arr.data.ptr, arr.data.ptr, count, datatype, nccl.NCCL_SUM, 
+        self.com.allReduce(arr.data.ptr, arr.data.ptr, count, datatype, nccl.NCCL_SUM,
             cp.cuda.get_current_stream().ptr)
 
     def __get_NCCL_count_dtype(self, arr):

ptypy/accelerate/cuda_pycuda/multi_gpu.py

@@ -5,7 +5,7 @@
 
 Findings:
 
-1) NCCL works with unit tests, but not in the engines. It seems to 
+1) NCCL works with unit tests, but not in the engines. It seems to
 add something to the existing pycuda Context or create a new one,
 as a later event recording on an exit wave transfer fails with
 'ivalid resource handle' Cuda Error. This error typically happens if for example
@@ -22,14 +22,14 @@
   - OpenMPI in a conda install needs to have the environment variable
   --> if cuda support isn't enabled, the application simply crashes with a seg fault
 
-4) For NCCL peer-to-peer transfers, the EXCLUSIVE compute mode cannot be used. 
+4) For NCCL peer-to-peer transfers, the EXCLUSIVE compute mode cannot be used.
    It should be in DEFAULT mode.
 
 5) NCCL support has been dropped from PyCUDA module, but can be used with CuPy module instead
 
 """
 
-from pkg_resources import parse_version
+from packaging.version import parse
 import numpy as np
 from pycuda import gpuarray
 import pycuda.driver as cuda
@@ -54,7 +54,7 @@
 have_cuda_mpi = (mpi4py is not None) and \
     "OMPI_MCA_opal_cuda_support" in os.environ and \
     os.environ["OMPI_MCA_opal_cuda_support"] == "true" and \
-    parse_version(parse_version(mpi4py.__version__).base_version) >= parse_version("3.1.0") and \
+    parse(parse(mpi4py.__version__).base_version) >= parse("3.1.0") and \
     hasattr(gpuarray.GPUArray, '__cuda_array_interface__') and \
     not ('PTYPY_USE_MPI' in os.environ)
 
@@ -97,7 +97,7 @@ def allReduceSum(self, arr):
         if parallel.MPIenabled:
             comm = parallel.comm
             comm.Allreduce(parallel.MPI.IN_PLACE, arr)
-            
+
 
 # pick the appropriate communicator depending on installed packages
 def get_multi_gpu_communicator(use_cuda_mpi=True):

ptypy/resources/__init__.py

@@ -1,15 +1,17 @@
-import pkg_resources
+from importlib.resources import files
 import numpy as np
 
-flowerfile = pkg_resources.resource_filename(__name__,'flowers.png')
-moonfile = pkg_resources.resource_filename(__name__,'moon.png')
-treefile = pkg_resources.resource_filename(__name__,'tree.png')
+_res_dir = files('ptypy.resources')
+flowerfile =  _res_dir / 'flowers.png'
+moonfile = _res_dir / 'moon.png'
+treefile = _res_dir / 'tree.png'
+
 
 def flower_obj(shape=None):
     from ptypy import utils as u
     import numpy as np
     from matplotlib.image import imread
-    
+
     im = u.rgb2complex(imread(flowerfile))
     if shape is not None:
         sh = u.expect2(shape)
@@ -37,7 +39,7 @@ def moon_pr(shape=None):
     from ptypy import utils as u
     import numpy as np
     from matplotlib.image import imread
-    
+
     im = u.rgb2complex(imread(moonfile))
     if shape is not None:
         sh = u.expect2(shape)

pyproject.toml

@@ -11,7 +11,7 @@ authors = [
 ]
 description = "Ptychography Reconstruction for Python"
 readme = "README.rst"
-requires-python = ">=3.7"
+requires-python = ">=3.9"
 classifiers = [
     "Programming Language :: Python :: 3",
     "Development Status :: 3 - Alpha",
@@ -31,7 +31,7 @@ dependencies = [
 [project.optional-dependencies]
 full = ["mpi4py","matplotlib","pyzmq","pillow", "pyfftw", "pyyaml"]
 cupy-cuda11x = ["mpi4py","matplotlib","pyzmq","pillow", "pyfftw", "cupy-cuda11x"]
-cupy-cuda12x = ["mpi4py","matplotlib","pyzmq","pillow", "pyfftw", "cupy-cuda12x"] 
+cupy-cuda12x = ["mpi4py","matplotlib","pyzmq","pillow", "pyfftw", "cupy-cuda12x"]
 
 [project.scripts]
 "ptypy.plot" = "ptypy.cli.plotter:ptypy_plot"
@@ -69,4 +69,4 @@ ptypy = "ptypy"
 
 [tool.setuptools.package-data]
 ptypy = ["resources/*",]
-"ptypy.accelerate.cuda_common" = ["*.cu", "*.cuh"]
+"ptypy.accelerate.cuda_common" = ["*.cu", "*.cuh"]

test/accelerate_tests/cuda_cupy_tests/multi_gpu_test.py

@@ -11,7 +11,7 @@
     from ptypy.accelerate.cuda_cupy import multi_gpu as mgpu
     from ptypy.utils import parallel
 
-from pkg_resources import parse_version
+from packaging.version import parse
 
 class GpuDataTest(CupyCudaTest):
     """
@@ -36,8 +36,8 @@ def setUp(self):
 
     @unittest.skipIf(parallel.rank != 0, "Only in MPI rank 0")
     def test_version(self):
-        v1 = parse_version("3.1.0")
-        v2 = parse_version(parse_version("3.1.0a").base_version)
+        v1 = parse("3.1.0")
+        v2 = parse(parse("3.1.0a").base_version)
 
         self.assertGreaterEqual(v2, v1)
 
@@ -61,7 +61,7 @@ def multigpu_tester(self, com):
 
     def test_multigpu_auto(self):
         self.multigpu_tester(mgpu.get_multi_gpu_communicator())
-        
+
     def test_multigpu_mpi(self):
         self.multigpu_tester(mgpu.MultiGpuCommunicatorMpi())
 
@@ -71,4 +71,4 @@ def test_multigpu_cudampi(self):
 
     @unittest.skipIf(not mgpu.have_nccl, "NCCL not available")
     def test_multigpu_nccl(self):
-        self.multigpu_tester(mgpu.MultiGpuCommunicatorNccl())
+        self.multigpu_tester(mgpu.MultiGpuCommunicatorNccl())

test/accelerate_tests/cuda_pycuda_tests/multi_gpu_test.py

@@ -13,7 +13,7 @@
     from ptypy.accelerate.cuda_pycuda import multi_gpu as mgpu
     from ptypy.utils import parallel
 
-from pkg_resources import parse_version
+from packaging.version import parse
 
 class GpuDataTest(unittest.TestCase):
     """
@@ -51,8 +51,8 @@ def tearDown(self):
 
     @unittest.skipIf(parallel.rank != 0, "Only in MPI rank 0")
     def test_version(self):
-        v1 = parse_version("3.1.0")
-        v2 = parse_version(parse_version("3.1.0a").base_version)
+        v1 = parse("3.1.0")
+        v2 = parse(parse("3.1.0a").base_version)
 
         self.assertGreaterEqual(v2, v1)