Bugfix 2675 move to pyproject.toml (#2699)

* 2675: move to pyproject.toml

* 2675: update toml, remove setup.py ref

* bugfix 2675: reinstate VERSION file

* bugfix 2675: hardcode version in  pyproject.toml

* use relative versions

* slightly modify import to make it easier to move produtil into metplus

* read version from version file

* disable install of run_metplus script since it is broken

* only install metplus and sub-packages, add version files to package data (parm cannot be added because it is non-package data)

* add newline back

* install produtil package so metplus can be imported in python

* remove unused import

* move produtil under metplus to differentiate it from the full NCEP produtil package and isolate it to metplus when installing metplus as a package -- previously produtil would be installed as its own package when metplus is installed

* fixed more imports

* Create links for parm directory and scripts/run_metplus.py under metplus so the parm files and run script will be installed as part of the metplus python package. Modified config_metplus to find the parm directory from under metplus to be consistent between running via python package and via scripts

* Exclude parm from sonar sources because it will be found under metplus. Add metplus/parm/** to list of coverage exclusions

* fixed more imports

* add parm back to list of sonar sources

---------

Co-authored-by: George McCabe <[email protected]>

.coveragerc

@@ -3,3 +3,4 @@ relative_files = True
 source = metplus
 omit =
   metplus/wrappers/cyclone_plotter_wrapper.py
+  metplus/produtil/**

internal/scripts/docker/Dockerfile

@@ -58,5 +58,5 @@ RUN if [ ${OBTAIN_SOURCE_CODE} != "none" ]; then \
   sed -i 's|MET_INSTALL_DIR = /path/to|MET_INSTALL_DIR = /usr/local|g' parm/metplus_config/*.conf; \
   sed -i 's|OUTPUT_BASE = /path/to|OUTPUT_BASE = /data/output|g' parm/metplus_config/*.conf; \
   sed -i 's|INPUT_BASE = /path/to|INPUT_BASE = /data/input/METplus_Data|g' parm/metplus_config/*.conf; \
-  python3 setup.py install; \
+  pip install .; \
 fi

internal/scripts/sonarqube/sonar-project.properties

@@ -3,8 +3,8 @@ sonar.projectKey=METplus
 sonar.projectName=METplus
 sonar.projectVersion=SONAR_PROJECT_VERSION
 sonar.branch.name=SONAR_BRANCH_NAME
-sonar.sources=docs,internal,manage_externals,metplus,parm,produtil,ush
-sonar.coverage.exclusions=internal/tests/**,parm/**,internal/scripts/**,manage_externals/**,docs/**,produtil/**,ush/**,metplus/wrappers/cyclone_plotter_wrapper.py
+sonar.sources=docs,internal,manage_externals,metplus,parm,ush
+sonar.coverage.exclusions=internal/tests/**,parm/**,metplus/parm/**,internal/scripts/**,manage_externals/**,docs/**,metplus/produtil/**,ush/**,metplus/wrappers/cyclone_plotter_wrapper.py
 sonar.python.coverage.reportPaths=coverage.xml
 sonar.sourceEncoding=UTF-8
 

internal/tests/pytests/util/run_util/test_run_util.py

@@ -4,7 +4,6 @@
 import os
 import re
 
-import produtil
 import metplus.util.run_util as ru
 import metplus.util.wrapper_init as wi
 from metplus.wrappers.ensemble_stat_wrapper import EnsembleStatWrapper

metplus/parm

@@ -0,0 +1 @@
+../parm

produtil/config.py → metplus/produtil/config.py

@@ -11,19 +11,18 @@
 # decides what symbols are imported by "from produtil.config import *"
 __all__=['from_file','confwalker','ProdConfig','ENVIRONMENT','ProdTask']
 
-import collections,re,string,os,logging,threading
+import collections,re,os,logging,threading
 import os.path,sys
 import datetime
-import produtil.fileop, produtil.datastore
-import produtil.numerics, produtil.log
+import metplus.produtil.fileop as fileop
 
 import configparser
 from configparser import ConfigParser
 from io import StringIO
 
-from produtil.datastore import Datastore,Task
+from metplus.produtil.datastore import Datastore,Task
 
-from produtil.numerics import to_datetime
+from metplus.produtil.numerics import to_datetime, to_datetime_rel, fcst_hr_min
 from string import Formatter
 from configparser import NoOptionError,NoSectionError
 
@@ -379,7 +378,7 @@ def get_value(self,key,args,kwargs):
                 v=ap6.strftime(ANL_P6_KEYS[key])
             elif '__ftime' in kwargs and '__atime' in kwargs and \
                     key in TIME_DIFF_KEYS:
-                (ihours,iminutes)=produtil.numerics.fcst_hr_min(
+                (ihours,iminutes)=fcst_hr_min(
                     kwargs['__ftime'],kwargs['__atime'])
                 if key=='fahr':
                     v=int(ihours)
@@ -702,7 +701,7 @@ def from_args(self,args=None,allow_files=True,allow_options=True,
             elif not os.path.isfile(path):
                 logger.error(path+': conf file is not a regular file.')
                 sys.exit(2)
-            elif not produtil.fileop.isnonempty(path) and verbose:
+            elif not fileop.isnonempty(path) and verbose:
                 logger.warning(
                     path+': conf file is empty.  Will continue anyway.')
             if verbose: logger.info('Conf input: '+repr(path))
@@ -823,7 +822,7 @@ def getdatastore(self):
         with self:
             if self._datastore is None:
                 dsfile=self.getstr('config','datastore')
-                self._datastore=produtil.datastore.Datastore(dsfile,
+                self._datastore=Datastore(dsfile,
                     logger=self.log('datastore'))
             return self._datastore
 
@@ -1001,7 +1000,7 @@ def makedirs(self,*args):
         with self:
             dirs=[self.getstr('dir',arg) for arg in args]
         for makeme in dirs:
-            produtil.fileop.makedirs(makeme)
+            fileop.makedirs(makeme)
     def keys(self,sec):
         """!get options in a section
 
@@ -1107,13 +1106,13 @@ def timestrinterp(self,sec,string,ftime=None,atime=None,**kwargs):
         @param atime the analysis time or None
         @param kwargs more variables for string expansion"""
         if atime is not None:
-            atime=produtil.numerics.to_datetime(atime)
+            atime=to_datetime(atime)
         else:
             atime=self.cycle
         if ftime is None:
             ftime=atime
         else:
-            ftime=produtil.numerics.to_datetime_rel(ftime,atime)
+            ftime=to_datetime_rel(ftime,atime)
         with self:
             return self._time_formatter.format(string,__section=sec,
                 __key='__string__',__depth=0,__conf=self._conf,ENV=ENVIRONMENT,
@@ -1343,7 +1342,7 @@ def getbool(self,sec,opt,default=None,badtypeok=False,morevars=None,taskvars=Non
 
 ########################################################################
 
-class ProdTask(produtil.datastore.Task):
+class ProdTask(Task):
     """!A subclass of produtil.datastore.Task that provides a variety
     of convenience functions related to unix conf files and
     logging."""

produtil/datastore.py → metplus/produtil/datastore.py

@@ -6,9 +6,10 @@
 Datum, which is the base class of anything that can be stored in the
 Datastore."""
 
-import sqlite3, threading, collections, re, contextlib, time, random,\
-    traceback, datetime, logging, os, time
-import produtil.fileop, produtil.locking, produtil.sigsafety, produtil.log
+import sqlite3, threading, collections, re, contextlib, datetime, logging, os, time
+import metplus.produtil.fileop as fileop
+from metplus.produtil.locking import LockFile
+from metplus.produtil.log import jlogger
 
 ##@var __all__
 # Symbols exported by "from produtil.datastore import *"
@@ -175,7 +176,7 @@ def __init__(self,filename,logger=None,locking=True):
         lockfile=filename+'.lock'
         if logger is not None:
             logger.debug('Lockfile is %s for database %s'%(lockfile,filename))
-        self._file_lock=produtil.locking.LockFile(
+        self._file_lock=LockFile(
             lockfile,logger=logger,max_tries=300,sleep_time=0.1,first_warn=50)
         self._transtack=collections.defaultdict(list)
         with self.transaction():
@@ -884,7 +885,7 @@ def undeliver(self,delete=True,logger=None):
         @param logger a logging.Logger for log messages"""
         loc=self.location
         if loc and delete:
-            produtil.fileop.remove_file(filename=loc,logger=logger,info=True)
+            fileop.remove_file(filename=loc,logger=logger,info=True)
         self.available=False
     def deliver(self,location=None,frominfo=None,keep=True,logger=None,
                 copier=None):
@@ -917,7 +918,7 @@ def deliver(self,location=None,frominfo=None,keep=True,logger=None,
             raise UnknownLocation(
                 '%s: no location known when delivering product.  Specify a '
                 'location to deliver().'%(self.did))
-        produtil.fileop.deliver_file(frominfo,loc,keep=keep,logger=logger,
+        fileop.deliver_file(frominfo,loc,keep=keep,logger=logger,
                                      copier=copier)
         if setloc:
             self.set_loc_avail(loc,True)
@@ -966,7 +967,7 @@ def check(self,frominfo=None,minsize=None,minage=None,logger=None):
             minsize=int(self.get('minsize',0))
         if minage is None:
             minage=int(self.get('minage',20))
-        if not produtil.fileop.check_file(loc,min_size=minsize,
+        if not fileop.check_file(loc,min_size=minsize,
                 min_mtime_age=minage):
             if self.available:
                 self.available=False
@@ -1093,7 +1094,7 @@ def jlogfile(self):
         intended to receive only major errors, and per-job start and
         completion information.  This is equivalent to simply
         accessing produtil.log.jlogger."""
-        return produtil.log.jlogger
+        return jlogger
 
     def postmsg(self,message,*args,**kwargs):
         """!same as produtil.log.jlogger.info()
@@ -1103,7 +1104,7 @@ def postmsg(self,message,*args,**kwargs):
         @param message the message
         @param args positional arguments for string replacement
         @param kwargs keyword arguments for string replacement."""
-        produtil.log.jlogger.info(message,*args,**kwargs)
+        jlogger.info(message,*args,**kwargs)
 
     def setstate(self,val):
         """!Sets the state of this job.

produtil/dbnalert.py → metplus/produtil/dbnalert.py

@@ -6,10 +6,9 @@
 __all__=["DBNAlert"]
 
 import logging, os
-import produtil.run
 
-from produtil.prog import Runner
-from produtil.run import checkrun, batchexe, alias, run
+from metplus.produtil.prog import Runner
+from metplus.produtil.run import batchexe, alias, run
 
 # Globals:
 

produtil/fileop.py → metplus/produtil/fileop.py

@@ -18,7 +18,7 @@
          'netcdfver','touch']
 
 import os,tempfile,filecmp,stat,shutil,errno,random,time,fcntl,math,logging
-import produtil.cluster, produtil.pipeline
+import metplus.produtil.cluster as cluster
 
 module_logger=logging.getLogger('produtil.fileop')
 
@@ -403,9 +403,9 @@ def deliver_file(infile,outfile,keep=True,verify=False,blocksize=1048576,
       the temp_file_object is an object that can be used to write to 
       the file.  The copier should NOT close the temp_file_object. """
     if preserve_group is None:
-        preserve_group = not produtil.cluster.group_quotas()
+        preserve_group = not cluster.group_quotas()
     if copy_acl is None:
-        copy_acl = produtil.cluster.use_acl_for_rstdata()
+        copy_acl = cluster.use_acl_for_rstdata()
     if copier is not None:
         # Cannot simply do a "move" if we are using an external
         # function to copy.

produtil/locking.py → metplus/produtil/locking.py

@@ -14,9 +14,9 @@
 ...  the file is now unlocked ...
 @endcode"""
 
-import fcntl, time, errno, os.path
-import produtil.retry as retry
-import produtil.fileop
+import fcntl, errno, os.path
+import metplus.produtil.retry as retry
+import metplus.produtil.fileop as fileop
 
 ##@var __all__
 # Symbols exported by "from produtil.locking import *"
@@ -110,7 +110,7 @@ def acquire_impl(self):
                                   'the process was exiting.')
         thedir=os.path.dirname(self._filename)
         if thedir:
-            produtil.fileop.makedirs(thedir)
+            fileop.makedirs(thedir)
         if self._fd is None:
             self._fd=open(self._filename,'wb')
         try:

produtil/log.py → metplus/produtil/log.py

@@ -12,7 +12,7 @@
             'MasterLogHandler','JLogHandler','set_jlogfile' ]
 
 import logging, os, sys, traceback, threading
-import produtil.batchsystem
+import metplus.produtil.batchsystem as batchsystem
 
 ##@var logthread
 # string for log messages to indicate thread number/name 
@@ -379,7 +379,7 @@ def configureLogging(jlogfile=None,
     # Configure log formatting:
     jobstr=os.environ.get('job',None)
     if jobstr is None:
-        jobstr=produtil.batchsystem.jobname()
+        jobstr=batchsystem.jobname()
     jobstr=str(jobstr).replace('(','_').replace(')','_').replace('%','_')
     # Format for jlogfile domain logging to jlogfile:
     jformat=JLogFormatter(

produtil/mpi_impl/__init__.py → metplus/produtil/mpi_impl/__init__.py

@@ -125,7 +125,8 @@
 # __init__.py on how to modify it to achieve these steps.
 
 import logging
-import produtil.fileop
+import metplus.produtil.fileop
+import metplus.produtil.prog
 
 ##@var __all__
 # An empty list that indicates no symbols are exported by "from
@@ -220,8 +221,8 @@ def register_implementations(logger=None):
     # no_implementation=None is used to detect if
     # register_implementations was called.
     global no_implementation
-    import produtil.mpi_impl.no_mpi
-    no_implementation=produtil.mpi_impl.no_mpi.Implementation.detect()
+    from metplus.produtil.mpi_impl.no_mpi import Implementation
+    no_implementation=Implementation.detect()
 
     # Now add each implementation.  We need to wrap each around a
     # try...except so that NCEP Central Operations can delete the
@@ -230,64 +231,64 @@ def register_implementations(logger=None):
 
     try:
         # If we have srun, and we're in a pack group...
-        import produtil.mpi_impl.srun_pack_groups
-        add_implementation(produtil.mpi_impl.srun_pack_groups.Implementation)
+        import metplus.produtil.mpi_impl.srun_pack_groups
+        add_implementation(metplus.produtil.mpi_impl.srun_pack_groups.Implementation)
     except ImportError:
         pass
 
     try:
         # This must be after the pack group case.
         # If we have srun and SLURM resources...
-        import produtil.mpi_impl.srun
-        add_implementation(produtil.mpi_impl.srun.Implementation)
+        import metplus.produtil.mpi_impl.srun
+        add_implementation(metplus.produtil.mpi_impl.srun.Implementation)
     except ImportError:
         pass
 
     try:
-        import produtil.mpi_impl.inside_aprun
-        add_implementation(produtil.mpi_impl.inside_aprun.Implementation)
+        import metplus.produtil.mpi_impl.inside_aprun
+        add_implementation(metplus.produtil.mpi_impl.inside_aprun.Implementation)
     except ImportError:
         pass
 
     try:
-        import produtil.mpi_impl.pbs_cray_intel
-        add_implementation(produtil.mpi_impl.pbs_cray_intel.Implementation)
+        import metplus.produtil.mpi_impl.pbs_cray_intel
+        add_implementation(metplus.produtil.mpi_impl.pbs_cray_intel.Implementation)
     except ImportError:
         pass
 
     try:
-        import produtil.mpi_impl.lsf_cray_intel
-        add_implementation(produtil.mpi_impl.lsf_cray_intel.Implementation)
+        import metplus.produtil.mpi_impl.lsf_cray_intel
+        add_implementation(metplus.produtil.mpi_impl.lsf_cray_intel.Implementation)
     except ImportError:
         pass
 
     try:
-        import produtil.mpi_impl.impi
-        add_implementation(produtil.mpi_impl.impi.Implementation)
+        import metplus.produtil.mpi_impl.impi
+        add_implementation(metplus.produtil.mpi_impl.impi.Implementation)
     except ImportError:
         pass
 
     try:
-        import produtil.mpi_impl.mpirun_lsf
-        add_implementation(produtil.mpi_impl.mpirun_lsf.Implementation)
+        import metplus.produtil.mpi_impl.mpirun_lsf
+        add_implementation(metplus.produtil.mpi_impl.mpirun_lsf.Implementation)
     except ImportError:
         pass
 
     try:
-        import produtil.mpi_impl.mpiexec_mpt
-        add_implementation(produtil.mpi_impl.mpiexec_mpt.Implementation)
+        import metplus.produtil.mpi_impl.mpiexec_mpt
+        add_implementation(metplus.produtil.mpi_impl.mpiexec_mpt.Implementation)
     except ImportError:
         pass
 
     try:
-        import produtil.mpi_impl.mpiexec
-        add_implementation(produtil.mpi_impl.mpiexec.Implementation)
+        import metplus.produtil.mpi_impl.mpiexec
+        add_implementation(metplus.produtil.mpi_impl.mpiexec.Implementation)
     except ImportError:
         pass
 
     try:
-        import produtil.mpi_impl.srun_shell
-        add_implementation(produtil.mpi_impl.srun_shell.Implementation)
+        import metplus.produtil.mpi_impl.srun_shell
+        add_implementation(metplus.produtil.mpi_impl.srun_shell.Implementation)
     except ImportError:
         pass
 
@@ -315,7 +316,7 @@ def get_mpi(mpi_name=NO_NAME,force=False,logger=None,**kwargs):
     @raise NotImplementedError if the MPI implementation is unknown,
     or if the implementation is unavailble on this machine, and
     force=False"""
-
+    import metplus.produtil.mpi_impl.no_mpi as no_mpi
     if logger is None:
         logger=logging.getLogger('mpi_impl')
 
@@ -351,8 +352,8 @@ def get_mpi(mpi_name=NO_NAME,force=False,logger=None,**kwargs):
             result=detect(
                 force=force,logger=logger,**kwargs)
         except (Exception,
-                produtil.fileop.FileOpError,
-                produtil.prog.ExitStatusException):
+                metplus.produtil.fileop.FileOpError,
+                metplus.produtil.prog.ExitStatusException):
             # Ignore exceptions related to an inability to detect the
             # MPI implementation.  We assume the issue has already
             # been logged, and we move on to the next implementation's