Merge branch 'master' into ext-source-type

README.md

@@ -1,2 +1,9 @@
-# ivpm
-IP and Verification Package Manager
+# IP and Verification Package Manager (IVPM)
+
+IVPM is a Python- and Git-centric utility for managing external 
+project dependencies.  It was initially designed to manage dependencies
+for hardware design projects, but has been used on a variety of other
+project styles including purely-software projects.
+
+You can find more detailed documentation on IVPM here: [IVPM docs](https://fvutils.github.io/ivpm)
+

src/ivpm/__init__.py

@@ -20,6 +20,7 @@
 #*
 #****************************************************************************
 import os
+from .ivpm_subprocess import ivpm_popen
 from .pkg_info_loader import PkgInfoLoader
 from .pkg_compile_flags import PkgCompileFlags
 from .pkg_info import PkgInfo

src/ivpm/__main__.py

@@ -13,6 +13,7 @@
 
 from ivpm.packages_info import PackagesInfo
 from ivpm.proj_info import ProjInfo
+from .cmds.cmd_activate import CmdActivate
 from .cmds.cmd_build import CmdBuild
 from .cmds.cmd_init import CmdInit
 from .cmds.cmd_update import CmdUpdate
@@ -171,9 +172,20 @@ def get_parser(parser_ext = None):
     subparser.required = True
     subparser.dest = 'command'
 
+    activate_cmd = subparser.add_parser("activate",
+        help="Starts a new shell that contains the activated python virtual environment")
+    activate_cmd.add_argument("-c",
+        help="When specified, executes the specified string")
+    activate_cmd.add_argument("-p", "--project-dir", dest="project_dir",
+        help="Specifies the project directory to use (default: cwd)")
+    activate_cmd.add_argument("args", nargs='*')
+    activate_cmd.set_defaults(func=CmdActivate())
+
     build_cmd = subparser.add_parser("build",
         help="Build all sub-projects with an IVPM-supported build infrastructure (Python)")
-    build_cmd.add_argument("-d", "--debug", 
+    build_cmd.add_argument("-d", "--dep-set", dest="dep_set", 
+        help="Uses dependencies from specified dep-set instead of 'default-dev'")
+    build_cmd.add_argument("-g", "--debug", 
         action="store_true",
         help="Enables debug for native extensions")
     build_cmd.set_defaults(func=CmdBuild())

src/ivpm/cmds/cmd_activate.py

@@ -0,0 +1,63 @@
+import os
+import sys
+import dataclasses as dc
+import subprocess
+from ivpm.project_info_reader import ProjectInfoReader
+from ivpm.utils import fatal
+
+@dc.dataclass
+class CmdActivate(object):
+
+    def __call__(self, args):
+        if args.project_dir is None:
+            # If a default is not provided, use the current directory
+#            print("Note: project_dir not specified ; using working directory")
+            args.project_dir = os.getcwd()
+
+        proj_info = ProjectInfoReader(args.project_dir).read()
+
+        if proj_info is None:
+            fatal("Failed to locate IVPM meta-data (eg ivpm.yaml)")
+
+        packages_dir = os.path.join(args.project_dir, "packages")
+        if not os.path.isdir(packages_dir):
+            fatal("No packages directory ; must run ivpm update first")
+
+        python_dir = os.path.join(packages_dir, "python")
+        if not os.path.isdir(python_dir):
+            fatal("No packages/python directory ; must run ivpm update first")
+
+        activate = os.path.join(python_dir, "bin/activate")
+
+        # TODO: consider non-bash shells and non-Linux platforms
+        shell = getattr(os.environ, "SHELL", "bash")
+        cmd = None
+        if shell.find("bash") != -1:
+            cmd = [shell, "-rcfile",  activate]
+
+            if args.c is not None:
+                cmd.extend(["-c", args.c])
+
+        cmd.extend(args.args)
+
+        env = os.environ.copy()
+        env["IVPM_PROJECT"] = args.project_dir
+        env["IVPM_PACKAGES"] = os.path.join(args.project_dir, "packages")
+        # env["VIRTUAL_ENV_DISABLE_PROMPT"] = "1"
+
+        # PS1 = getattr(env, "PS1", None)
+        # print("PS1: %s" % str(PS1))
+        # if PS1 is not None:
+        #     PS1 = "(ivpm) %s" % PS1
+        # else:
+        #     PS1 = "\\[\\](ivpm) "
+        # env["PS1"] = PS1
+
+        for es in proj_info.env_settings:
+            es.apply(env)
+
+        result = subprocess.run(
+            cmd,
+            env=env)
+        sys.exit(result.returncode)
+

src/ivpm/cmds/cmd_build.py

@@ -48,7 +48,7 @@ def __call__(self, args):
 
         ds_name = "default-dev"
 
-        if hasattr(args, "dep_set") and args.dep_set is not None:
+        if getattr(args, "dep_set") is not None:
             ds_name = args.dep_set
 
         if ds_name not in proj_info.dep_set_m.keys():

src/ivpm/env_spec.py

@@ -0,0 +1,95 @@
+#****************************************************************************
+#* env_spec.py
+#*
+#* Copyright 2023 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import sys
+import enum
+from typing import Any, Dict
+
+class EnvSpec(object):
+
+    class Act(enum.Enum):
+        Set = enum.auto()
+        Path = enum.auto()
+        PathAppend = enum.auto()
+        PathPrepend = enum.auto()
+
+    def __init__(self,
+                 var : str,
+                 val : Any,
+                 act : 'EnvSpec.Act'):
+        self.var = var
+        self.val = val
+        self.act = act
+
+    def apply(self, env : Dict[str,str]):
+        if isinstance(self.val, list):
+            for i,v in enumerate(self.val):
+                self.val[i] = self.expand(v, env)
+        else:
+            self.val = self.expand(self.val, env)
+
+        if self.act == EnvSpec.Act.Set:
+            val = self.val
+            if isinstance(val, list):
+                val = " ".join(val)
+            env[self.var] = val
+        elif self.act == EnvSpec.Act.Path:
+            val = self.val
+            if isinstance(val, list):
+                val = ":".join(val)
+            env[self.var] = val
+        elif self.act == EnvSpec.Act.PathAppend:
+            val = self.val
+            if isinstance(val, list):
+                val = ":".join(val)
+            if self.var in env.keys():
+                env[self.var] = env[self.var] + ":" + val
+            else:
+                env[self.var] = val
+        elif self.act == EnvSpec.Act.PathPrepend:
+            val = self.val
+            if isinstance(val, list):
+                val = ":".join(val)
+            if self.var in env.keys():
+                env[self.var] = val + ":" + env[self.var]
+            else:
+                env[self.var] = val
+        else:
+            raise Exception("Unknown action: %s" % str(self.act))
+
+    def expand(self, var, env):
+        idx = 0
+        while idx < len(var):
+            idx1 = var.find('${', idx)
+
+            if idx1 == -1:
+                break
+            idx2 = var.find('}', idx1)
+            if idx2 == -1:
+                idx = idx1+2
+            else:
+                key = var[idx1+2:idx2]
+                if key in env.keys():
+                    var = var[:idx1] + env[key] + var[idx2+1:]
+                else:
+                    idx = idx2+1
+        return var
+

src/ivpm/ivpm_subprocess.py

@@ -0,0 +1,45 @@
+
+import os
+import subprocess
+from .project_info_reader import ProjectInfoReader
+
+
+def ivpm_popen(cmd, **kwargs):
+    """
+    Wrapper around subprocess.Popen that configures paths from 
+    the nearest IVPM project.
+    """
+    ivpm_project = getattr(kwargs, "ivpm_project", None)
+
+    if ivpm_project is None:
+        # Search up from the invocation location
+        cwd = os.getcwd()
+        while cwd is not None and cwd != "/" and ivpm_project is None:
+            if os.path.exists(os.path.join(cwd, "ivpm.yaml")):
+                ivpm_project = cwd
+            else:
+                cwd = os.path.dirname(cwd)
+
+    if ivpm_project is not None:
+        # Update environment variables
+        proj_info = ProjectInfoReader(ivpm_project).read()
+
+        if proj_info is None:
+            raise Exception("Failed to read ivpm.yaml @ %s" % ivpm_project)
+
+        env = getattr(kwargs, "env", os.environ.copy())
+        env["IVPM_PROJECT"] = ivpm_project
+        env["IVPM_PACKAGES"] = os.path.join(ivpm_project, "packages")
+
+        # Add the virtual-environment path
+        if "PATH" in env.keys():
+            env["PATH"] = os.path.join(ivpm_project, "packages/python/bin") + ":" + env["PATH"]
+        else:
+            env["PATH"] = os.path.join(ivpm_project, "packages/python/bin")
+
+        for es in proj_info.env_settings:
+            es.apply(env)
+
+        kwargs["env"] = env
+
+    return subprocess.Popen(cmd, **kwargs)

src/ivpm/ivpm_yaml_reader.py

@@ -5,9 +5,11 @@
 '''
 import os
 import yaml_srcinfo_loader
+from typing import Dict, List
 from yaml_srcinfo_loader.srcinfo import SrcInfo
 from .package_factory import PackageFactory
 from .package import Package
+from .env_spec import EnvSpec
 
 import yaml
 
@@ -87,6 +89,13 @@ def read(self, fp, name) -> ProjInfo:
                     os.path.dirname(name),
                     ps_kind,
                     ps[ps_kind])
+
+        if "env" in pkg.keys():
+            es = pkg["env"]
+            for evar in es:
+                self.process_env_directive(
+                    ret,
+                    evar)
 
         return ret
 
@@ -200,4 +209,32 @@ def read_path_set(self, info : ProjInfo, path, ps_kind : str, ps):
             for p in ps[p_kind]:
                 path_kind_s[p_kind].append(os.path.join(path, p))
 
+    def process_env_directive(self,
+                              info : ProjInfo,
+                              evar : Dict):
+        if "name" not in evar.keys():
+            raise Exception("No variable-name specified: %s" % str(evar))
+        act = None
+        act_s = None
+        val = None
+        for an,av in [
+            ("value", EnvSpec.Act.Set),
+            ("path", EnvSpec.Act.Path),
+            ("path-append", EnvSpec.Act.PathAppend),
+            ("path-prepend", EnvSpec.Act.PathPrepend)]:
+            if an in evar.keys():
+                if act is not None:
+                    raise Exception("Multiple variable-setting directives specified: %s and %s" % (
+                        act_s, an))
+                act_s = an
+                act = av
+                val = evar[an]
+
+        if act is None:
+            raise Exception(
+                "No variable-directive setting (value, path, path-append, path-prepend) specified")
+        info.env_settings.append(EnvSpec(evar["name"], val, act))
+
+
+
 

src/ivpm/packages_mf_reader.py

@@ -8,7 +8,6 @@
 from ivpm.msg import fatal
 from ivpm.package import Package, Ext2SourceType, SourceType
 from ivpm.packages_info import PackagesInfo
-from lib2to3.pgen2.token import COLON
 
 
 class PackagesMfReader(object):

src/ivpm/pkg_info_rgy.py

@@ -41,12 +41,15 @@ def load(self):
         plugins = entry_points(group='ivpm.pkginfo')
 
         for p in plugins:
-            ext_t = p.load()
-            ext = ext_t()
-            if ext.name not in self.info_m.keys():
-                self.info_m[ext.name] = ext
-            else:
-                raise Exception("Duplicate package %s" % ext.name)
+            try:
+                ext_t = p.load()
+                ext = ext_t()
+                if ext.name not in self.info_m.keys():
+                    self.info_m[ext.name] = ext
+                else:
+                    raise Exception("Duplicate package %s" % ext.name)
+            except Exception as e:
+                print("IVPM: failed to load plugin (%s)" % str(e))
 
         # Finally, iterate through the path looking for leftovers (?)
         for path in sys.path:
@@ -55,16 +58,15 @@ def load(self):
                     if os.path.isfile(os.path.join(path, pkg, "pkginfo.py")):
                         try:
                             pkginfo_m = importlib.import_module("%s.pkginfo" % pkg)
-                            if not hasattr(pkginfo_m, "PkgInfo"):
-                                raise Exception("pkginfo missing PkgInfo")
-                            pkginfo_c = getattr(pkginfo_m, "PkgInfo", None)
-                            if pkginfo_c is not None:
-                                pkginfo = pkginfo_c()
-                                if pkginfo.name not in self.info_m.keys():
-                                    self.info_m[pkginfo.name] = pkginfo
-                        except Exception as e:
-                            pass
-
+                        except ModuleNotFoundError as e:
+                            print("Note: package @ %s has pkginfo.py, but cannot load %s.pkginfo (%s)" % (path, pkg, str(e)))
+                            continue
+                        if not hasattr(pkginfo_m, "PkgInfo"):
+                            print("Note: pkginfo (%s) module exists, but is missing PkgInfo", os.path.join(path, pkg))
+                            continue
+                        pkginfo = getattr(pkginfo_m, "PkgInfo")()
+                        if pkginfo.name not in self.info_m.keys():
+                            self.info_m[pkginfo.name] = pkginfo
 
     def hasPkg(self, pkg):
         return pkg in self.info_m.keys()

src/ivpm/proj_info.py

@@ -6,6 +6,7 @@
 from enum import Enum, auto
 from ivpm.packages_info import PackagesInfo
 from typing import Dict, List
+from .env_spec import EnvSpec
 
 class ProjInfo():
     def __init__(self, is_src):
@@ -27,6 +28,7 @@ def __init__(self, is_src):
 
         self.process_deps = True
         self.paths : Dict[str, Dict[str, List[str]]] = {}
+        self.env_settings : List[EnvSpec] = []
 
     def has_dep_set(self, name):
         return name in self.dep_set_m.keys()