Merge pull request #8 from schandrika/poetry_install_changes

Fix for issue eclipse-volttron#196. vctl should use poetry instead of pip to install agents

src/volttron/client/commands/install_agents.py

@@ -58,16 +58,16 @@ class InstallRuntimeError(RuntimeError):
 _stderr = sys.stderr
 
 
-def install_requirements(agent_source):
-    req_file = os.path.join(agent_source, "requirements.txt")
-
-    if os.path.exists(req_file):
-        _log.info(f"Installing requirements for agent from {req_file}.")
-        cmds = ["pip", "install", "-r", req_file]
-        try:
-            execute_command(cmds, logger=_log, err_prefix="Error installing requirements")
-        except InstallRuntimeError:
-            sys.exit(1)
+# def install_requirements(agent_source):
+#     req_file = os.path.join(agent_source, "requirements.txt")
+#
+#     if os.path.exists(req_file):
+#         _log.info(f"Installing requirements for agent from {req_file}.")
+#         cmds = ["pip", "install", "-r", req_file]
+#         try:
+#             execute_command(cmds, logger=_log, err_prefix="Error installing requirements")
+#         except InstallRuntimeError:
+#             sys.exit(1)
 
 
 def install_agent_directory(opts, publickey=None, secretkey=None):

src/volttron/server/aip.py

@@ -343,7 +343,7 @@ def set_agent_user_permissions(self, volttron_agent_user, agent_uuid, agent_dir)
         #  VOLTTRON 8.0 once CSR is implemented for
         #  federation and shovel. The below lines can be removed then
         if self.message_bus == "rmq":
-            os.chmod(os.path.join(cc.get_volttron_home, "certificates/private"), 0o755)
+            os.chmod(os.path.join(cc.get_volttron_home(), "certificates/private"), 0o755)
             self.set_acl_for_path(
                 "r",
                 volttron_agent_user,
@@ -520,18 +520,12 @@ def install_agent(self,
         # removing last/only instance of an agent
         backup_agent_file = self.backup_agent_data(agent_uuid, vip_identity)
 
-        agent_name = self.install_agent_source(agent, force, pre_release)
+        agent_name, site_package_dir = self.install_agent_source(agent, force, pre_release)
 
         # get default vip_identity if vip_identity is not passed
         # default value will be in "agent_name-default-vip-id" file in site-packages dir
         if vip_identity is None:
-            # get site-packages dir using agent's package name
-            # installed package name is agent_name without version. Version is at the end. ex. agent-name-0.2.1
-            installed_package_name = agent_name[0:agent_name.rfind("-")]
-            cmd = [sys.executable, "-m", "pip", "show", installed_package_name]
-            response = execute_command(cmd)
-            site_package_dir = re.search(".*\nLocation: (.*)", response).groups()[0].strip()
-            # get default vip id
+            # get default vip id if one is specified in src
             default_vip_id_file = os.path.join(site_package_dir, f"{agent_name}-default-vip-id")
             _log.info(f"Default vip id file is {default_vip_id_file}")
             if os.path.isfile(default_vip_id_file):
@@ -621,89 +615,63 @@ def _raise_error_if_identity_exists_without_force(self, vip_identity: str, force
     def install_agent_source(self, agent: str, force: bool = False, pre_release: bool = False):
         _log.info(f"AGENT_WHEEL: {agent}")
 
-        if force:
-            cmd = [sys.executable, "-m", "pip", "install", "--force-reinstall"]
-        else:
-            cmd = [sys.executable, "-m", "pip", "install"]
-
+        cmd = ["poetry", "--directory", self._server_opts.poetry_project_path.as_posix(), "add"]
+        # TODO - no --force-reinstall for poetry. will have to uninstall and reinstall
+        # if force:
+        #     cmd.append("--force-reinstall")
         if pre_release:
-            cmd.append("--pre")
+            cmd.append("--allow-prereleases")
 
         cmd.append(agent)
         _log.debug(f"Executing agent install command : {cmd}")
         response = execute_command(cmd)
+        # if above cmd returned non-zero code it would throw exception.
+        # if we are here we succeeded installing some compatible version of the package.
+        # Now find agent name and version installed
         agent_name = None
-        last_line = response.strip().split("\n")[-1]
-        _log.debug(f"last line of response {last_line}")
-        find_success = re.match("Successfully installed (.*)", last_line)
-
-        if find_success:
-            _log.debug(f"Successfully installed package: {find_success}")
-            # if successfully installed packages includes the agent then agent_name should be returned.
-            agent_name = self._get_agent_name_on_success(
-                find_success, self._construct_package_name_from_agent_wheel(agent))
-
-        # agent was not installed. Check if agent package already exists
-        if not agent_name:
-            # output in this case depends on whether you install from pypi or wheel
-            if agent.endswith(".whl") and os.path.isfile(agent):
-                # format :
-                # {package name} is already installed with the same version as the provided wheel.
-                agent_name = self._parse_wheel_install_response(response, agent)
-            else:
-                # install from pypi - output will have format
-                # Requirement already satisfied: {name} in ./.venv/lib/python3.8/site-packages ({version})
-                agent_name = self._parse_pypi_install_response(response, agent)
-        return agent_name
+        if agent.endswith(".whl") and os.path.isfile(agent):
+            agent_name = self._construct_package_name_from_agent_wheel(agent)
+        else:
+            # this is a pypi package.
+            # if vctl install got source dir, it would have got built into a whl before getting shipped to server
+            # it could be just a package-name(ex. volttron-listner)
+            # or package-name with version constraints- ex. volttron-agent@latest, volttron-agent>=1.0.0
+            # so match till we hit a character that is NOT alpha numeric character or  _ or -
+            m = re.match("[\w\-]+", agent)
+            if m:
+                agent_name = m[0]
+        if agent_name is None:
+            # ideally we should never get here! if we get here we haven't handled some specific input format.
+            raise RuntimeError(f"Something went wrong when installing {agent}. "
+                               f"Unable to parse agent name of the package.\n"
+                               f"Agent install command used: {' '.join(cmd)} \n"
+                               f"Command completed with exit code 0 and stdout "
+                               f"response {response}")
+
+        # now get the version installed, because poetry add could have been for volttron-agent@latest.
+        # we need to find the specific version installed
+        cmd = ["pip", "show", agent_name]
+        response = execute_command(cmd)
+        version = re.search(".*\nVersion: (.*)", response).groups()[0].strip()
+        site_package_dir = re.search(".*\nLocation: (.*)", response).groups()[0].strip()
+        if site_package_dir is None:
+            # we should not get here unless pip changed format of pip show output.
+            raise RuntimeError(f"Unable to find installed location of {agent} based on pip show command")
+        if version is None:
+            # we should not get here unless pip changed format of pip show output.
+            raise RuntimeError(f"Unable to find installed version of {agent} based on pip show command")
+
+        return agent_name + "-" + version, site_package_dir
 
     @staticmethod
     def _construct_package_name_from_agent_wheel(agent_wheel):
         agent_name = agent_wheel
         if agent_wheel.endswith(".whl"):
             wheel = agent_wheel.split("/")[-1]
-            agent_name = wheel.replace("-py3-none-any.whl", "").replace("_", "-")
+            agent_name_with_version = wheel.replace("-py3-none-any.whl", "").replace("_", "-")
+            agent_name = agent_name_with_version[:agent_name_with_version.rfind("-")]
         return agent_name
 
-    @staticmethod
-    def _get_agent_name_on_success(find_success, agent_package):
-        _log.info(f"searching for: {agent_package} in response of successful pip install")
-        for package in find_success.groups()[0].split():
-            # search for the agent name that we want. agent_package might not have version number
-            _log.debug(f"package: {package}")
-            if package.startswith(agent_package):
-                return package
-        # List of successfully installed packages does not include agent package. maybe we installed only
-        # some missing dependencies but agent package already exists. Return none
-        return None
-
-    @staticmethod
-    def _parse_wheel_install_response(response, agent_wheel):
-        groups = None
-        result = re.search(
-            ".*\n(.*) is already installed with the same version as the provided wheel", response)
-        if result:
-            groups = result.groups()
-        if groups:
-            find_already_installed = groups[0].strip()
-            cmd = ["pip", "show", find_already_installed]
-            response = execute_command(cmd)
-            version = re.search(".*\nVersion: (.*)", response).groups()[0].strip()
-            return find_already_installed + "-" + version
-        raise ValueError(f"Couldn't install {agent_wheel}\n{response}")
-
-    @staticmethod
-    def _parse_pypi_install_response(response, package_name):
-        _log.info(f"Parsing pypi install response {response}")
-        line = response.split('\r\n')[0]
-        match = re.match(f"Requirement already satisfied: {package_name}.*\\((.*)\\)", line)
-        groups = None
-        if match:
-            groups = match.groups()
-        if groups:
-            version = groups[0].strip()
-            return package_name + "-" + version
-        raise ValueError(f"Couldn't install {package_name}\n{response}")
-
     def _setup_agent_vip_id(self, agent_name, vip_identity=None):
         # agent_path = os.path.join(self.install_dir, agent_name)
         # name = self.agent_name(agent_name)

src/volttron/server/run_server.py

@@ -57,7 +57,7 @@
 from volttron.types.events import volttron_home_set_evnt
 from volttron.types.peer import ServicePeerNotifier
 from volttron.types.server_config import ServerConfig, ServiceConfigs
-from volttron.utils import ClientContext as cc
+from volttron.utils import ClientContext as cc, execute_command
 from volttron.utils import (get_version, store_message_bus_config)
 from volttron.utils.persistance import load_create_store
 
@@ -112,16 +112,36 @@ def run_server():
     opts = parser.parse_args(args)
 
     # Handle the fact that we don't use store_true and config that requires
-    # inverse.  This is not a switch but a mode of operation so we change
+    # inverse.  This is not a switch but a mode of operation, so we change
     # from the string to a boolean value here.
     opts.agent_isolation_mode = opts.agent_isolation_mode != 'False'
-
+    dev_mode = opts.dev_mode
     # Update the server options with the command line parameter options.
     server_options.update(opts)
     server_options.store()
+
+    # create poetry project and poetry lock file in VOLTTRON_HOME
+    if dev_mode:
+        if not os.path.isfile(server_options.poetry_project_path/"pyproject.toml"):
+            raise ValueError("VOLTTRON is run with --dev but unable to fund pyproject.toml is current directory - "
+                             f"{server_options.poetry_project_path}")
+    else:
+        setup_poetry_project(server_options.poetry_project_path)
+
     start_volttron_process(server_options)
 
 
+def setup_poetry_project(volttron_home):
+    toml = os.path.join(volttron_home, "pyproject.toml")
+    if not os.path.isfile(toml):
+        cmd = [sys.executable, "-m", "poetry", "init", "--directory", volttron_home,
+               "--name", "volttron", "--author", "volttron <[email protected]>", "--quiet"]
+        execute_command(cmd)
+    cmd = [sys.executable, "-m", "pip", "list", "--format", "freeze", "|", "grep", "-v", "volttron==",
+           "|", "xargs", "poetry", "add", "--directory", volttron_home]
+    execute_command(cmd)
+
+
 def start_volttron_process(options: ServerOptions):
     opts = options
     # Change working dir
@@ -615,6 +635,12 @@ def build_arg_parser(options: ServerOptions) -> argparse.ArgumentParser:
                         help=argparse.SUPPRESS)
     parser.add_argument("--auth-disabled", action="store_false", help=argparse.SUPPRESS)
     parser.add_argument("--show-config", action="store_true", help=argparse.SUPPRESS)
+    parser.add_argument("--dev",
+                        action="store_true",
+                        dest="dev_mode",
+                        help="development mode with poetry environment to build volttron libraries from source/pypi"
+                        )
+
     parser.add_help_argument()
     parser.add_version_argument(version="%(prog)s " + str(get_version()))
 

src/volttron/server/server_options.py

@@ -63,6 +63,7 @@ class ServerOptions:
     config_file: Path = None
     initialized: bool = False
     service_address: str = None
+    poetry_project_path: Path = None
 
     # services: list[ServiceData] = field(default_factory=list)
 
@@ -78,6 +79,9 @@ def __post_init__(self):
         if self.volttron_home is None:
             self.volttron_home = Path(os.environ.get("VOLTTRON_HOME", "~/.volttron")).expanduser()
 
+        if self.poetry_project_path is None:
+            self.poetry_project_path = self.volttron_home
+
         # Should be the only location where we create VOLTTRON_HOME
         if not self.volttron_home.is_dir():
             self.volttron_home.mkdir(mode=0o755, exist_ok=True, parents=True)
@@ -117,10 +121,17 @@ def update(self, opts: argparse.Namespace | dict):
         """
         address = set(opts.address)
         opts.address = list(address)
+
         if isinstance(opts, dict):
+            dev_mode = opts.pop("dev_mode")
             self.__dict__.update(opts)
         else:
-            self.__dict__.update(opts.__dict__)
+            d = opts.__dict__
+            dev_mode = d.pop("dev_mode")
+            self.__dict__.update(d)
+
+        if dev_mode:
+            self.poetry_project_path = Path(os.path.abspath(os.curdir))
 
     def store(self, file: Path = None):
         """
@@ -141,8 +152,8 @@ def store(self, file: Path = None):
             try:
                 # Don't save volttron_home within the config file.
                 if field.name not in ('volttron_home', 'services', 'config_file', 'initialized',
-                                      'service_address'):
-                    # More than one address can be present so we must be careful
+                                      'service_address', "poetry_project_path"):
+                    # More than one address can be present, so we must be careful
                     # with it.
                     if field.name == 'address':
                         found = set()