From 8b52a4695df7882ac1d50eae9fffcf970f836105 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Wed, 7 Aug 2024 13:41:18 -0700 Subject: [PATCH] changed vctl install to use poetry add instead of pip install. Added --dev option to volttron start to use poetry correctly. Fix for issue #196 --- .../client/commands/install_agents.py | 20 +-- src/volttron/server/aip.py | 124 +++++++----------- src/volttron/server/run_server.py | 32 ++++- src/volttron/server/server_options.py | 17 ++- 4 files changed, 99 insertions(+), 94 deletions(-) diff --git a/src/volttron/client/commands/install_agents.py b/src/volttron/client/commands/install_agents.py index 6e52fbc4c..8de4bb54d 100644 --- a/src/volttron/client/commands/install_agents.py +++ b/src/volttron/client/commands/install_agents.py @@ -58,16 +58,16 @@ class InstallRuntimeError(RuntimeError): _stderr = sys.stderr -def install_requirements(agent_source): - req_file = os.path.join(agent_source, "requirements.txt") - - if os.path.exists(req_file): - _log.info(f"Installing requirements for agent from {req_file}.") - cmds = ["pip", "install", "-r", req_file] - try: - execute_command(cmds, logger=_log, err_prefix="Error installing requirements") - except InstallRuntimeError: - sys.exit(1) +# def install_requirements(agent_source): +# req_file = os.path.join(agent_source, "requirements.txt") +# +# if os.path.exists(req_file): +# _log.info(f"Installing requirements for agent from {req_file}.") +# cmds = ["pip", "install", "-r", req_file] +# try: +# execute_command(cmds, logger=_log, err_prefix="Error installing requirements") +# except InstallRuntimeError: +# sys.exit(1) def install_agent_directory(opts, publickey=None, secretkey=None): diff --git a/src/volttron/server/aip.py b/src/volttron/server/aip.py index 4fd644ea9..f5c7b11f8 100644 --- a/src/volttron/server/aip.py +++ b/src/volttron/server/aip.py @@ -343,7 +343,7 @@ def set_agent_user_permissions(self, volttron_agent_user, agent_uuid, agent_dir) # VOLTTRON 8.0 once CSR is implemented for # federation and shovel. The below lines can be removed then if self.message_bus == "rmq": - os.chmod(os.path.join(cc.get_volttron_home, "certificates/private"), 0o755) + os.chmod(os.path.join(cc.get_volttron_home(), "certificates/private"), 0o755) self.set_acl_for_path( "r", volttron_agent_user, @@ -520,18 +520,12 @@ def install_agent(self, # removing last/only instance of an agent backup_agent_file = self.backup_agent_data(agent_uuid, vip_identity) - agent_name = self.install_agent_source(agent, force, pre_release) + agent_name, site_package_dir = self.install_agent_source(agent, force, pre_release) # get default vip_identity if vip_identity is not passed # default value will be in "agent_name-default-vip-id" file in site-packages dir if vip_identity is None: - # get site-packages dir using agent's package name - # installed package name is agent_name without version. Version is at the end. ex. agent-name-0.2.1 - installed_package_name = agent_name[0:agent_name.rfind("-")] - cmd = [sys.executable, "-m", "pip", "show", installed_package_name] - response = execute_command(cmd) - site_package_dir = re.search(".*\nLocation: (.*)", response).groups()[0].strip() - # get default vip id + # get default vip id if one is specified in src default_vip_id_file = os.path.join(site_package_dir, f"{agent_name}-default-vip-id") _log.info(f"Default vip id file is {default_vip_id_file}") if os.path.isfile(default_vip_id_file): @@ -621,89 +615,63 @@ def _raise_error_if_identity_exists_without_force(self, vip_identity: str, force def install_agent_source(self, agent: str, force: bool = False, pre_release: bool = False): _log.info(f"AGENT_WHEEL: {agent}") - if force: - cmd = [sys.executable, "-m", "pip", "install", "--force-reinstall"] - else: - cmd = [sys.executable, "-m", "pip", "install"] - + cmd = ["poetry", "--directory", self._server_opts.poetry_project_path.as_posix(), "add"] + # TODO - no --force-reinstall for poetry. will have to uninstall and reinstall + # if force: + # cmd.append("--force-reinstall") if pre_release: - cmd.append("--pre") + cmd.append("--allow-prereleases") cmd.append(agent) _log.debug(f"Executing agent install command : {cmd}") response = execute_command(cmd) + # if above cmd returned non-zero code it would throw exception. + # if we are here we succeeded installing some compatible version of the package. + # Now find agent name and version installed agent_name = None - last_line = response.strip().split("\n")[-1] - _log.debug(f"last line of response {last_line}") - find_success = re.match("Successfully installed (.*)", last_line) - - if find_success: - _log.debug(f"Successfully installed package: {find_success}") - # if successfully installed packages includes the agent then agent_name should be returned. - agent_name = self._get_agent_name_on_success( - find_success, self._construct_package_name_from_agent_wheel(agent)) - - # agent was not installed. Check if agent package already exists - if not agent_name: - # output in this case depends on whether you install from pypi or wheel - if agent.endswith(".whl") and os.path.isfile(agent): - # format : - # {package name} is already installed with the same version as the provided wheel. - agent_name = self._parse_wheel_install_response(response, agent) - else: - # install from pypi - output will have format - # Requirement already satisfied: {name} in ./.venv/lib/python3.8/site-packages ({version}) - agent_name = self._parse_pypi_install_response(response, agent) - return agent_name + if agent.endswith(".whl") and os.path.isfile(agent): + agent_name = self._construct_package_name_from_agent_wheel(agent) + else: + # this is a pypi package. + # if vctl install got source dir, it would have got built into a whl before getting shipped to server + # it could be just a package-name(ex. volttron-listner) + # or package-name with version constraints- ex. volttron-agent@latest, volttron-agent>=1.0.0 + # so match till we hit a character that is NOT alpha numeric character or _ or - + m = re.match("[\w\-]+", agent) + if m: + agent_name = m[0] + if agent_name is None: + # ideally we should never get here! if we get here we haven't handled some specific input format. + raise RuntimeError(f"Something went wrong when installing {agent}. " + f"Unable to parse agent name of the package.\n" + f"Agent install command used: {' '.join(cmd)} \n" + f"Command completed with exit code 0 and stdout " + f"response {response}") + + # now get the version installed, because poetry add could have been for volttron-agent@latest. + # we need to find the specific version installed + cmd = ["pip", "show", agent_name] + response = execute_command(cmd) + version = re.search(".*\nVersion: (.*)", response).groups()[0].strip() + site_package_dir = re.search(".*\nLocation: (.*)", response).groups()[0].strip() + if site_package_dir is None: + # we should not get here unless pip changed format of pip show output. + raise RuntimeError(f"Unable to find installed location of {agent} based on pip show command") + if version is None: + # we should not get here unless pip changed format of pip show output. + raise RuntimeError(f"Unable to find installed version of {agent} based on pip show command") + + return agent_name + "-" + version, site_package_dir @staticmethod def _construct_package_name_from_agent_wheel(agent_wheel): agent_name = agent_wheel if agent_wheel.endswith(".whl"): wheel = agent_wheel.split("/")[-1] - agent_name = wheel.replace("-py3-none-any.whl", "").replace("_", "-") + agent_name_with_version = wheel.replace("-py3-none-any.whl", "").replace("_", "-") + agent_name = agent_name_with_version[:agent_name_with_version.rfind("-")] return agent_name - @staticmethod - def _get_agent_name_on_success(find_success, agent_package): - _log.info(f"searching for: {agent_package} in response of successful pip install") - for package in find_success.groups()[0].split(): - # search for the agent name that we want. agent_package might not have version number - _log.debug(f"package: {package}") - if package.startswith(agent_package): - return package - # List of successfully installed packages does not include agent package. maybe we installed only - # some missing dependencies but agent package already exists. Return none - return None - - @staticmethod - def _parse_wheel_install_response(response, agent_wheel): - groups = None - result = re.search( - ".*\n(.*) is already installed with the same version as the provided wheel", response) - if result: - groups = result.groups() - if groups: - find_already_installed = groups[0].strip() - cmd = ["pip", "show", find_already_installed] - response = execute_command(cmd) - version = re.search(".*\nVersion: (.*)", response).groups()[0].strip() - return find_already_installed + "-" + version - raise ValueError(f"Couldn't install {agent_wheel}\n{response}") - - @staticmethod - def _parse_pypi_install_response(response, package_name): - _log.info(f"Parsing pypi install response {response}") - line = response.split('\r\n')[0] - match = re.match(f"Requirement already satisfied: {package_name}.*\\((.*)\\)", line) - groups = None - if match: - groups = match.groups() - if groups: - version = groups[0].strip() - return package_name + "-" + version - raise ValueError(f"Couldn't install {package_name}\n{response}") - def _setup_agent_vip_id(self, agent_name, vip_identity=None): # agent_path = os.path.join(self.install_dir, agent_name) # name = self.agent_name(agent_name) diff --git a/src/volttron/server/run_server.py b/src/volttron/server/run_server.py index 0e814f8df..54d6e05a9 100644 --- a/src/volttron/server/run_server.py +++ b/src/volttron/server/run_server.py @@ -57,7 +57,7 @@ from volttron.types.events import volttron_home_set_evnt from volttron.types.peer import ServicePeerNotifier from volttron.types.server_config import ServerConfig, ServiceConfigs -from volttron.utils import ClientContext as cc +from volttron.utils import ClientContext as cc, execute_command from volttron.utils import (get_version, store_message_bus_config) from volttron.utils.persistance import load_create_store @@ -112,16 +112,36 @@ def run_server(): opts = parser.parse_args(args) # Handle the fact that we don't use store_true and config that requires - # inverse. This is not a switch but a mode of operation so we change + # inverse. This is not a switch but a mode of operation, so we change # from the string to a boolean value here. opts.agent_isolation_mode = opts.agent_isolation_mode != 'False' - + dev_mode = opts.dev_mode # Update the server options with the command line parameter options. server_options.update(opts) server_options.store() + + # create poetry project and poetry lock file in VOLTTRON_HOME + if dev_mode: + if not os.path.isfile(server_options.poetry_project_path/"pyproject.toml"): + raise ValueError("VOLTTRON is run with --dev but unable to fund pyproject.toml is current directory - " + f"{server_options.poetry_project_path}") + else: + setup_poetry_project(server_options.poetry_project_path) + start_volttron_process(server_options) +def setup_poetry_project(volttron_home): + toml = os.path.join(volttron_home, "pyproject.toml") + if not os.path.isfile(toml): + cmd = [sys.executable, "-m", "poetry", "init", "--directory", volttron_home, + "--name", "volttron", "--author", "volttron ", "--quiet"] + execute_command(cmd) + cmd = [sys.executable, "-m", "pip", "list", "--format", "freeze", "|", "grep", "-v", "volttron==", + "|", "xargs", "poetry", "add", "--directory", volttron_home] + execute_command(cmd) + + def start_volttron_process(options: ServerOptions): opts = options # Change working dir @@ -615,6 +635,12 @@ def build_arg_parser(options: ServerOptions) -> argparse.ArgumentParser: help=argparse.SUPPRESS) parser.add_argument("--auth-disabled", action="store_false", help=argparse.SUPPRESS) parser.add_argument("--show-config", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--dev", + action="store_true", + dest="dev_mode", + help="development mode with poetry environment to build volttron libraries from source/pypi" + ) + parser.add_help_argument() parser.add_version_argument(version="%(prog)s " + str(get_version())) diff --git a/src/volttron/server/server_options.py b/src/volttron/server/server_options.py index 5956d2c00..d5f330b93 100644 --- a/src/volttron/server/server_options.py +++ b/src/volttron/server/server_options.py @@ -63,6 +63,7 @@ class ServerOptions: config_file: Path = None initialized: bool = False service_address: str = None + poetry_project_path: Path = None # services: list[ServiceData] = field(default_factory=list) @@ -78,6 +79,9 @@ def __post_init__(self): if self.volttron_home is None: self.volttron_home = Path(os.environ.get("VOLTTRON_HOME", "~/.volttron")).expanduser() + if self.poetry_project_path is None: + self.poetry_project_path = self.volttron_home + # Should be the only location where we create VOLTTRON_HOME if not self.volttron_home.is_dir(): self.volttron_home.mkdir(mode=0o755, exist_ok=True, parents=True) @@ -117,10 +121,17 @@ def update(self, opts: argparse.Namespace | dict): """ address = set(opts.address) opts.address = list(address) + if isinstance(opts, dict): + dev_mode = opts.pop("dev_mode") self.__dict__.update(opts) else: - self.__dict__.update(opts.__dict__) + d = opts.__dict__ + dev_mode = d.pop("dev_mode") + self.__dict__.update(d) + + if dev_mode: + self.poetry_project_path = Path(os.path.abspath(os.curdir)) def store(self, file: Path = None): """ @@ -141,8 +152,8 @@ def store(self, file: Path = None): try: # Don't save volttron_home within the config file. if field.name not in ('volttron_home', 'services', 'config_file', 'initialized', - 'service_address'): - # More than one address can be present so we must be careful + 'service_address', "poetry_project_path"): + # More than one address can be present, so we must be careful # with it. if field.name == 'address': found = set()