diff --git a/azdev.pyproj b/azdev.pyproj index 6d40c214..b4e17809 100644 --- a/azdev.pyproj +++ b/azdev.pyproj @@ -49,6 +49,7 @@ + Code diff --git a/azdev/help.py b/azdev/help.py index 9b1b21c9..34bc60f4 100644 --- a/azdev/help.py +++ b/azdev/help.py @@ -15,20 +15,31 @@ helps['setup'] = """ short-summary: Set up your environment for development of Azure CLI command modules and/or extensions. examples: - - name: Fully interactive setup. + - name: Fully interactive setup (Must be run in an existing virtual environment). text: azdev setup - - name: Install only the CLI in dev mode and search for the existing repo. - text: azdev setup -c + - name: Install CLI and setup an extensions repo in an existing virtual environment. Will create a azure directory and config in the current virtual environment. + Note the existing virtual environment could created by VENV or PYENV. + text: azdev setup -c azure-cli -r azure-cli-extensions - - name: Install public CLI and setup an extensions repo. Do not install any extensions. + - name: Same as above, but install the `alias` extension in the existing virtual environment too. + text: azdev setup -c azure-cli -r azure-cli-extensions -e alias + + - name: Same as above, but will use the CLI repo path in local .azdev config, or the one in global .azdev config if not found the local one. text: azdev setup -r azure-cli-extensions - - name: Install CLI in dev mode, along with the extensions repo. Auto-find the CLI repo and install the `alias` extension in dev mode. - text: azdev setup -c -r azure-cli-extensions -e alias + - name: Same as above, but only install CLI without setup an extensions repo. + text: azdev setup -c azure-cli + + - name: Install CLI and setup an extensions repo in a new virtual environment. Will create a azure directory and config in the current virtual environment. + Note -s is using VENV to create a new virtual environment, should un-install PYENV if you have. + text: azdev setup -c azure-cli -r azure-cli-extensions -s env1 + + - name: Same as above, but do not setup new azure directory and config in this virtual environment + text: azdev setup -c azure-cli -r azure-cli-extensions -s env1 -g - - name: Install only the CLI in dev mode and resolve dependencies from setup.py. - text: azdev setup -c -d setup.py + - name: Same as above, but copy over system level azure settings into new virtual environment azure settings + text: azdev setup -c azure-cli -r azure-cli-extensions -s env1 --copy """ diff --git a/azdev/operations/help/__init__.py b/azdev/operations/help/__init__.py index 30c5af45..0f216224 100644 --- a/azdev/operations/help/__init__.py +++ b/azdev/operations/help/__init__.py @@ -20,7 +20,8 @@ from azure.cli.core.extension.operations import list_available_extensions, list_extensions as list_cli_extensions # pylint: disable=import-error from azdev.utilities import ( display, heading, subheading, - get_cli_repo_path, get_path_table + get_cli_repo_path, get_path_table, + require_virtual_env ) from azdev.utilities.tools import require_azure_cli diff --git a/azdev/operations/pypi.py b/azdev/operations/pypi.py index 55009f47..c58e3a8c 100644 --- a/azdev/operations/pypi.py +++ b/azdev/operations/pypi.py @@ -16,7 +16,8 @@ from azdev.utilities import ( display, heading, subheading, cmd, py_cmd, get_path_table, - pip_cmd, COMMAND_MODULE_PREFIX, require_azure_cli, find_files) + pip_cmd, COMMAND_MODULE_PREFIX, require_azure_cli, require_virtual_env, + find_files) logger = get_logger(__name__) @@ -131,6 +132,7 @@ def verify_versions(): import tempfile import shutil + require_virtual_env() require_azure_cli() heading('Verify CLI Versions') diff --git a/azdev/operations/setup.py b/azdev/operations/setup.py index 87aa4968..6f8921a9 100644 --- a/azdev/operations/setup.py +++ b/azdev/operations/setup.py @@ -6,7 +6,9 @@ import os from shutil import copytree, rmtree +import shutil import time +import sys from knack.log import get_logger from knack.util import CLIError @@ -14,9 +16,11 @@ from azdev.operations.extensions import ( list_extensions, add_extension_repo, remove_extension) from azdev.params import Flag +import azdev.utilities.const as const +import azdev.utilities.venv as venv from azdev.utilities import ( - display, heading, subheading, pip_cmd, find_file, - get_azdev_config_dir, get_azdev_config, require_virtual_env, get_azure_config) + display, heading, subheading, pip_cmd, find_file, get_env_path, + get_azdev_config_dir, get_azdev_config, get_azure_config, shell_cmd) logger = get_logger(__name__) @@ -196,8 +200,8 @@ def add_ext_repo(path): # repo directory. To use multiple extension repos or identify a repo outside the cwd, they must specify # the path. if prompt_y_n('\nDo you plan to develop CLI extensions?'): - display('\nGreat! Input the paths for the extension repos you wish to develop for, one per ' - 'line. You can add as many repos as you like. (TIP: to quickly get started, press RETURN to ' + display('\nGreat! Input the path for the extension repos you wish to develop for. ' + '(TIP: to quickly get started, press RETURN to ' 'use your current working directory).') first_repo = True while True: @@ -245,14 +249,170 @@ def add_ext_repo(path): raise CLIError('Installation aborted.') -def setup(cli_path=None, ext_repo_path=None, ext=None, deps=None): +def _validate_input(cli_path, ext_repo_path, set_env, copy, use_global, ext): + if copy and use_global: + raise CLIError("Copy and use global are mutally exlcusive.") + if cli_path == "pypi" and any([use_global, copy, set_env]): + raise CLIError("pypi for cli path is mutally exlcusive with global copy and set env") + if not cli_path and any([use_global, copy, set_env]): + raise CLIError("if global, copy, or set env are set then both an extensions repo " + " and a cli repo must be specified") + if not ext_repo_path and ext: + raise CLIError("Extesions provided to be installed but no extensions path was given") - require_virtual_env() - start = time.time() +def _check_paths(cli_path, ext_repo_path): + if not os.path.isdir(cli_path): + raise CLIError("The cli path is not a valid directory, please check the path") + if ext_repo_path and not os.path.isdir(ext_repo_path): + raise CLIError("The cli extensions path is not a valid directory, please check the path") + + +def _check_shell(): + if 'SHELL' in os.environ and const.IS_WINDOWS and 'bash.exe' in os.environ['SHELL']: + heading("WARNING: You are running bash in Windows, the setup may not work correctly and " + "command may have unexpected behavior") + from knack.prompting import prompt_y_n + if not prompt_y_n('Would you like to continue with the install?'): + sys.exit(0) + + +def _check_env(set_env): + if not set_env: + if not get_env_path(): + raise CLIError('You are not running in a virtual enviroment and have not chosen to set one up.') + _check_pyenv() + elif 'VIRTUAL_ENV' in os.environ: + raise CLIError("You are already running in a virtual enviroment, yet you want to set a new one up") + + +def _check_pyenv(): + if 'PYENV_VIRTUAL_ENV' in os.environ: + if const.IS_WINDOWS: + raise CLIError('AZDEV does not support setup in a pyenv-win virtual environment.') + activate_path = os.path.join( + os.environ['PYENV_ROOT'], 'plugins', 'pyenv-virtualenv', 'bin', 'pyenv-sh-activate') + venv.edit_pyenv_activate(activate_path) + + +def setup(cli_path=None, ext_repo_path=None, ext=None, deps=None, set_env=None, copy=None, use_global=None): + _check_env(set_env) + + _check_shell() heading('Azure CLI Dev Setup') + # cases for handling legacy install + if not any([cli_path, ext_repo_path]) or cli_path == "pypi": + display("WARNING: Installing azdev in legacy mode. Run with atleast -c " + "to install the latest azdev wihout \"pypi\"\n") + return _handle_legacy(cli_path, ext_repo_path, ext, deps, time.time()) + if 'CONDA_PREFIX' in os.environ: + raise CLIError('CONDA virutal enviroments are not supported outside' + ' of interactive mode or when -c and -r are provided') + + if not cli_path: + cli_path = _handle_no_cli_path() + + _validate_input(cli_path, ext_repo_path, set_env, copy, use_global, ext) + _check_paths(cli_path, ext_repo_path) + + if set_env: + shell_cmd((const.VENV_CMD if const.IS_WINDOWS else const.VENV_CMD3) + set_env, raise_ex=False) + azure_path = os.path.join(os.path.abspath(os.getcwd()), set_env) + else: + azure_path = os.environ.get('VIRTUAL_ENV') + + dot_azure_config = os.path.join(azure_path, '.azure') + dot_azdev_config = os.path.join(azure_path, '.azdev') + + # clean up venv dirs if they already existed + # and this is a reinstall/new setup + if os.path.isdir(dot_azure_config): + shutil.rmtree(dot_azure_config) + if os.path.isdir(dot_azdev_config): + shutil.rmtree(dot_azdev_config) + + global_az_config = os.path.expanduser(os.path.join('~', '.azure')) + global_azdev_config = os.path.expanduser(os.path.join('~', '.azdev')) + azure_config_path = os.path.join(dot_azure_config, const.CONFIG_NAME) + azdev_config_path = os.path.join(dot_azdev_config, const.CONFIG_NAME) + + if os.path.isdir(global_az_config) and copy: + shutil.copytree(global_az_config, dot_azure_config) + if os.path.isdir(global_azdev_config): + shutil.copytree(global_azdev_config, dot_azdev_config) + else: + os.mkdir(dot_azdev_config) + file = open(azdev_config_path, "w") + file.close() + elif not use_global and not copy: + os.mkdir(dot_azure_config) + os.mkdir(dot_azdev_config) + file_az, file_dev = open(azure_config_path, "w"), open(azdev_config_path, "w") + file_az.close() + file_dev.close() + elif os.path.isdir(global_az_config): + dot_azure_config, dot_azdev_config = global_az_config, global_azdev_config + azure_config_path = os.path.join(dot_azure_config, const.CONFIG_NAME) + else: + raise CLIError( + "Global AZ config is not set up, yet it was specified to be used.") + + # set env vars for get azure config and get azdev config + os.environ['AZURE_CONFIG_DIR'], os.environ['AZDEV_CONFIG_DIR'] = dot_azure_config, dot_azdev_config + config = get_azure_config() + if not config.get('cloud', 'name', None): + config.set_value('cloud', 'name', 'AzureCloud') + if ext_repo_path: + config.set_value(const.EXT_SECTION, const.AZ_DEV_SRC, os.path.abspath(ext_repo_path)) + venv.edit_activate(azure_path, dot_azure_config, dot_azdev_config) + if cli_path: + config.set_value('clipath', const.AZ_DEV_SRC, os.path.abspath(cli_path)) + venv.install_cli(os.path.abspath(cli_path), azure_path) + config = get_azdev_config() + config.set_value('ext', 'repo_paths', os.path.abspath(ext_repo_path) if ext_repo_path else '_NONE_') + config.set_value('cli', 'repo_path', os.path.abspath(cli_path)) + _copy_config_files() + if ext and ext_repo_path: + venv.install_extensions(azure_path, ext) + + if not set_env: + heading("The setup was successful! Please run or re-run the virtual environment activation script.") + else: + heading("The setup was successful!") + return None + + +def _get_azdev_cli_path(config_file_path): + if not os.path.exists(config_file_path): + return None + + import configparser + with open(config_file_path, "r") as file: + config_parser = configparser.RawConfigParser() + config_parser.read_string(file.read()) + if config_parser.has_section('cli') and config_parser.has_option('cli', 'repo_path'): + return config_parser.get('cli', 'repo_path') + return None + + +def _handle_no_cli_path(): + local_azdev_config = os.path.join(os.environ.get('VIRTUAL_ENV'), '.azdev', const.CONFIG_NAME) + cli_path = _get_azdev_cli_path(local_azdev_config) + if cli_path is None: + display('Not found cli path in local azdev config file: ' + local_azdev_config) + display('Will use the one in global azdev config.') + global_azdev_config = os.path.expanduser(os.path.join('~', '.azdev', const.CONFIG_NAME)) + cli_path = _get_azdev_cli_path(global_azdev_config) + if cli_path is None: + raise CLIError('Not found cli path in global azdev config file: ' + global_azdev_config) + display('cli_path: ' + cli_path) + return cli_path + + +def _handle_legacy(cli_path, ext_repo_path, ext, deps, start): + ext_repo_path = [ext_repo_path] if ext_repo_path else None ext_to_install = [] if not any([cli_path, ext_repo_path, ext]): cli_path, ext_repo_path, ext_to_install = _interactive_setup() @@ -279,7 +439,6 @@ def setup(cli_path=None, ext_repo_path=None, ext=None, deps=None): # must add the necessary repo to add an extension if ext and not ext_repo_path: raise CLIError('usage error: --repo EXT_REPO [EXT_REPO ...] [--ext EXT_NAME ...]') - get_azure_config().set_value('extension', 'dev_sources', '') if ext_repo_path: # add extension repo(s) @@ -313,11 +472,10 @@ def setup(cli_path=None, ext_repo_path=None, ext=None, deps=None): # upgrade to latest pip pip_cmd('install --upgrade pip -q', 'Upgrading pip...') - _install_cli(cli_path, deps=deps) - _install_extensions(ext_to_install) + if ext_repo_path: + _install_extensions(ext_to_install) _copy_config_files() - end = time.time() elapsed_min = int((end - start) / 60) elapsed_sec = int(end - start) % 60 diff --git a/azdev/operations/style.py b/azdev/operations/style.py index 3c0c1835..24bd6be9 100644 --- a/azdev/operations/style.py +++ b/azdev/operations/style.py @@ -201,7 +201,7 @@ def _config_file_path(style_type="pylint"): ext_repo_path = filter( lambda x: "azure-cli-extension" in x, - get_azdev_config().get("ext", "repo_paths").split(), + get_azdev_config().get("ext", "repo_paths").split(','), ) try: ext_repo_path = next(ext_repo_path) diff --git a/azdev/params.py b/azdev/params.py index 4b7cafde..71afac94 100644 --- a/azdev/params.py +++ b/azdev/params.py @@ -34,10 +34,13 @@ def load_arguments(self, _): c.argument('git_repo', options_list='--repo', arg_group='Git', help='Path to the Git repo to check.') with ArgumentsContext(self, 'setup') as c: - c.argument('cli_path', options_list=['--cli', '-c'], nargs='?', const=Flag, help="Path to an existing Azure CLI repo. Omit value to search for the repo or use special value 'EDGE' to install the latest developer edge build.") - c.argument('ext_repo_path', options_list=['--repo', '-r'], nargs='+', help='Space-separated list of paths to existing Azure CLI extensions repos.') + c.argument('cli_path', options_list=['--cli', '-c'], type=str, help="Path to an existing Azure CLI repo. Use special value 'EDGE' to install the latest developer edge build. Note: if not provide, will use the one in local .azdev config, if not exist will use the one in global .azdev config.") + c.argument('ext_repo_path', options_list=['--repo', '-r'], type=str, help='Path to existing Azure CLI extensions repos.') c.argument('ext', options_list=['--ext', '-e'], nargs='+', help="Space-separated list of extensions to install initially. Use '*' to install all extensions.") c.argument('deps', options_list=['--deps-from', '-d'], choices=['requirements.txt', 'setup.py'], default='requirements.txt', help="Choose the file to resolve dependencies.") + c.argument('set_env', options_list=['--set-env', '-s'], type=str, help="Will create a virtual enviroment with the given env name") + c.argument('copy', options_list='--copy', action='store_true', help="Will copy entire global .azure diretory to the newly created virtual enviroment .azure direcotry if it exist") + c.argument('use_global', options_list=['--use-global', '-g'], action='store_true', help="Will use the default global system .azure config") with ArgumentsContext(self, 'test') as c: c.argument('discover', options_list='--discover', action='store_true', help='Build an index of test names so that you don\'t need to specify fully qualified test paths.') diff --git a/azdev/utilities/__init__.py b/azdev/utilities/__init__.py index 40f76e1b..f16e59e2 100644 --- a/azdev/utilities/__init__.py +++ b/azdev/utilities/__init__.py @@ -14,7 +14,8 @@ call, cmd, py_cmd, - pip_cmd + pip_cmd, + shell_cmd ) from .const import ( COMMAND_MODULE_PREFIX, @@ -22,8 +23,7 @@ IS_WINDOWS, ENV_VAR_TEST_MODULES, ENV_VAR_TEST_LIVE, - ENV_VAR_VIRTUAL_ENV, - EXT_REPO_NAME + ENV_VAR_VIRTUAL_ENV ) from .display import ( display, @@ -76,7 +76,6 @@ 'ENV_VAR_TEST_MODULES', 'ENV_VAR_TEST_LIVE', 'ENV_VAR_VIRTUAL_ENV', - 'EXT_REPO_NAME', 'IS_WINDOWS', 'extract_module_name', 'find_file', diff --git a/azdev/utilities/command.py b/azdev/utilities/command.py index 55d3a1da..cc55b289 100644 --- a/azdev/utilities/command.py +++ b/azdev/utilities/command.py @@ -9,7 +9,7 @@ import sys from knack.log import get_logger -from knack.util import CommandResultItem +from knack.util import CommandResultItem, CLIError logger = get_logger(__name__) @@ -36,8 +36,7 @@ def cmd(command, message=False, show_stderr=True, **kwargs): :param kwargs: Any kwargs supported by subprocess.Popen :returns: CommandResultItem object. """ - from azdev.utilities import IS_WINDOWS, display - + from . import IS_WINDOWS, display # use default message if custom not provided if message is True: message = 'Running: {}\n'.format(command) @@ -56,6 +55,34 @@ def cmd(command, message=False, show_stderr=True, **kwargs): return CommandResultItem(err.output, exit_code=err.returncode, error=err) +def shell_cmd(command, message=False, stderr=None, stdout=None, check=True, raise_ex=True, timeout=None, + executable=None, capture_output=False): + + # use default message if custom not provided + if message is True: + message = '\nRunning: {}\n'.format(command) + from . import display + if message: + display(message) + + try: + output = subprocess.run(command, + stdout=subprocess.PIPE if capture_output else stdout, + stderr=subprocess.PIPE if capture_output else stderr, + check=check, + timeout=timeout, + executable=executable, + shell=True) + if capture_output: + return CommandResultItem(output.stdout.decode('utf-8').strip(), exit_code=0, error=None) + except subprocess.CalledProcessError as err: + if raise_ex: + raise err + logger.debug(err) + raise CLIError("Command " + command + " failed. Trying running with --debug for more info") + return None + + def py_cmd(command, message=False, show_stderr=True, is_module=True, **kwargs): """ Run a script or command with Python. diff --git a/azdev/utilities/config.py b/azdev/utilities/config.py index 69a5d0e3..7b5963f7 100644 --- a/azdev/utilities/config.py +++ b/azdev/utilities/config.py @@ -5,7 +5,7 @@ # ----------------------------------------------------------------------------- import os - +import sys from knack.config import CLIConfig @@ -21,7 +21,11 @@ def get_azdev_config_dir(): """ Returns the user's .azdev directory. """ from azdev.utilities import get_env_path env_name = None - _, env_name = os.path.splitdrive(get_env_path()) + if not get_env_path(): + _, env_name = os.path.splitdrive(sys.executable) + else: + _, env_name = os.path.splitdrive(get_env_path()) + azdev_dir = os.getenv('AZDEV_CONFIG_DIR', None) or os.path.expanduser(os.path.join('~', '.azdev')) if not env_name: return azdev_dir diff --git a/azdev/utilities/const.py b/azdev/utilities/const.py index d0db7900..607a2ee2 100644 --- a/azdev/utilities/const.py +++ b/azdev/utilities/const.py @@ -6,10 +6,21 @@ import sys +AZ_AZDEV_DIR = 'AZDEV_CONFIG_DIR' +AZ_CONFIG_DIR = 'AZURE_CONFIG_DIR' +AZ_DEV_SRC = 'dev_sources' COMMAND_MODULE_PREFIX = 'azure-cli-' +CONFIG_NAME = 'config' EXTENSION_PREFIX = 'azext_' -EXT_REPO_NAME = 'azure-cli-extensions' +EXT_SECTION = 'extension' IS_WINDOWS = sys.platform.lower() in ['windows', 'win32'] +PIP_E_CMD = 'pip install -e ' +PIP_R_CMD = 'pip install -r ' +UN_ACTIVATE = 'activate' +UN_BIN = 'bin' +UN_EXPORT = 'export' +VENV_CMD = 'python -m venv --system-site-packages ' +VENV_CMD3 = 'python3 -m venv --system-site-packages ' ENV_VAR_TEST_MODULES = 'AZDEV_TEST_TESTS' # comma-separated list of modules to test ENV_VAR_VIRTUAL_ENV = ['VIRTUAL_ENV', 'CONDA_PREFIX'] # used by system to identify virtual environment diff --git a/azdev/utilities/path.py b/azdev/utilities/path.py index 1f15c7d3..ae30620b 100644 --- a/azdev/utilities/path.py +++ b/azdev/utilities/path.py @@ -5,11 +5,12 @@ # ----------------------------------------------------------------------------- import os -from glob import glob +from glob import glob from knack.util import CLIError - -from .const import COMMAND_MODULE_PREFIX, EXTENSION_PREFIX, ENV_VAR_VIRTUAL_ENV +from six.moves import configparser +from azdev.utilities import get_azure_config, display +from . import const def extract_module_name(path): @@ -34,7 +35,7 @@ def get_env_path(): :returns: Path (str) to the virtual env or None. """ env_path = None - for item in ENV_VAR_VIRTUAL_ENV: + for item in const.ENV_VAR_VIRTUAL_ENV: env_path = os.environ.get(item) if env_path: break @@ -70,12 +71,15 @@ def get_ext_repo_paths(): :returns: Path (str) to Azure CLI dev extension repos. """ - from configparser import NoSectionError - from .config import get_azdev_config + from configparser import NoSectionError, NoOptionError try: - return get_azdev_config().get('ext', 'repo_paths').split(',') + return get_azure_config().get(const.EXT_SECTION, const.AZ_DEV_SRC).split(',') except NoSectionError: - raise CLIError('Unable to retrieve extensions repo path from config. Please run `azdev setup`.') + raise CLIError('Unable to retrieve extensions repo path from config. Please run `azdev setup` ' + 'with -r to set an extensions repo.') + except NoOptionError: + raise CLIError('Unable to retrieve the option {} from azure config section [{}]'.format( + const.AZ_DEV_SRC, const.EXT_SECTION)) def find_file(file_name): @@ -119,11 +123,14 @@ def make_dirs(path): def get_name_index(invert=False, include_whl_extensions=False): """ Returns a dictionary containing the long and short names of modules and extensions is {SHORT:LONG} format or {LONG:SHORT} format when invert=True. """ - from azure.cli.core.extension import EXTENSIONS_DIR # pylint: disable=import-error + config = get_azure_config() # pylint: disable=import-error + try: + EXTENSIONS_DIR = config.get(const.EXT_SECTION, const.AZ_DEV_SRC) + except (configparser.NoSectionError, configparser.NoOptionError): + EXTENSIONS_DIR = "" table = {} cli_repo_path = get_cli_repo_path() - ext_repo_paths = get_ext_repo_paths() # unified azure-cli package (2.0.68 and later) paths = os.path.normcase( @@ -133,9 +140,9 @@ def get_name_index(invert=False, include_whl_extensions=False): ) modules_paths = glob(paths) core_paths = glob(os.path.normcase(os.path.join(cli_repo_path, 'src', '*', 'setup.py'))) - ext_paths = [x for x in find_files(ext_repo_paths, '*.*-info') if 'site-packages' not in x] + ext_paths = [x for x in find_files(EXTENSIONS_DIR, '*.*-info') if 'site-packages' not in x] whl_ext_paths = [] - if include_whl_extensions: + if include_whl_extensions and EXTENSIONS_DIR: whl_ext_paths = [x for x in find_files(EXTENSIONS_DIR, '*.*-info') if 'site-packages' not in x] def _update_table(paths, key): @@ -149,15 +156,15 @@ def _update_table(paths, key): if key == 'ext': short_name = base_name for item in os.listdir(folder): - if item.startswith(EXTENSION_PREFIX): + if item.startswith(const.EXTENSION_PREFIX): long_name = item break - elif base_name.startswith(COMMAND_MODULE_PREFIX): + elif base_name.startswith(const.COMMAND_MODULE_PREFIX): long_name = base_name - short_name = base_name.replace(COMMAND_MODULE_PREFIX, '') or '__main__' + short_name = base_name.replace(const.COMMAND_MODULE_PREFIX, '') or '__main__' else: short_name = base_name - long_name = '{}{}'.format(COMMAND_MODULE_PREFIX, base_name) + long_name = '{}{}'.format(const.COMMAND_MODULE_PREFIX, base_name) if not invert: table[short_name] = long_name else: @@ -190,8 +197,14 @@ def get_path_table(include_only=None, include_whl_extensions=False): } } """ - from azure.cli.core.extension import EXTENSIONS_DIR # pylint: disable=import-error - + config = get_azure_config() # pylint: disable=import-error + try: + EXTENSIONS_DIR = config.get(const.EXT_SECTION, const.AZ_DEV_SRC) + os.chdir(EXTENSIONS_DIR) + except (configparser.NoSectionError, configparser.NoOptionError, TypeError): + display("WARNING: No extension path found, only modules will be available. " + "rerun setup with -r to make extensions available\n") + EXTENSIONS_DIR = "" # determine whether the call will filter or return all if isinstance(include_only, str): include_only = [include_only] @@ -199,7 +212,6 @@ def get_path_table(include_only=None, include_whl_extensions=False): table = {} cli_repo_path = get_cli_repo_path() - ext_repo_paths = get_ext_repo_paths() paths = os.path.normcase( os.path.join( @@ -208,7 +220,7 @@ def get_path_table(include_only=None, include_whl_extensions=False): ) modules_paths = glob(paths) core_paths = glob(os.path.normcase(os.path.join(cli_repo_path, 'src', '*', 'setup.py'))) - ext_paths = [x for x in find_files(ext_repo_paths, '*.*-info') if 'site-packages' not in x] + ext_paths = [x for x in find_files(EXTENSIONS_DIR, '*.*-info') if 'site-packages' not in x] whl_ext_paths = [x for x in find_files(EXTENSIONS_DIR, '*.*-info') if 'site-packages' not in x] def _update_table(package_paths, key): @@ -221,10 +233,10 @@ def _update_table(package_paths, key): if key == 'ext': short_name = base_name - long_name = next((item for item in os.listdir(folder) if item.startswith(EXTENSION_PREFIX)), None) + long_name = next((item for item in os.listdir(folder) if item.startswith(const.EXTENSION_PREFIX)), None) else: short_name = base_name - long_name = '{}{}'.format(COMMAND_MODULE_PREFIX, base_name) + long_name = '{}{}'.format(const.COMMAND_MODULE_PREFIX, base_name) if get_all: table[key][long_name if key == 'ext' else short_name] = folder diff --git a/azdev/utilities/tools.py b/azdev/utilities/tools.py index 47396a75..2fbb1b52 100644 --- a/azdev/utilities/tools.py +++ b/azdev/utilities/tools.py @@ -4,7 +4,9 @@ # license information. # ----------------------------------------------------------------------------- +import os from knack.util import CLIError +from . import const def require_virtual_env(): @@ -13,6 +15,12 @@ def require_virtual_env(): env = get_env_path() if not env: raise CLIError('This command can only be run from an active virtual environment.') + if not os.environ.get(const.AZ_CONFIG_DIR): + raise CLIError( + "AZURE_CONFIG_DIR env var is not set. Please run 'azdev setup'") + if not os.path.exists(os.path.join(os.environ[const.AZ_CONFIG_DIR], "config")): + raise CLIError( + "The Azure config file does not exist. Please run 'azdev setup'") def require_azure_cli(): diff --git a/azdev/utilities/venv.py b/azdev/utilities/venv.py new file mode 100644 index 00000000..67285bec --- /dev/null +++ b/azdev/utilities/venv.py @@ -0,0 +1,152 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ----------------------------------------------------------------------------- + +import os +import subprocess +import platform +from knack.util import CLIError +import azdev.operations.extensions +from azdev.utilities import display, shell_cmd +from . import const + + +def edit_activate(azure_config_path, dot_azure_config, dot_azdev_config): + if const.IS_WINDOWS: + ps1_edit(azure_config_path, dot_azure_config, dot_azdev_config) + bat_edit(azure_config_path, dot_azure_config, dot_azdev_config) + else: + unix_edit(azure_config_path, dot_azure_config, dot_azdev_config) + + +def edit_pyenv_activate(activate_path): + set_azdev_content = 'set -gx AZDEV_CONFIG_DIR "${prefix}/.azdev";\n' + set_azure_content = 'set -gx AZURE_CONFIG_DIR "${prefix}/.azure";\n' + export_azdev_content = 'export AZDEV_CONFIG_DIR="${prefix}/.azdev";\n' + export_azure_content = 'export AZURE_CONFIG_DIR="${prefix}/.azure";\n' + + insert(activate_path, set_azdev_content, get_line_num(activate_path, "set -gx VIRTUAL_ENV \"${prefix}\";")) + insert(activate_path, set_azure_content, get_line_num(activate_path, "set -gx VIRTUAL_ENV \"${prefix}\";")) + insert(activate_path, export_azdev_content, get_line_num(activate_path, "export VIRTUAL_ENV=\"${prefix}\";")) + insert(activate_path, export_azure_content, get_line_num(activate_path, "export VIRTUAL_ENV=\"${prefix}\";")) + + +def insert(file_path, content, line_num): + if line_num == -1: + raise CLIError("Cannot find a proper place to set/export environment variable.") + if get_line_num(file_path, content) == -1: + with open(file_path, 'r+') as reader: + file_content = reader.readlines() + file_content.insert(line_num, content) + reader.seek(0) + reader.writelines(file_content) + + +def get_line_num(file_path, target): + with open(file_path, 'r') as reader: + for num, line in enumerate(reader, 1): + if target in line: + return num + return -1 + + +def unix_edit(azure_config_path, dot_azure_config, dot_azdev_config): + activate_path = os.path.join(azure_config_path, const.UN_BIN, + const.UN_ACTIVATE) + content = open(activate_path, "r").readlines() + + # check if already ran setup before + if const.AZ_CONFIG_DIR not in content[0]: + content = [const.AZ_CONFIG_DIR + '=' + dot_azure_config + '\n', + const.UN_EXPORT + ' ' + const.AZ_CONFIG_DIR + '\n', + const.AZ_AZDEV_DIR + '=' + dot_azdev_config + '\n', + const.UN_EXPORT + ' ' + const.AZ_AZDEV_DIR + '\n'] + content + with open(activate_path, "w") as file: + file.writelines(content) + + +def bat_edit(azure_config_path, dot_azure_config, dot_azdev_config): + activate_path = os.path.join(azure_config_path, 'Scripts', + 'activate.bat') + content = open(activate_path, "r").readlines() + if const.AZ_CONFIG_DIR not in content[1]: + content = content[0:1] + ['set ' + const.AZ_CONFIG_DIR + + '=' + dot_azure_config + '\n', + 'set ' + const.AZ_AZDEV_DIR + + '=' + dot_azdev_config] + content[1::] + with open(activate_path, "w") as file: + file.writelines(content) + + +def ps1_edit(azure_config_path, dot_azure_config, dot_azdev_config): + activate_path = os.path.join(azure_config_path, 'Scripts', + 'Activate.ps1') + content = open(activate_path, "r").read() + idx = content.find('$env:VIRTUAL_ENV') + if idx < 0: + raise CLIError("hmm, it looks like Activate.ps1 does" + " not set the virutal enviroment variable VIRTUAL_ENV") + if content.find('$env:AZURE_CONFIG_DIR') < 0: + content = content[:idx] + '$env:AZURE_CONFIG_DIR' + " = " + \ + "\"" + dot_azure_config + "\"; " + \ + "$env:AZDEV_CONFIG_DIR = " + \ + "\"" + dot_azdev_config + "\"; " + \ + content[idx:] + with open(activate_path, "w") as file: + file.write(content) + + +def install_cli(cli_path, venv_path): + src_path = os.path.join(cli_path, 'src') + activate_path = (os.path.join(venv_path, 'Scripts', 'activate') + if const.IS_WINDOWS else 'source ' + os.path.join(venv_path, const.UN_BIN, const.UN_ACTIVATE)) + delimiter = ' && ' if const.IS_WINDOWS else '; ' + executable = None if const.IS_WINDOWS else '/bin/bash' + display("\nvenv activate path is " + str(activate_path)) + shell_cmd(activate_path + delimiter + 'pip install --ignore-installed azure-common', + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, raise_ex=False, executable=executable) + display("\nInstalling telemetry ") + shell_cmd(activate_path + delimiter + const.PIP_E_CMD + os.path.join(src_path, 'azure-cli-telemetry'), + stdout=subprocess.DEVNULL, raise_ex=False, stderr=subprocess.DEVNULL, executable=executable) + display("\nInstalling core ") + shell_cmd(activate_path + delimiter + const.PIP_E_CMD + os.path.join(src_path, 'azure-cli-core'), + stdout=subprocess.DEVNULL, raise_ex=False, stderr=subprocess.DEVNULL, executable=executable) + shell_cmd(activate_path + delimiter + const.PIP_E_CMD + os.path.join(src_path, 'azure-cli-testsdk'), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, raise_ex=False, executable=executable) + display("\nInstalling cli ") + shell_cmd(activate_path + delimiter + const.PIP_E_CMD + os.path.join(src_path, 'azure-cli'), + raise_ex=False, executable=executable) + req_file = 'requirements.py3.{}.txt'.format(platform.system().lower() if const.IS_WINDOWS else platform.system()) + req_file = "{}/src/azure-cli/{}".format(cli_path, req_file) + display("Installing " + req_file) + shell_cmd(activate_path + delimiter + const.PIP_R_CMD + req_file, raise_ex=False, executable=executable) + + +def install_extensions(venv_path, extensions): + activate_path = os.path.join(venv_path, 'Scripts', 'activate') if const.IS_WINDOWS else 'source ' + os.path.join( + venv_path, const.UN_BIN, const.UN_ACTIVATE) + delimiter = ' && ' if const.IS_WINDOWS else '; ' + executable = None if const.IS_WINDOWS else '/bin/bash' + all_ext = azdev.operations.extensions.list_extensions() + if extensions == ['*']: + display("\nInstalling all extensions") + for i in all_ext: + shell_cmd(activate_path + delimiter + const.PIP_E_CMD + i['path'], executable=executable) + extensions = False + else: + display("\nInstalling the following extensions: " + str(extensions)) + extensions = set(extensions) + k = 0 + while k < len(all_ext) and extensions: + if all_ext[k]['name'] in extensions: + shell_cmd(activate_path + delimiter + const.PIP_E_CMD + all_ext[k]['path'], + executable=executable) + extensions.remove(all_ext[k]['name']) + k += 1 + if extensions: + raise CLIError("The following extensions were not found. Ensure you have added " + "the repo using `--repo/-r PATH`.\n {}".format('\n '.join(extensions))) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 33877a5b..adf8e3cf 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -122,9 +122,6 @@ jobs: - bash: | set -ev - python -m venv env - . env/bin/activate - # install azdev from source code pip install -e . pip install pytest-cov @@ -132,8 +129,8 @@ jobs: # azdev setup git clone --quiet https://github.com/Azure/azure-cli.git git clone --quiet https://github.com/Azure/azure-cli-extensions.git - azdev setup -c ./azure-cli -r ./azure-cli-extensions - + azdev setup -c ./azure-cli -r ./azure-cli-extensions -s env + source env/bin/activate azdev --version python -m pytest azdev/ --ignore=azdev/mod_templates --junitxml=junit/test-results.xml --cov=azdev --cov-report=xml @@ -176,6 +173,7 @@ jobs: set -ev . scripts/ci/install.sh # verify azdev setup worked + source env/bin/activate az -h displayName: 'Test azdev setup' @@ -240,7 +238,7 @@ jobs: set -ev . scripts/ci/install.sh # verify azdev linter works - azdev linter acr + azdev linter CLI displayName: 'Test azdev linter' - job: TestAzdevStyle @@ -272,6 +270,7 @@ jobs: set -ev . scripts/ci/install.sh # verify azdev style works + source env/bin/activate azdev style redis displayName: 'Test azdev style' @@ -311,44 +310,3 @@ jobs: . env/bin/activate azdev perf benchmark "version" "network vnet -h" "rest -h" "storage account" -o table displayName: "Execution Performance" - -# - job: PerformanceCheckOnWindows -# displayName: "Performance Check on Windows" -# dependsOn: BuildPythonWheel -# pool: -# vmImage: 'vs2017-win2016' -# strategy: -# matrix: -# Python36: -# python.version: '3.6' -# Python38: -# python.version: '3.8' -# steps: -# - task: DownloadPipelineArtifact@1 -# displayName: 'Download Build' -# inputs: -# TargetPath: '$(Build.ArtifactStagingDirectory)/pypi' -# artifactName: pypi -# - task: UsePythonVersion@0 -# displayName: 'Use Python $(python.version)' -# inputs: -# versionSpec: '$(python.version)' -# - powershell: | -# python -m venv env -# .\env\Scripts\Activate.ps1 -# pip install --user -U pip setuptools wheel -q -# pip install --user $(find ${BUILD_ARTIFACTSTAGINGDIRECTORY}/pypi -name *.tar.gz) -q -# git clone https://github.com/Azure/azure-cli.git -# git clone https://github.com/Azure/azure-cli-extensions.git -# azdev setup -c -r azure-cli-extensions - -# azdev --version -# displayName: 'Azdev Setup' -# - powershell: | -# .\env\Scripts\Activate.ps1 -# azdev perf load-times -# displayName: "Load Performance" -# - powershell: | -# .\env\Scripts\Activate.ps1 -# azdev perf benchmark "version" "network vnet -h" "rest -h" "storage account" -# displayName: "Execution Performance" diff --git a/scripts/ci/install.sh b/scripts/ci/install.sh index 24950a6e..2b561820 100644 --- a/scripts/ci/install.sh +++ b/scripts/ci/install.sh @@ -4,9 +4,10 @@ set -ev echo "Install azdev into virtual environment" python -m venv env -. env/bin/activate +source env/bin/activate pip install -U pip setuptools wheel -q pip install $(find ${BUILD_ARTIFACTSTAGINGDIRECTORY}/pypi -name *.tar.gz) -q git clone https://github.com/Azure/azure-cli.git git clone https://github.com/Azure/azure-cli-extensions.git -azdev setup -c -r azure-cli-extensions +azdev setup -c azure-cli -r azure-cli-extensions +source env/bin/activate \ No newline at end of file diff --git a/setup.py b/setup.py index 3c5bdeca..947399f8 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,7 @@ 'azdev.utilities', ], install_requires=[ + 'virtualenv', 'docutils', 'flake8', 'gitpython', @@ -68,7 +69,7 @@ 'knack', 'mock', 'pytest>=5.0.0', - 'pytest-xdist', # depends on pytest-forked + 'pytest-xdist', # depends on pytest-forked 'pyyaml', 'requests', 'sphinx==1.6.7',