diff --git a/azdev.pyproj b/azdev.pyproj
index 6d40c214..b4e17809 100644
--- a/azdev.pyproj
+++ b/azdev.pyproj
@@ -49,6 +49,7 @@
+
Code
diff --git a/azdev/help.py b/azdev/help.py
index 9b1b21c9..34bc60f4 100644
--- a/azdev/help.py
+++ b/azdev/help.py
@@ -15,20 +15,31 @@
helps['setup'] = """
short-summary: Set up your environment for development of Azure CLI command modules and/or extensions.
examples:
- - name: Fully interactive setup.
+ - name: Fully interactive setup (Must be run in an existing virtual environment).
text: azdev setup
- - name: Install only the CLI in dev mode and search for the existing repo.
- text: azdev setup -c
+ - name: Install CLI and setup an extensions repo in an existing virtual environment. Will create a azure directory and config in the current virtual environment.
+ Note the existing virtual environment could created by VENV or PYENV.
+ text: azdev setup -c azure-cli -r azure-cli-extensions
- - name: Install public CLI and setup an extensions repo. Do not install any extensions.
+ - name: Same as above, but install the `alias` extension in the existing virtual environment too.
+ text: azdev setup -c azure-cli -r azure-cli-extensions -e alias
+
+ - name: Same as above, but will use the CLI repo path in local .azdev config, or the one in global .azdev config if not found the local one.
text: azdev setup -r azure-cli-extensions
- - name: Install CLI in dev mode, along with the extensions repo. Auto-find the CLI repo and install the `alias` extension in dev mode.
- text: azdev setup -c -r azure-cli-extensions -e alias
+ - name: Same as above, but only install CLI without setup an extensions repo.
+ text: azdev setup -c azure-cli
+
+ - name: Install CLI and setup an extensions repo in a new virtual environment. Will create a azure directory and config in the current virtual environment.
+ Note -s is using VENV to create a new virtual environment, should un-install PYENV if you have.
+ text: azdev setup -c azure-cli -r azure-cli-extensions -s env1
+
+ - name: Same as above, but do not setup new azure directory and config in this virtual environment
+ text: azdev setup -c azure-cli -r azure-cli-extensions -s env1 -g
- - name: Install only the CLI in dev mode and resolve dependencies from setup.py.
- text: azdev setup -c -d setup.py
+ - name: Same as above, but copy over system level azure settings into new virtual environment azure settings
+ text: azdev setup -c azure-cli -r azure-cli-extensions -s env1 --copy
"""
diff --git a/azdev/operations/help/__init__.py b/azdev/operations/help/__init__.py
index 30c5af45..0f216224 100644
--- a/azdev/operations/help/__init__.py
+++ b/azdev/operations/help/__init__.py
@@ -20,7 +20,8 @@
from azure.cli.core.extension.operations import list_available_extensions, list_extensions as list_cli_extensions # pylint: disable=import-error
from azdev.utilities import (
display, heading, subheading,
- get_cli_repo_path, get_path_table
+ get_cli_repo_path, get_path_table,
+ require_virtual_env
)
from azdev.utilities.tools import require_azure_cli
diff --git a/azdev/operations/pypi.py b/azdev/operations/pypi.py
index 55009f47..c58e3a8c 100644
--- a/azdev/operations/pypi.py
+++ b/azdev/operations/pypi.py
@@ -16,7 +16,8 @@
from azdev.utilities import (
display, heading, subheading, cmd, py_cmd, get_path_table,
- pip_cmd, COMMAND_MODULE_PREFIX, require_azure_cli, find_files)
+ pip_cmd, COMMAND_MODULE_PREFIX, require_azure_cli, require_virtual_env,
+ find_files)
logger = get_logger(__name__)
@@ -131,6 +132,7 @@ def verify_versions():
import tempfile
import shutil
+ require_virtual_env()
require_azure_cli()
heading('Verify CLI Versions')
diff --git a/azdev/operations/setup.py b/azdev/operations/setup.py
index 87aa4968..6f8921a9 100644
--- a/azdev/operations/setup.py
+++ b/azdev/operations/setup.py
@@ -6,7 +6,9 @@
import os
from shutil import copytree, rmtree
+import shutil
import time
+import sys
from knack.log import get_logger
from knack.util import CLIError
@@ -14,9 +16,11 @@
from azdev.operations.extensions import (
list_extensions, add_extension_repo, remove_extension)
from azdev.params import Flag
+import azdev.utilities.const as const
+import azdev.utilities.venv as venv
from azdev.utilities import (
- display, heading, subheading, pip_cmd, find_file,
- get_azdev_config_dir, get_azdev_config, require_virtual_env, get_azure_config)
+ display, heading, subheading, pip_cmd, find_file, get_env_path,
+ get_azdev_config_dir, get_azdev_config, get_azure_config, shell_cmd)
logger = get_logger(__name__)
@@ -196,8 +200,8 @@ def add_ext_repo(path):
# repo directory. To use multiple extension repos or identify a repo outside the cwd, they must specify
# the path.
if prompt_y_n('\nDo you plan to develop CLI extensions?'):
- display('\nGreat! Input the paths for the extension repos you wish to develop for, one per '
- 'line. You can add as many repos as you like. (TIP: to quickly get started, press RETURN to '
+ display('\nGreat! Input the path for the extension repos you wish to develop for. '
+ '(TIP: to quickly get started, press RETURN to '
'use your current working directory).')
first_repo = True
while True:
@@ -245,14 +249,170 @@ def add_ext_repo(path):
raise CLIError('Installation aborted.')
-def setup(cli_path=None, ext_repo_path=None, ext=None, deps=None):
+def _validate_input(cli_path, ext_repo_path, set_env, copy, use_global, ext):
+ if copy and use_global:
+ raise CLIError("Copy and use global are mutally exlcusive.")
+ if cli_path == "pypi" and any([use_global, copy, set_env]):
+ raise CLIError("pypi for cli path is mutally exlcusive with global copy and set env")
+ if not cli_path and any([use_global, copy, set_env]):
+ raise CLIError("if global, copy, or set env are set then both an extensions repo "
+ " and a cli repo must be specified")
+ if not ext_repo_path and ext:
+ raise CLIError("Extesions provided to be installed but no extensions path was given")
- require_virtual_env()
- start = time.time()
+def _check_paths(cli_path, ext_repo_path):
+ if not os.path.isdir(cli_path):
+ raise CLIError("The cli path is not a valid directory, please check the path")
+ if ext_repo_path and not os.path.isdir(ext_repo_path):
+ raise CLIError("The cli extensions path is not a valid directory, please check the path")
+
+
+def _check_shell():
+ if 'SHELL' in os.environ and const.IS_WINDOWS and 'bash.exe' in os.environ['SHELL']:
+ heading("WARNING: You are running bash in Windows, the setup may not work correctly and "
+ "command may have unexpected behavior")
+ from knack.prompting import prompt_y_n
+ if not prompt_y_n('Would you like to continue with the install?'):
+ sys.exit(0)
+
+
+def _check_env(set_env):
+ if not set_env:
+ if not get_env_path():
+ raise CLIError('You are not running in a virtual enviroment and have not chosen to set one up.')
+ _check_pyenv()
+ elif 'VIRTUAL_ENV' in os.environ:
+ raise CLIError("You are already running in a virtual enviroment, yet you want to set a new one up")
+
+
+def _check_pyenv():
+ if 'PYENV_VIRTUAL_ENV' in os.environ:
+ if const.IS_WINDOWS:
+ raise CLIError('AZDEV does not support setup in a pyenv-win virtual environment.')
+ activate_path = os.path.join(
+ os.environ['PYENV_ROOT'], 'plugins', 'pyenv-virtualenv', 'bin', 'pyenv-sh-activate')
+ venv.edit_pyenv_activate(activate_path)
+
+
+def setup(cli_path=None, ext_repo_path=None, ext=None, deps=None, set_env=None, copy=None, use_global=None):
+ _check_env(set_env)
+
+ _check_shell()
heading('Azure CLI Dev Setup')
+ # cases for handling legacy install
+ if not any([cli_path, ext_repo_path]) or cli_path == "pypi":
+ display("WARNING: Installing azdev in legacy mode. Run with atleast -c "
+ "to install the latest azdev wihout \"pypi\"\n")
+ return _handle_legacy(cli_path, ext_repo_path, ext, deps, time.time())
+ if 'CONDA_PREFIX' in os.environ:
+ raise CLIError('CONDA virutal enviroments are not supported outside'
+ ' of interactive mode or when -c and -r are provided')
+
+ if not cli_path:
+ cli_path = _handle_no_cli_path()
+
+ _validate_input(cli_path, ext_repo_path, set_env, copy, use_global, ext)
+ _check_paths(cli_path, ext_repo_path)
+
+ if set_env:
+ shell_cmd((const.VENV_CMD if const.IS_WINDOWS else const.VENV_CMD3) + set_env, raise_ex=False)
+ azure_path = os.path.join(os.path.abspath(os.getcwd()), set_env)
+ else:
+ azure_path = os.environ.get('VIRTUAL_ENV')
+
+ dot_azure_config = os.path.join(azure_path, '.azure')
+ dot_azdev_config = os.path.join(azure_path, '.azdev')
+
+ # clean up venv dirs if they already existed
+ # and this is a reinstall/new setup
+ if os.path.isdir(dot_azure_config):
+ shutil.rmtree(dot_azure_config)
+ if os.path.isdir(dot_azdev_config):
+ shutil.rmtree(dot_azdev_config)
+
+ global_az_config = os.path.expanduser(os.path.join('~', '.azure'))
+ global_azdev_config = os.path.expanduser(os.path.join('~', '.azdev'))
+ azure_config_path = os.path.join(dot_azure_config, const.CONFIG_NAME)
+ azdev_config_path = os.path.join(dot_azdev_config, const.CONFIG_NAME)
+
+ if os.path.isdir(global_az_config) and copy:
+ shutil.copytree(global_az_config, dot_azure_config)
+ if os.path.isdir(global_azdev_config):
+ shutil.copytree(global_azdev_config, dot_azdev_config)
+ else:
+ os.mkdir(dot_azdev_config)
+ file = open(azdev_config_path, "w")
+ file.close()
+ elif not use_global and not copy:
+ os.mkdir(dot_azure_config)
+ os.mkdir(dot_azdev_config)
+ file_az, file_dev = open(azure_config_path, "w"), open(azdev_config_path, "w")
+ file_az.close()
+ file_dev.close()
+ elif os.path.isdir(global_az_config):
+ dot_azure_config, dot_azdev_config = global_az_config, global_azdev_config
+ azure_config_path = os.path.join(dot_azure_config, const.CONFIG_NAME)
+ else:
+ raise CLIError(
+ "Global AZ config is not set up, yet it was specified to be used.")
+
+ # set env vars for get azure config and get azdev config
+ os.environ['AZURE_CONFIG_DIR'], os.environ['AZDEV_CONFIG_DIR'] = dot_azure_config, dot_azdev_config
+ config = get_azure_config()
+ if not config.get('cloud', 'name', None):
+ config.set_value('cloud', 'name', 'AzureCloud')
+ if ext_repo_path:
+ config.set_value(const.EXT_SECTION, const.AZ_DEV_SRC, os.path.abspath(ext_repo_path))
+ venv.edit_activate(azure_path, dot_azure_config, dot_azdev_config)
+ if cli_path:
+ config.set_value('clipath', const.AZ_DEV_SRC, os.path.abspath(cli_path))
+ venv.install_cli(os.path.abspath(cli_path), azure_path)
+ config = get_azdev_config()
+ config.set_value('ext', 'repo_paths', os.path.abspath(ext_repo_path) if ext_repo_path else '_NONE_')
+ config.set_value('cli', 'repo_path', os.path.abspath(cli_path))
+ _copy_config_files()
+ if ext and ext_repo_path:
+ venv.install_extensions(azure_path, ext)
+
+ if not set_env:
+ heading("The setup was successful! Please run or re-run the virtual environment activation script.")
+ else:
+ heading("The setup was successful!")
+ return None
+
+
+def _get_azdev_cli_path(config_file_path):
+ if not os.path.exists(config_file_path):
+ return None
+
+ import configparser
+ with open(config_file_path, "r") as file:
+ config_parser = configparser.RawConfigParser()
+ config_parser.read_string(file.read())
+ if config_parser.has_section('cli') and config_parser.has_option('cli', 'repo_path'):
+ return config_parser.get('cli', 'repo_path')
+ return None
+
+
+def _handle_no_cli_path():
+ local_azdev_config = os.path.join(os.environ.get('VIRTUAL_ENV'), '.azdev', const.CONFIG_NAME)
+ cli_path = _get_azdev_cli_path(local_azdev_config)
+ if cli_path is None:
+ display('Not found cli path in local azdev config file: ' + local_azdev_config)
+ display('Will use the one in global azdev config.')
+ global_azdev_config = os.path.expanduser(os.path.join('~', '.azdev', const.CONFIG_NAME))
+ cli_path = _get_azdev_cli_path(global_azdev_config)
+ if cli_path is None:
+ raise CLIError('Not found cli path in global azdev config file: ' + global_azdev_config)
+ display('cli_path: ' + cli_path)
+ return cli_path
+
+
+def _handle_legacy(cli_path, ext_repo_path, ext, deps, start):
+ ext_repo_path = [ext_repo_path] if ext_repo_path else None
ext_to_install = []
if not any([cli_path, ext_repo_path, ext]):
cli_path, ext_repo_path, ext_to_install = _interactive_setup()
@@ -279,7 +439,6 @@ def setup(cli_path=None, ext_repo_path=None, ext=None, deps=None):
# must add the necessary repo to add an extension
if ext and not ext_repo_path:
raise CLIError('usage error: --repo EXT_REPO [EXT_REPO ...] [--ext EXT_NAME ...]')
-
get_azure_config().set_value('extension', 'dev_sources', '')
if ext_repo_path:
# add extension repo(s)
@@ -313,11 +472,10 @@ def setup(cli_path=None, ext_repo_path=None, ext=None, deps=None):
# upgrade to latest pip
pip_cmd('install --upgrade pip -q', 'Upgrading pip...')
-
_install_cli(cli_path, deps=deps)
- _install_extensions(ext_to_install)
+ if ext_repo_path:
+ _install_extensions(ext_to_install)
_copy_config_files()
-
end = time.time()
elapsed_min = int((end - start) / 60)
elapsed_sec = int(end - start) % 60
diff --git a/azdev/operations/style.py b/azdev/operations/style.py
index 3c0c1835..24bd6be9 100644
--- a/azdev/operations/style.py
+++ b/azdev/operations/style.py
@@ -201,7 +201,7 @@ def _config_file_path(style_type="pylint"):
ext_repo_path = filter(
lambda x: "azure-cli-extension" in x,
- get_azdev_config().get("ext", "repo_paths").split(),
+ get_azdev_config().get("ext", "repo_paths").split(','),
)
try:
ext_repo_path = next(ext_repo_path)
diff --git a/azdev/params.py b/azdev/params.py
index 4b7cafde..71afac94 100644
--- a/azdev/params.py
+++ b/azdev/params.py
@@ -34,10 +34,13 @@ def load_arguments(self, _):
c.argument('git_repo', options_list='--repo', arg_group='Git', help='Path to the Git repo to check.')
with ArgumentsContext(self, 'setup') as c:
- c.argument('cli_path', options_list=['--cli', '-c'], nargs='?', const=Flag, help="Path to an existing Azure CLI repo. Omit value to search for the repo or use special value 'EDGE' to install the latest developer edge build.")
- c.argument('ext_repo_path', options_list=['--repo', '-r'], nargs='+', help='Space-separated list of paths to existing Azure CLI extensions repos.')
+ c.argument('cli_path', options_list=['--cli', '-c'], type=str, help="Path to an existing Azure CLI repo. Use special value 'EDGE' to install the latest developer edge build. Note: if not provide, will use the one in local .azdev config, if not exist will use the one in global .azdev config.")
+ c.argument('ext_repo_path', options_list=['--repo', '-r'], type=str, help='Path to existing Azure CLI extensions repos.')
c.argument('ext', options_list=['--ext', '-e'], nargs='+', help="Space-separated list of extensions to install initially. Use '*' to install all extensions.")
c.argument('deps', options_list=['--deps-from', '-d'], choices=['requirements.txt', 'setup.py'], default='requirements.txt', help="Choose the file to resolve dependencies.")
+ c.argument('set_env', options_list=['--set-env', '-s'], type=str, help="Will create a virtual enviroment with the given env name")
+ c.argument('copy', options_list='--copy', action='store_true', help="Will copy entire global .azure diretory to the newly created virtual enviroment .azure direcotry if it exist")
+ c.argument('use_global', options_list=['--use-global', '-g'], action='store_true', help="Will use the default global system .azure config")
with ArgumentsContext(self, 'test') as c:
c.argument('discover', options_list='--discover', action='store_true', help='Build an index of test names so that you don\'t need to specify fully qualified test paths.')
diff --git a/azdev/utilities/__init__.py b/azdev/utilities/__init__.py
index 40f76e1b..f16e59e2 100644
--- a/azdev/utilities/__init__.py
+++ b/azdev/utilities/__init__.py
@@ -14,7 +14,8 @@
call,
cmd,
py_cmd,
- pip_cmd
+ pip_cmd,
+ shell_cmd
)
from .const import (
COMMAND_MODULE_PREFIX,
@@ -22,8 +23,7 @@
IS_WINDOWS,
ENV_VAR_TEST_MODULES,
ENV_VAR_TEST_LIVE,
- ENV_VAR_VIRTUAL_ENV,
- EXT_REPO_NAME
+ ENV_VAR_VIRTUAL_ENV
)
from .display import (
display,
@@ -76,7 +76,6 @@
'ENV_VAR_TEST_MODULES',
'ENV_VAR_TEST_LIVE',
'ENV_VAR_VIRTUAL_ENV',
- 'EXT_REPO_NAME',
'IS_WINDOWS',
'extract_module_name',
'find_file',
diff --git a/azdev/utilities/command.py b/azdev/utilities/command.py
index 55d3a1da..cc55b289 100644
--- a/azdev/utilities/command.py
+++ b/azdev/utilities/command.py
@@ -9,7 +9,7 @@
import sys
from knack.log import get_logger
-from knack.util import CommandResultItem
+from knack.util import CommandResultItem, CLIError
logger = get_logger(__name__)
@@ -36,8 +36,7 @@ def cmd(command, message=False, show_stderr=True, **kwargs):
:param kwargs: Any kwargs supported by subprocess.Popen
:returns: CommandResultItem object.
"""
- from azdev.utilities import IS_WINDOWS, display
-
+ from . import IS_WINDOWS, display
# use default message if custom not provided
if message is True:
message = 'Running: {}\n'.format(command)
@@ -56,6 +55,34 @@ def cmd(command, message=False, show_stderr=True, **kwargs):
return CommandResultItem(err.output, exit_code=err.returncode, error=err)
+def shell_cmd(command, message=False, stderr=None, stdout=None, check=True, raise_ex=True, timeout=None,
+ executable=None, capture_output=False):
+
+ # use default message if custom not provided
+ if message is True:
+ message = '\nRunning: {}\n'.format(command)
+ from . import display
+ if message:
+ display(message)
+
+ try:
+ output = subprocess.run(command,
+ stdout=subprocess.PIPE if capture_output else stdout,
+ stderr=subprocess.PIPE if capture_output else stderr,
+ check=check,
+ timeout=timeout,
+ executable=executable,
+ shell=True)
+ if capture_output:
+ return CommandResultItem(output.stdout.decode('utf-8').strip(), exit_code=0, error=None)
+ except subprocess.CalledProcessError as err:
+ if raise_ex:
+ raise err
+ logger.debug(err)
+ raise CLIError("Command " + command + " failed. Trying running with --debug for more info")
+ return None
+
+
def py_cmd(command, message=False, show_stderr=True, is_module=True, **kwargs):
""" Run a script or command with Python.
diff --git a/azdev/utilities/config.py b/azdev/utilities/config.py
index 69a5d0e3..7b5963f7 100644
--- a/azdev/utilities/config.py
+++ b/azdev/utilities/config.py
@@ -5,7 +5,7 @@
# -----------------------------------------------------------------------------
import os
-
+import sys
from knack.config import CLIConfig
@@ -21,7 +21,11 @@ def get_azdev_config_dir():
""" Returns the user's .azdev directory. """
from azdev.utilities import get_env_path
env_name = None
- _, env_name = os.path.splitdrive(get_env_path())
+ if not get_env_path():
+ _, env_name = os.path.splitdrive(sys.executable)
+ else:
+ _, env_name = os.path.splitdrive(get_env_path())
+
azdev_dir = os.getenv('AZDEV_CONFIG_DIR', None) or os.path.expanduser(os.path.join('~', '.azdev'))
if not env_name:
return azdev_dir
diff --git a/azdev/utilities/const.py b/azdev/utilities/const.py
index d0db7900..607a2ee2 100644
--- a/azdev/utilities/const.py
+++ b/azdev/utilities/const.py
@@ -6,10 +6,21 @@
import sys
+AZ_AZDEV_DIR = 'AZDEV_CONFIG_DIR'
+AZ_CONFIG_DIR = 'AZURE_CONFIG_DIR'
+AZ_DEV_SRC = 'dev_sources'
COMMAND_MODULE_PREFIX = 'azure-cli-'
+CONFIG_NAME = 'config'
EXTENSION_PREFIX = 'azext_'
-EXT_REPO_NAME = 'azure-cli-extensions'
+EXT_SECTION = 'extension'
IS_WINDOWS = sys.platform.lower() in ['windows', 'win32']
+PIP_E_CMD = 'pip install -e '
+PIP_R_CMD = 'pip install -r '
+UN_ACTIVATE = 'activate'
+UN_BIN = 'bin'
+UN_EXPORT = 'export'
+VENV_CMD = 'python -m venv --system-site-packages '
+VENV_CMD3 = 'python3 -m venv --system-site-packages '
ENV_VAR_TEST_MODULES = 'AZDEV_TEST_TESTS' # comma-separated list of modules to test
ENV_VAR_VIRTUAL_ENV = ['VIRTUAL_ENV', 'CONDA_PREFIX'] # used by system to identify virtual environment
diff --git a/azdev/utilities/path.py b/azdev/utilities/path.py
index 1f15c7d3..ae30620b 100644
--- a/azdev/utilities/path.py
+++ b/azdev/utilities/path.py
@@ -5,11 +5,12 @@
# -----------------------------------------------------------------------------
import os
-from glob import glob
+from glob import glob
from knack.util import CLIError
-
-from .const import COMMAND_MODULE_PREFIX, EXTENSION_PREFIX, ENV_VAR_VIRTUAL_ENV
+from six.moves import configparser
+from azdev.utilities import get_azure_config, display
+from . import const
def extract_module_name(path):
@@ -34,7 +35,7 @@ def get_env_path():
:returns: Path (str) to the virtual env or None.
"""
env_path = None
- for item in ENV_VAR_VIRTUAL_ENV:
+ for item in const.ENV_VAR_VIRTUAL_ENV:
env_path = os.environ.get(item)
if env_path:
break
@@ -70,12 +71,15 @@ def get_ext_repo_paths():
:returns: Path (str) to Azure CLI dev extension repos.
"""
- from configparser import NoSectionError
- from .config import get_azdev_config
+ from configparser import NoSectionError, NoOptionError
try:
- return get_azdev_config().get('ext', 'repo_paths').split(',')
+ return get_azure_config().get(const.EXT_SECTION, const.AZ_DEV_SRC).split(',')
except NoSectionError:
- raise CLIError('Unable to retrieve extensions repo path from config. Please run `azdev setup`.')
+ raise CLIError('Unable to retrieve extensions repo path from config. Please run `azdev setup` '
+ 'with -r to set an extensions repo.')
+ except NoOptionError:
+ raise CLIError('Unable to retrieve the option {} from azure config section [{}]'.format(
+ const.AZ_DEV_SRC, const.EXT_SECTION))
def find_file(file_name):
@@ -119,11 +123,14 @@ def make_dirs(path):
def get_name_index(invert=False, include_whl_extensions=False):
""" Returns a dictionary containing the long and short names of modules and extensions is {SHORT:LONG} format or
{LONG:SHORT} format when invert=True. """
- from azure.cli.core.extension import EXTENSIONS_DIR # pylint: disable=import-error
+ config = get_azure_config() # pylint: disable=import-error
+ try:
+ EXTENSIONS_DIR = config.get(const.EXT_SECTION, const.AZ_DEV_SRC)
+ except (configparser.NoSectionError, configparser.NoOptionError):
+ EXTENSIONS_DIR = ""
table = {}
cli_repo_path = get_cli_repo_path()
- ext_repo_paths = get_ext_repo_paths()
# unified azure-cli package (2.0.68 and later)
paths = os.path.normcase(
@@ -133,9 +140,9 @@ def get_name_index(invert=False, include_whl_extensions=False):
)
modules_paths = glob(paths)
core_paths = glob(os.path.normcase(os.path.join(cli_repo_path, 'src', '*', 'setup.py')))
- ext_paths = [x for x in find_files(ext_repo_paths, '*.*-info') if 'site-packages' not in x]
+ ext_paths = [x for x in find_files(EXTENSIONS_DIR, '*.*-info') if 'site-packages' not in x]
whl_ext_paths = []
- if include_whl_extensions:
+ if include_whl_extensions and EXTENSIONS_DIR:
whl_ext_paths = [x for x in find_files(EXTENSIONS_DIR, '*.*-info') if 'site-packages' not in x]
def _update_table(paths, key):
@@ -149,15 +156,15 @@ def _update_table(paths, key):
if key == 'ext':
short_name = base_name
for item in os.listdir(folder):
- if item.startswith(EXTENSION_PREFIX):
+ if item.startswith(const.EXTENSION_PREFIX):
long_name = item
break
- elif base_name.startswith(COMMAND_MODULE_PREFIX):
+ elif base_name.startswith(const.COMMAND_MODULE_PREFIX):
long_name = base_name
- short_name = base_name.replace(COMMAND_MODULE_PREFIX, '') or '__main__'
+ short_name = base_name.replace(const.COMMAND_MODULE_PREFIX, '') or '__main__'
else:
short_name = base_name
- long_name = '{}{}'.format(COMMAND_MODULE_PREFIX, base_name)
+ long_name = '{}{}'.format(const.COMMAND_MODULE_PREFIX, base_name)
if not invert:
table[short_name] = long_name
else:
@@ -190,8 +197,14 @@ def get_path_table(include_only=None, include_whl_extensions=False):
}
}
"""
- from azure.cli.core.extension import EXTENSIONS_DIR # pylint: disable=import-error
-
+ config = get_azure_config() # pylint: disable=import-error
+ try:
+ EXTENSIONS_DIR = config.get(const.EXT_SECTION, const.AZ_DEV_SRC)
+ os.chdir(EXTENSIONS_DIR)
+ except (configparser.NoSectionError, configparser.NoOptionError, TypeError):
+ display("WARNING: No extension path found, only modules will be available. "
+ "rerun setup with -r to make extensions available\n")
+ EXTENSIONS_DIR = ""
# determine whether the call will filter or return all
if isinstance(include_only, str):
include_only = [include_only]
@@ -199,7 +212,6 @@ def get_path_table(include_only=None, include_whl_extensions=False):
table = {}
cli_repo_path = get_cli_repo_path()
- ext_repo_paths = get_ext_repo_paths()
paths = os.path.normcase(
os.path.join(
@@ -208,7 +220,7 @@ def get_path_table(include_only=None, include_whl_extensions=False):
)
modules_paths = glob(paths)
core_paths = glob(os.path.normcase(os.path.join(cli_repo_path, 'src', '*', 'setup.py')))
- ext_paths = [x for x in find_files(ext_repo_paths, '*.*-info') if 'site-packages' not in x]
+ ext_paths = [x for x in find_files(EXTENSIONS_DIR, '*.*-info') if 'site-packages' not in x]
whl_ext_paths = [x for x in find_files(EXTENSIONS_DIR, '*.*-info') if 'site-packages' not in x]
def _update_table(package_paths, key):
@@ -221,10 +233,10 @@ def _update_table(package_paths, key):
if key == 'ext':
short_name = base_name
- long_name = next((item for item in os.listdir(folder) if item.startswith(EXTENSION_PREFIX)), None)
+ long_name = next((item for item in os.listdir(folder) if item.startswith(const.EXTENSION_PREFIX)), None)
else:
short_name = base_name
- long_name = '{}{}'.format(COMMAND_MODULE_PREFIX, base_name)
+ long_name = '{}{}'.format(const.COMMAND_MODULE_PREFIX, base_name)
if get_all:
table[key][long_name if key == 'ext' else short_name] = folder
diff --git a/azdev/utilities/tools.py b/azdev/utilities/tools.py
index 47396a75..2fbb1b52 100644
--- a/azdev/utilities/tools.py
+++ b/azdev/utilities/tools.py
@@ -4,7 +4,9 @@
# license information.
# -----------------------------------------------------------------------------
+import os
from knack.util import CLIError
+from . import const
def require_virtual_env():
@@ -13,6 +15,12 @@ def require_virtual_env():
env = get_env_path()
if not env:
raise CLIError('This command can only be run from an active virtual environment.')
+ if not os.environ.get(const.AZ_CONFIG_DIR):
+ raise CLIError(
+ "AZURE_CONFIG_DIR env var is not set. Please run 'azdev setup'")
+ if not os.path.exists(os.path.join(os.environ[const.AZ_CONFIG_DIR], "config")):
+ raise CLIError(
+ "The Azure config file does not exist. Please run 'azdev setup'")
def require_azure_cli():
diff --git a/azdev/utilities/venv.py b/azdev/utilities/venv.py
new file mode 100644
index 00000000..67285bec
--- /dev/null
+++ b/azdev/utilities/venv.py
@@ -0,0 +1,152 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+import os
+import subprocess
+import platform
+from knack.util import CLIError
+import azdev.operations.extensions
+from azdev.utilities import display, shell_cmd
+from . import const
+
+
+def edit_activate(azure_config_path, dot_azure_config, dot_azdev_config):
+ if const.IS_WINDOWS:
+ ps1_edit(azure_config_path, dot_azure_config, dot_azdev_config)
+ bat_edit(azure_config_path, dot_azure_config, dot_azdev_config)
+ else:
+ unix_edit(azure_config_path, dot_azure_config, dot_azdev_config)
+
+
+def edit_pyenv_activate(activate_path):
+ set_azdev_content = 'set -gx AZDEV_CONFIG_DIR "${prefix}/.azdev";\n'
+ set_azure_content = 'set -gx AZURE_CONFIG_DIR "${prefix}/.azure";\n'
+ export_azdev_content = 'export AZDEV_CONFIG_DIR="${prefix}/.azdev";\n'
+ export_azure_content = 'export AZURE_CONFIG_DIR="${prefix}/.azure";\n'
+
+ insert(activate_path, set_azdev_content, get_line_num(activate_path, "set -gx VIRTUAL_ENV \"${prefix}\";"))
+ insert(activate_path, set_azure_content, get_line_num(activate_path, "set -gx VIRTUAL_ENV \"${prefix}\";"))
+ insert(activate_path, export_azdev_content, get_line_num(activate_path, "export VIRTUAL_ENV=\"${prefix}\";"))
+ insert(activate_path, export_azure_content, get_line_num(activate_path, "export VIRTUAL_ENV=\"${prefix}\";"))
+
+
+def insert(file_path, content, line_num):
+ if line_num == -1:
+ raise CLIError("Cannot find a proper place to set/export environment variable.")
+ if get_line_num(file_path, content) == -1:
+ with open(file_path, 'r+') as reader:
+ file_content = reader.readlines()
+ file_content.insert(line_num, content)
+ reader.seek(0)
+ reader.writelines(file_content)
+
+
+def get_line_num(file_path, target):
+ with open(file_path, 'r') as reader:
+ for num, line in enumerate(reader, 1):
+ if target in line:
+ return num
+ return -1
+
+
+def unix_edit(azure_config_path, dot_azure_config, dot_azdev_config):
+ activate_path = os.path.join(azure_config_path, const.UN_BIN,
+ const.UN_ACTIVATE)
+ content = open(activate_path, "r").readlines()
+
+ # check if already ran setup before
+ if const.AZ_CONFIG_DIR not in content[0]:
+ content = [const.AZ_CONFIG_DIR + '=' + dot_azure_config + '\n',
+ const.UN_EXPORT + ' ' + const.AZ_CONFIG_DIR + '\n',
+ const.AZ_AZDEV_DIR + '=' + dot_azdev_config + '\n',
+ const.UN_EXPORT + ' ' + const.AZ_AZDEV_DIR + '\n'] + content
+ with open(activate_path, "w") as file:
+ file.writelines(content)
+
+
+def bat_edit(azure_config_path, dot_azure_config, dot_azdev_config):
+ activate_path = os.path.join(azure_config_path, 'Scripts',
+ 'activate.bat')
+ content = open(activate_path, "r").readlines()
+ if const.AZ_CONFIG_DIR not in content[1]:
+ content = content[0:1] + ['set ' + const.AZ_CONFIG_DIR +
+ '=' + dot_azure_config + '\n',
+ 'set ' + const.AZ_AZDEV_DIR +
+ '=' + dot_azdev_config] + content[1::]
+ with open(activate_path, "w") as file:
+ file.writelines(content)
+
+
+def ps1_edit(azure_config_path, dot_azure_config, dot_azdev_config):
+ activate_path = os.path.join(azure_config_path, 'Scripts',
+ 'Activate.ps1')
+ content = open(activate_path, "r").read()
+ idx = content.find('$env:VIRTUAL_ENV')
+ if idx < 0:
+ raise CLIError("hmm, it looks like Activate.ps1 does"
+ " not set the virutal enviroment variable VIRTUAL_ENV")
+ if content.find('$env:AZURE_CONFIG_DIR') < 0:
+ content = content[:idx] + '$env:AZURE_CONFIG_DIR' + " = " + \
+ "\"" + dot_azure_config + "\"; " + \
+ "$env:AZDEV_CONFIG_DIR = " + \
+ "\"" + dot_azdev_config + "\"; " + \
+ content[idx:]
+ with open(activate_path, "w") as file:
+ file.write(content)
+
+
+def install_cli(cli_path, venv_path):
+ src_path = os.path.join(cli_path, 'src')
+ activate_path = (os.path.join(venv_path, 'Scripts', 'activate')
+ if const.IS_WINDOWS else 'source ' + os.path.join(venv_path, const.UN_BIN, const.UN_ACTIVATE))
+ delimiter = ' && ' if const.IS_WINDOWS else '; '
+ executable = None if const.IS_WINDOWS else '/bin/bash'
+ display("\nvenv activate path is " + str(activate_path))
+ shell_cmd(activate_path + delimiter + 'pip install --ignore-installed azure-common',
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL, raise_ex=False, executable=executable)
+ display("\nInstalling telemetry ")
+ shell_cmd(activate_path + delimiter + const.PIP_E_CMD + os.path.join(src_path, 'azure-cli-telemetry'),
+ stdout=subprocess.DEVNULL, raise_ex=False, stderr=subprocess.DEVNULL, executable=executable)
+ display("\nInstalling core ")
+ shell_cmd(activate_path + delimiter + const.PIP_E_CMD + os.path.join(src_path, 'azure-cli-core'),
+ stdout=subprocess.DEVNULL, raise_ex=False, stderr=subprocess.DEVNULL, executable=executable)
+ shell_cmd(activate_path + delimiter + const.PIP_E_CMD + os.path.join(src_path, 'azure-cli-testsdk'),
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL, raise_ex=False, executable=executable)
+ display("\nInstalling cli ")
+ shell_cmd(activate_path + delimiter + const.PIP_E_CMD + os.path.join(src_path, 'azure-cli'),
+ raise_ex=False, executable=executable)
+ req_file = 'requirements.py3.{}.txt'.format(platform.system().lower() if const.IS_WINDOWS else platform.system())
+ req_file = "{}/src/azure-cli/{}".format(cli_path, req_file)
+ display("Installing " + req_file)
+ shell_cmd(activate_path + delimiter + const.PIP_R_CMD + req_file, raise_ex=False, executable=executable)
+
+
+def install_extensions(venv_path, extensions):
+ activate_path = os.path.join(venv_path, 'Scripts', 'activate') if const.IS_WINDOWS else 'source ' + os.path.join(
+ venv_path, const.UN_BIN, const.UN_ACTIVATE)
+ delimiter = ' && ' if const.IS_WINDOWS else '; '
+ executable = None if const.IS_WINDOWS else '/bin/bash'
+ all_ext = azdev.operations.extensions.list_extensions()
+ if extensions == ['*']:
+ display("\nInstalling all extensions")
+ for i in all_ext:
+ shell_cmd(activate_path + delimiter + const.PIP_E_CMD + i['path'], executable=executable)
+ extensions = False
+ else:
+ display("\nInstalling the following extensions: " + str(extensions))
+ extensions = set(extensions)
+ k = 0
+ while k < len(all_ext) and extensions:
+ if all_ext[k]['name'] in extensions:
+ shell_cmd(activate_path + delimiter + const.PIP_E_CMD + all_ext[k]['path'],
+ executable=executable)
+ extensions.remove(all_ext[k]['name'])
+ k += 1
+ if extensions:
+ raise CLIError("The following extensions were not found. Ensure you have added "
+ "the repo using `--repo/-r PATH`.\n {}".format('\n '.join(extensions)))
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 33877a5b..adf8e3cf 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -122,9 +122,6 @@ jobs:
- bash: |
set -ev
- python -m venv env
- . env/bin/activate
-
# install azdev from source code
pip install -e .
pip install pytest-cov
@@ -132,8 +129,8 @@ jobs:
# azdev setup
git clone --quiet https://github.com/Azure/azure-cli.git
git clone --quiet https://github.com/Azure/azure-cli-extensions.git
- azdev setup -c ./azure-cli -r ./azure-cli-extensions
-
+ azdev setup -c ./azure-cli -r ./azure-cli-extensions -s env
+ source env/bin/activate
azdev --version
python -m pytest azdev/ --ignore=azdev/mod_templates --junitxml=junit/test-results.xml --cov=azdev --cov-report=xml
@@ -176,6 +173,7 @@ jobs:
set -ev
. scripts/ci/install.sh
# verify azdev setup worked
+ source env/bin/activate
az -h
displayName: 'Test azdev setup'
@@ -240,7 +238,7 @@ jobs:
set -ev
. scripts/ci/install.sh
# verify azdev linter works
- azdev linter acr
+ azdev linter CLI
displayName: 'Test azdev linter'
- job: TestAzdevStyle
@@ -272,6 +270,7 @@ jobs:
set -ev
. scripts/ci/install.sh
# verify azdev style works
+ source env/bin/activate
azdev style redis
displayName: 'Test azdev style'
@@ -311,44 +310,3 @@ jobs:
. env/bin/activate
azdev perf benchmark "version" "network vnet -h" "rest -h" "storage account" -o table
displayName: "Execution Performance"
-
-# - job: PerformanceCheckOnWindows
-# displayName: "Performance Check on Windows"
-# dependsOn: BuildPythonWheel
-# pool:
-# vmImage: 'vs2017-win2016'
-# strategy:
-# matrix:
-# Python36:
-# python.version: '3.6'
-# Python38:
-# python.version: '3.8'
-# steps:
-# - task: DownloadPipelineArtifact@1
-# displayName: 'Download Build'
-# inputs:
-# TargetPath: '$(Build.ArtifactStagingDirectory)/pypi'
-# artifactName: pypi
-# - task: UsePythonVersion@0
-# displayName: 'Use Python $(python.version)'
-# inputs:
-# versionSpec: '$(python.version)'
-# - powershell: |
-# python -m venv env
-# .\env\Scripts\Activate.ps1
-# pip install --user -U pip setuptools wheel -q
-# pip install --user $(find ${BUILD_ARTIFACTSTAGINGDIRECTORY}/pypi -name *.tar.gz) -q
-# git clone https://github.com/Azure/azure-cli.git
-# git clone https://github.com/Azure/azure-cli-extensions.git
-# azdev setup -c -r azure-cli-extensions
-
-# azdev --version
-# displayName: 'Azdev Setup'
-# - powershell: |
-# .\env\Scripts\Activate.ps1
-# azdev perf load-times
-# displayName: "Load Performance"
-# - powershell: |
-# .\env\Scripts\Activate.ps1
-# azdev perf benchmark "version" "network vnet -h" "rest -h" "storage account"
-# displayName: "Execution Performance"
diff --git a/scripts/ci/install.sh b/scripts/ci/install.sh
index 24950a6e..2b561820 100644
--- a/scripts/ci/install.sh
+++ b/scripts/ci/install.sh
@@ -4,9 +4,10 @@ set -ev
echo "Install azdev into virtual environment"
python -m venv env
-. env/bin/activate
+source env/bin/activate
pip install -U pip setuptools wheel -q
pip install $(find ${BUILD_ARTIFACTSTAGINGDIRECTORY}/pypi -name *.tar.gz) -q
git clone https://github.com/Azure/azure-cli.git
git clone https://github.com/Azure/azure-cli-extensions.git
-azdev setup -c -r azure-cli-extensions
+azdev setup -c azure-cli -r azure-cli-extensions
+source env/bin/activate
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 3c5bdeca..947399f8 100644
--- a/setup.py
+++ b/setup.py
@@ -61,6 +61,7 @@
'azdev.utilities',
],
install_requires=[
+ 'virtualenv',
'docutils',
'flake8',
'gitpython',
@@ -68,7 +69,7 @@
'knack',
'mock',
'pytest>=5.0.0',
- 'pytest-xdist', # depends on pytest-forked
+ 'pytest-xdist', # depends on pytest-forked
'pyyaml',
'requests',
'sphinx==1.6.7',