From 31e19470f08b954edba45d1228e6b03a51bc2fd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Fri, 12 Aug 2022 10:58:57 +0200 Subject: [PATCH] add support for multi_env installers (#509) * add support for multi_env installers (WIP) * make sure conda is in base * docs and validation * fix api * add for osx-pkg too * allow per-env channels, environment and environment_file * update docs / schema checks * update examples * fix api call * fix example * better error msg * default is None, not () * fix some extra_envs assumptions * support for extra_envs.exclude; restore global menu_packages support; clarify channels(_remap) handling * do not remove tk on windows example * add tests * disable exit on error temporarily for this test * rework example and tests; exclude is buggy * test was wrong :D * better tests for dav1d --- CONSTRUCT.md | 30 ++++- constructor/construct.py | 58 ++++++++- constructor/fcp.py | 179 ++++++++++++++++++--------- constructor/header.sh | 41 ++++-- constructor/main.py | 13 ++ constructor/nsis/main.nsi.tmpl | 25 +--- constructor/osx/post_extract.sh | 34 ++++- constructor/osxpkg.py | 17 ++- constructor/preconda.py | 81 ++++++++++-- constructor/shar.py | 20 ++- constructor/winexe.py | 72 ++++++++++- examples/extra_envs/construct.yaml | 20 +++ examples/extra_envs/dav1d_env.yaml | 5 + examples/extra_envs/test_install.bat | 18 +++ examples/extra_envs/test_install.sh | 26 ++++ 15 files changed, 525 insertions(+), 114 deletions(-) create mode 100644 examples/extra_envs/construct.yaml create mode 100644 examples/extra_envs/dav1d_env.yaml create mode 100644 examples/extra_envs/test_install.bat create mode 100644 examples/extra_envs/test_install.sh diff --git a/CONSTRUCT.md b/CONSTRUCT.md index f7506d110..dc340f3c5 100644 --- a/CONSTRUCT.md +++ b/CONSTRUCT.md @@ -132,7 +132,7 @@ Path to an environment file to construct from. If this option is present, the create a temporary environment, constructor will build and installer from that, and the temporary environment will be removed. This ensures that constructor is using the precise local conda configuration to discover -and install the packages. +and install the packages. The created environment MUST include `python`. ## `transmute_file_type` @@ -158,6 +158,34 @@ _type:_ string
The channel alias that would be assumed for the created installer (only useful if it includes conda). +## `extra_envs` + +_required:_ no
+_type:_ dictionary
+Create more environments in addition to the default `base` provided by `specs`, +`environment` or `environment_file`. This should be a map of `str` (environment +name) to a dictionary of options: +- `specs` (list of str): which packages to install in that environment +- `environment` (str): same as global option, for this env +- `environment_file` (str): same as global option, for this env +- `channels` (list of str): using these channels; if not provided, the global + value is used. To override inheritance, set it to an empty list. +- `channels_remap` (list of str): same as global option, for this env; + if not provided, the global value is used. To override inheritance, set it to + an empty list. +- `user_requested_specs` (list of str): same as the global option, but for this env; + if not provided, global value is _not_ used + +Notes: +- `ignore_duplicate_files` will always be considered `True` if `extra_envs` is in use. +- `conda` needs to be present in the `base` environment (via `specs`) +- support for `menu_packages` is planned, but not possible right now. For now, all packages + in an `extra_envs` config will be allowed to create their shortcuts. +- If a global `exclude` option is used, it will have an effect on the environments created + by `extra_envs` too. For example, if the global environment excludes `tk`, none of the + extra environmentss will have it either. Unlike the global option, an error will not be + thrown if the excluded package is not found in the packages required by the extra environment. + ## `installer_filename` _required:_ no
diff --git a/constructor/construct.py b/constructor/construct.py index 2866a1bb7..976b2db92 100644 --- a/constructor/construct.py +++ b/constructor/construct.py @@ -104,7 +104,7 @@ create a temporary environment, constructor will build and installer from that, and the temporary environment will be removed. This ensures that constructor is using the precise local conda configuration to discover -and install the packages. +and install the packages. The created environment MUST include `python`. '''), ('transmute_file_type', False, str, ''' @@ -124,6 +124,32 @@ (only useful if it includes conda). '''), + ('extra_envs', False, (dict,), ''' +Create more environments in addition to the default `base` provided by `specs`, +`environment` or `environment_file`. This should be a map of `str` (environment +name) to a dictionary of options: +- `specs` (list of str): which packages to install in that environment +- `environment` (str): same as global option, for this env +- `environment_file` (str): same as global option, for this env +- `channels` (list of str): using these channels; if not provided, the global + value is used. To override inheritance, set it to an empty list. +- `channels_remap` (list of str): same as global option, for this env; + if not provided, the global value is used. To override inheritance, set it to + an empty list. +- `user_requested_specs` (list of str): same as the global option, but for this env; + if not provided, global value is _not_ used + +Notes: +- `ignore_duplicate_files` will always be considered `True` if `extra_envs` is in use. +- `conda` needs to be present in the `base` environment (via `specs`) +- support for `menu_packages` is planned, but not possible right now. For now, all packages + in an `extra_envs` config will be allowed to create their shortcuts. +- If a global `exclude` option is used, it will have an effect on the environments created + by `extra_envs` too. For example, if the global environment excludes `tk`, none of the + extra environmentss will have it either. Unlike the global option, an error will not be + thrown if the excluded package is not found in the packages required by the extra environment. +'''), + ('installer_filename', False, str, ''' The filename of the installer being created. If not supplied, a reasonable default will determined by the `name`, `version`, platform, and installer type. @@ -425,6 +451,20 @@ ] +_EXTRA_ENVS_SCHEMA = { + "specs": (list, tuple), + "environment": (str,), + "environment_file": (str,), + "channels": (list, tuple), + "channels_remap": (list, tuple), + "user_requested_specs": (list, tuple), + "exclude": (list, tuple), + # TODO: we can't support menu_packages for extra envs yet + # will implement when the PR for new menuinst lands + # "menu_packages": (list, tuple), +} + + def ns_platform(platform): p = platform return dict( @@ -557,6 +597,22 @@ def verify(info): if not pat.match(value) or value.endswith(('.', '-')): sys.exit("Error: invalid %s '%s'" % (key, value)) + for env_name, env_data in info.get("extra_envs", {}).items(): + disallowed = ('/', ' ', ':', '#') + if any(character in env_name for character in disallowed): + sys.exit( + f"Environment names (keys in 'extra_envs') cannot contain any of {disallowed}. " + f"You tried to use: {env_name}" + ) + for key, value in env_data.items(): + if key not in _EXTRA_ENVS_SCHEMA: + sys.exit(f"Key '{key}' not supported in 'extra_envs'.") + types = _EXTRA_ENVS_SCHEMA[key] + if not isinstance(value, types): + types_str = " or ".join([type_.__name__ for type_ in types]) + sys.exit(f"Value for 'extra_envs.{env_name}.{key}' " + f"must be an instance of {types_str}") + def generate_doc(): print('generate_doc() is deprecated. Use scripts/make_docs.py instead') diff --git a/constructor/fcp.py b/constructor/fcp.py index fc8e3d7c7..424b69a9f 100644 --- a/constructor/fcp.py +++ b/constructor/fcp.py @@ -54,15 +54,16 @@ def check_duplicates(precs): sys.exit(f"Error: {name} listed multiple times: {' , '.join(filenames)}") -def exclude_packages(precs, exclude=()): +def exclude_packages(precs, exclude=(), error_on_absence=True): for name in exclude: for bad_char in ' =<>*': if bad_char in name: sys.exit("Error: did not expect '%s' in package name: %s" % (bad_char, name)) - unknown_precs = set(exclude).difference(prec.name for prec in precs) - if unknown_precs: - sys.exit(f"Error: no package(s) named {', '.join(unknown_precs)} to remove") + if error_on_absence: + unknown_precs = set(exclude).difference(prec.name for prec in precs) + if unknown_precs: + sys.exit(f"Error: no package(s) named {', '.join(unknown_precs)} to remove") return [prec for prec in precs if prec.name not in exclude] @@ -141,8 +142,8 @@ def _fetch(download_dir, precs): return tuple(pc.iter_records()) -def check_duplicates_files(pc_recs, platform, ignore_duplicate_files=True): - print('Checking for duplicate files ...') +def check_duplicates_files(pc_recs, platform, duplicate_files="error"): + assert duplicate_files in ("warn", "skip", "error") map_members_scase = defaultdict(set) map_members_icase = defaultdict(lambda: {'files': set(), 'fns': set()}) @@ -173,12 +174,16 @@ def check_duplicates_files(pc_recs, platform, ignore_duplicate_files=True): map_members_icase[short_path_lower]['files'].add(short_path) map_members_icase[short_path_lower]['fns'].add(fn) + if duplicate_files == "skip": + return total_tarball_size, total_extracted_pkgs_size + + print('Checking for duplicate files ...') for member in map_members_scase: fns = map_members_scase[member] - msg_str = "File '%s' found in multiple packages: %s" % ( - member, ', '.join(fns)) if len(fns) > 1: - if ignore_duplicate_files: + msg_str = "File '%s' found in multiple packages: %s" % ( + member, ', '.join(fns)) + if duplicate_files == "warn": print('Warning: {}'.format(msg_str)) else: sys.exit('Error: {}'.format(msg_str)) @@ -191,7 +196,9 @@ def check_duplicates_files(pc_recs, platform, ignore_duplicate_files=True): msg_str = "Files %s found in the package(s): %s" % ( str(files)[1:-1], ', '.join(fns)) if len(files) > 1: - if ignore_duplicate_files or platform.startswith('linux'): + msg_str = "Files %s found in the package(s): %s" % ( + str(files)[1:-1], ', '.join(fns)) + if duplicate_files == "warn" or platform.startswith('linux'): print('Warning: {}'.format(msg_str)) else: sys.exit('Error: {}'.format(msg_str)) @@ -240,15 +247,21 @@ def _precs_from_environment(environment, download_dir, user_conda): return precs -def _main(name, version, download_dir, platform, channel_urls=(), channels_remap=(), specs=(), - exclude=(), menu_packages=(), ignore_duplicate_files=True, environment=None, - environment_file=None, verbose=True, dry_run=False, conda_exe="conda.exe", - transmute_file_type=''): +def _solve_precs(name, version, download_dir, platform, channel_urls=(), channels_remap=(), specs=(), + exclude=(), menu_packages=(), environment=None, environment_file=None, + verbose=True, conda_exe="conda.exe", extra_env=False): # Add python to specs, since all installers need a python interpreter. In the future we'll # probably want to add conda too. - specs = (*specs, "python") + # JRG: This only applies to the `base` environment; `extra_envs` are exempt + if not extra_env: + specs = (*specs, "python") if verbose: - print("specs:", specs) + if environment: + print(f"specs: ") + elif environment_file: + print(f"specs: ") + else: + print("specs:", specs) # Append channels_remap srcs to channel_urls channel_urls = (*channel_urls, *(x['src'] for x in channels_remap)) @@ -268,17 +281,14 @@ def _main(name, version, download_dir, platform, channel_urls=(), channels_remap # We need a conda for the native platform in order to do environment # based installations. sys.exit("CONDA_EXE env variable is empty. Need to activate a conda env.") - # make the environment, if needed if environment_file: from subprocess import check_call - environment = tempfile.mkdtemp() new_env = os.environ.copy() new_env["CONDA_SUBDIR"] = platform check_call([user_conda, "env", "create", "--file", environment_file, - "--prefix", environment], universal_newlines=True, env=new_env) - + "--prefix", environment, "--quiet"], universal_newlines=True, env=new_env) # obtain the package records if environment: precs = _precs_from_environment(environment, download_dir, user_conda) @@ -292,39 +302,45 @@ def _main(name, version, download_dir, platform, channel_urls=(), channels_remap ) precs = list(solver.solve_final_state()) - # move python first - python_prec = next(prec for prec in precs if prec.name == "python") - precs.remove(python_prec) - precs.insert(0, python_prec) + + python_prec = next((prec for prec in precs if prec.name == "python"), None) + if python_prec: + precs.remove(python_prec) + precs.insert(0, python_prec) + elif not extra_env: + # the base environment must always have python; this has been addressed + # at the beginning of _main() but we can still get here through the + # environment_file option + sys.exit("python MUST be part of the base environment") warn_menu_packages_missing(precs, menu_packages) check_duplicates(precs) - precs = exclude_packages(precs, exclude) + precs = exclude_packages(precs, exclude, error_on_absence=not extra_env) if verbose: more_recent_versions = _find_out_of_date_precs(precs, channel_urls, platform) _show(name, version, platform, download_dir, precs, more_recent_versions) - if dry_run: - return None, None, None, None + if environment_file: + import shutil + + shutil.rmtree(environment) + + return precs + +def _fetch_precs(precs, download_dir, transmute_file_type=''): pc_recs = _fetch(download_dir, precs) # Constructor cache directory can have multiple packages from different # installer creations. Filter out those which the solver picked. precs_fns = [x.fn for x in precs] pc_recs = [x for x in pc_recs if x.fn in precs_fns] - _urls = [(pc_rec.url, pc_rec.md5) for pc_rec in pc_recs] has_conda = any(pc_rec.name == 'conda' for pc_rec in pc_recs) - approx_tarballs_size, approx_pkgs_size = check_duplicates_files( - pc_recs, platform, ignore_duplicate_files - ) - dists = list(prec.fn for prec in precs) if transmute_file_type != '': - _urls = {os.path.basename(url): (url, md5) for url, md5 in _urls} new_dists = [] import conda_package_handling.api for dist in dists: @@ -335,32 +351,75 @@ def _main(name, version, download_dir, platform, channel_urls=(), channels_remap new_file_name = "%s%s" % (dist[:-8], transmute_file_type) new_dists.append(new_file_name) new_file_name = os.path.join(download_dir, new_file_name) - if not os.path.exists(new_file_name): - print("transmuting %s" % dist) - failed_files = conda_package_handling.api.transmute( - os.path.join(download_dir, dist), - transmute_file_type, - out_folder=download_dir, - ) - if failed_files: - message = "\n".join( - " %s failed with: %s" % x for x in failed_files.items() - ) - raise RuntimeError("Transmution failed:\n%s" % message) - url, md5 = _urls[dist] - url = url[:-len(".tar.bz2")] + transmute_file_type - md5 = hash_files([new_file_name]) - _urls[dist] = (url, md5) + if os.path.exists(new_file_name): + continue + print("transmuting %s" % dist) + conda_package_handling.api.transmute(os.path.join(download_dir, dist), + transmute_file_type, out_folder=download_dir) else: new_dists.append(dist) dists = new_dists - _urls = list(_urls.values()) - if environment_file: - import shutil + return pc_recs, _urls, dists, has_conda - shutil.rmtree(environment) - return _urls, dists, approx_tarballs_size, approx_pkgs_size, has_conda + +def _main(name, version, download_dir, platform, channel_urls=(), channels_remap=(), specs=(), + exclude=(), menu_packages=(), ignore_duplicate_files=True, environment=None, + environment_file=None, verbose=True, dry_run=False, conda_exe="conda.exe", + transmute_file_type='', extra_envs=None): + precs = _solve_precs( + name, version, download_dir, platform, channel_urls=channel_urls, + channels_remap=channels_remap, specs=specs, exclude=exclude, + menu_packages=menu_packages, environment=environment, + environment_file=environment_file, verbose=verbose, conda_exe=conda_exe + ) + + extra_envs_precs = {} + for env_name, env_config in (extra_envs or {}).items(): + if not any(prec.name == "conda" for prec in precs): + raise RuntimeError("conda needs to be present in `base` environment for extra_envs to work") + + if verbose: + print("Solving extra environment:", env_name) + extra_envs_precs[env_name] = _solve_precs( + f"{name}/envs/{env_name}", version, download_dir, platform, + channel_urls=env_config.get("channels", channel_urls), + channels_remap=env_config.get("channels_remap", channels_remap), + specs=env_config.get("specs", ()), + exclude=exclude, + menu_packages=env_config.get("menu_packages", ()), + environment=env_config.get("environment"), + environment_file=env_config.get("environment_file"), + verbose=verbose, + conda_exe=conda_exe, + extra_env=True, + ) + if dry_run: + return None, None, None, None, None + + pc_recs, _urls, dists, has_conda = _fetch_precs( + precs, download_dir, transmute_file_type=transmute_file_type + ) + + extra_envs_data = {} + for env_name, env_precs in extra_envs_precs.items(): + env_pc_recs, env_urls, env_dists, _ = _fetch_precs( + env_precs, download_dir, transmute_file_type=transmute_file_type + ) + extra_envs_data[env_name] = {"_urls": env_urls, "_dists": env_dists} + pc_recs += env_pc_recs + + duplicate_files = "warn" if ignore_duplicate_files else "error" + if extra_envs_data: # this can cause false positives + print("Info: Skipping duplicate files checks because `extra_envs` in use") + duplicate_files = "skip" + + pc_recs = list({rec: None for rec in pc_recs}) # deduplicate + approx_tarballs_size, approx_pkgs_size = check_duplicates_files( + pc_recs, platform, duplicate_files=duplicate_files + ) + + return _urls, dists, approx_tarballs_size, approx_pkgs_size, has_conda, extra_envs_data def main(info, verbose=True, dry_run=False, conda_exe="conda.exe"): @@ -377,6 +436,7 @@ def main(info, verbose=True, dry_run=False, conda_exe="conda.exe"): environment = info.get("environment", None) environment_file = info.get("environment_file", None) transmute_file_type = info.get("transmute_file_type", "") + extra_envs = info.get("extra_envs", {}) if not channel_urls and not channels_remap: sys.exit("Error: at least one entry in 'channels' or 'channels_remap' is required") @@ -395,14 +455,17 @@ def main(info, verbose=True, dry_run=False, conda_exe="conda.exe"): conda_context.proxy_servers = proxy_servers conda_context.ssl_verify = ssl_verify - _urls, dists, approx_tarballs_size, approx_pkgs_size, has_conda = _main( + (_urls, dists, approx_tarballs_size, approx_pkgs_size, + has_conda, extra_envs_info) = _main( name, version, download_dir, platform, channel_urls, channels_remap, specs, exclude, menu_packages, ignore_duplicate_files, environment, environment_file, - verbose, dry_run, conda_exe, transmute_file_type + verbose, dry_run, conda_exe, transmute_file_type, extra_envs ) - info["_urls"] = _urls - info["_dists"] = dists + info["_urls"] = _urls # needed to mock the repodata cache + info["_dists"] = dists # needed to tell conda what to install info["_approx_tarballs_size"] = approx_tarballs_size info["_approx_pkgs_size"] = approx_pkgs_size info["_has_conda"] = has_conda + # contains {env_name: [_dists, _urls]} for each extra environment + info["_extra_envs_info"] = extra_envs_info \ No newline at end of file diff --git a/constructor/header.sh b/constructor/header.sh index b32badf7f..47f3ddc40 100644 --- a/constructor/header.sh +++ b/constructor/header.sh @@ -482,32 +482,53 @@ export FORCE # https://github.com/conda/conda/pull/9073 mkdir -p ~/.conda > /dev/null 2>&1 +printf "\nInstalling base environment...\n\n" + CONDA_SAFETY_CHECKS=disabled \ CONDA_EXTRA_SAFETY_CHECKS=no \ -CONDA_CHANNELS=__CHANNELS__ \ +CONDA_CHANNELS="__CHANNELS__" \ CONDA_PKGS_DIRS="$PREFIX/pkgs" \ "$CONDA_EXEC" install --offline --file "$PREFIX/pkgs/env.txt" -yp "$PREFIX" || exit 1 - -if [ "$KEEP_PKGS" = "0" ]; then - rm -fr $PREFIX/pkgs/*.tar.bz2 - rm -fr $PREFIX/pkgs/*.conda -fi +rm -f "$PREFIX/pkgs/env.txt" __INSTALL_COMMANDS__ +#if has_conda +mkdir -p $PREFIX/envs +for env_pkgs in ${PREFIX}/pkgs/envs/*/; do + env_name=$(basename ${env_pkgs}) + if [[ "${env_name}" == "*" ]]; then + continue + fi + printf "\nInstalling ${env_name} environment...\n\n" + mkdir -p "$PREFIX/envs/$env_name" + + if [[ -f "${env_pkgs}channels.txt" ]]; then + env_channels=$(cat "${env_pkgs}channels.txt") + rm -f "${env_pkgs}channels.txt" + else + env_channels="__CHANNELS__" + fi + + # TODO: custom shortcuts per env? + CONDA_SAFETY_CHECKS=disabled \ + CONDA_EXTRA_SAFETY_CHECKS=no \ + CONDA_CHANNELS="$env_channels" \ + CONDA_PKGS_DIRS="$PREFIX/pkgs" \ + "$CONDA_EXEC" install --offline --file "${env_pkgs}env.txt" -yp "$PREFIX/envs/$env_name" || exit 1 + rm -f "${env_pkgs}env.txt" +done +#endif + POSTCONDA="$PREFIX/postconda.tar.bz2" "$CONDA_EXEC" constructor --prefix "$PREFIX" --extract-tarball < "$POSTCONDA" || exit 1 rm -f "$POSTCONDA" rm -f $PREFIX/conda.exe -rm -f $PREFIX/pkgs/env.txt rm -rf $PREFIX/install_tmp export TMP="$TMP_BACKUP" -#if has_conda -mkdir -p $PREFIX/envs -#endif #The templating doesn't support nested if statements #if has_post_install diff --git a/constructor/main.py b/constructor/main.py index 5cf5d8739..2123eedaa 100644 --- a/constructor/main.py +++ b/constructor/main.py @@ -123,6 +123,15 @@ def main_build(dir_path, output_dir='.', platform=cc_platform, if any((not s) for s in info[key]): sys.exit("Error: found empty element in '%s:'" % key) + for env_name, env_config in info.get("extra_envs", {}).items(): + if env_name in ("base", "root"): + raise ValueError(f"Environment name '{env_name}' cannot be used") + for config_key, value in env_config.copy().items(): + if isinstance(value, (list, tuple)): + env_config[config_key] = [val.strip() for val in value] + if config_key == "environment_file": + env_config[config_key] = abspath(join(dir_path, value)) + info['installer_type'] = itypes[0] fcp_main(info, verbose=verbose, dry_run=dry_run, conda_exe=conda_exe) if dry_run: @@ -157,6 +166,10 @@ def main_build(dir_path, output_dir='.', platform=cc_platform, fo.write('# installer: %s\n' % basename(info['_outpath'])) for dist in info['_dists']: fo.write('%s\n' % dist) + for env_name, env_info in info["_extra_envs_info"].items(): + fo.write(f"# extra_env: {env_name}\n") + for dist_ in env_info["_dists"]: + fo.write('%s\n' % dist_) def main(): diff --git a/constructor/nsis/main.nsi.tmpl b/constructor/nsis/main.nsi.tmpl index 02e665c2d..d75565a16 100644 --- a/constructor/nsis/main.nsi.tmpl +++ b/constructor/nsis/main.nsi.tmpl @@ -874,7 +874,6 @@ Section "Install" ${EndIf} SetOutPath "$INSTDIR\pkgs" - File __ENV_TXT__ File __URLS_FILE__ File __URLS_TXT_FILE__ File __POST_INSTALL__ @@ -884,38 +883,16 @@ Section "Install" System::Call 'kernel32::SetEnvironmentVariable(t,t)i("CONDA_SAFETY_CHECKS", "disabled").r0' System::Call 'kernel32::SetEnvironmentVariable(t,t)i("CONDA_EXTRA_SAFETY_CHECKS", "no").r0' - System::Call 'kernel32::SetEnvironmentVariable(t,t)i("CONDA_CHANNELS", __CHANNELS__).r0' System::Call 'kernel32::SetEnvironmentVariable(t,t)i("CONDA_PKGS_DIRS", "$INSTDIR\pkgs")".r0' @PKG_COMMANDS@ - SetDetailsPrint TextOnly - DetailPrint "Setting up the package cache ..." - nsExec::ExecToLog '"$INSTDIR\_conda.exe" constructor --prefix "$INSTDIR" --extract-conda-pkgs' - Pop $0 - SetDetailsPrint both - - SetDetailsPrint TextOnly - DetailPrint "Setting up the base environment ..." - # Need to use `--no-shortcuts` because the shortcuts are created in the following steps. The reason is that `conda install` - # doesn't support `menu_packages` entry of `construct.yaml` and will therefore create possible shorcuts - nsExec::ExecToLog '"$INSTDIR\_conda.exe" install --offline -yp "$INSTDIR" --file "$INSTDIR\pkgs\env.txt" --no-shortcuts' - Pop $0 - SetDetailsPrint both - - # Cleanup - SetOutPath "$INSTDIR" - Delete "$INSTDIR\pkgs\env.txt" + @SETUP_ENVS@ @WRITE_CONDARC@ AddSize @SIZE@ - # Restore shipped conda-meta\history for remapped - # channels and retain only the first transaction - SetOutPath "$INSTDIR\conda-meta" - File __CONDA_HISTORY__ - ${If} $Ana_CreateShortcuts_State = ${BST_CHECKED} DetailPrint "Creating @NAME@ menus..." push '"$INSTDIR\_conda.exe" constructor --prefix "$INSTDIR" --make-menus @MENU_PKGS@' diff --git a/constructor/osx/post_extract.sh b/constructor/osx/post_extract.sh index e5db3caf6..7c3bc0782 100644 --- a/constructor/osx/post_extract.sh +++ b/constructor/osx/post_extract.sh @@ -45,10 +45,42 @@ fi # Move the prepackaged history file into place mv "$PREFIX/pkgs/conda-meta/history" "$PREFIX/conda-meta/history" +rm -f "$PREFIX/env.txt" + +# Same, but for the extra environments + +mkdir -p $PREFIX/envs + +for env_pkgs in ${PREFIX}/pkgs/envs/*/; do + env_name=$(basename ${env_pkgs}) + if [[ "${env_name}" == "*" ]]; then + continue + fi + + notify "Installing ${env_name} packages..." + mkdir -p "$PREFIX/envs/$env_name/conda-meta" + touch "$PREFIX/envs/$env_name/conda-meta/history" + + if [[ -f "${env_pkgs}channels.txt" ]]; then + env_channels=$(cat "${env_pkgs}channels.txt") + rm -f "${env_pkgs}channels.txt" + else + env_channels=__CHANNELS__ + fi + # TODO: custom channels per env? + # TODO: custom shortcuts per env? + CONDA_SAFETY_CHECKS=disabled \ + CONDA_EXTRA_SAFETY_CHECKS=no \ + CONDA_CHANNELS="$env_channels" \ + CONDA_PKGS_DIRS="$PREFIX/pkgs" \ + "$CONDA_EXEC" install --offline --file "${env_pkgs}env.txt" -yp "$PREFIX/envs/$env_name" || exit 1 + # Move the prepackaged history file into place + mv "${env_pkgs}/conda-meta/history" "$PREFIX/envs/$env_name/conda-meta/history" + rm -f "${env_pkgs}env.txt" +done # Cleanup! rm -f "$CONDA_EXEC" -rm -f "$PREFIX/env.txt" find "$PREFIX/pkgs" -type d -empty -exec rmdir {} \; 2>/dev/null || : __WRITE_CONDARC__ diff --git a/constructor/osxpkg.py b/constructor/osxpkg.py index f6f319a00..08c6591d3 100644 --- a/constructor/osxpkg.py +++ b/constructor/osxpkg.py @@ -29,7 +29,14 @@ def write_readme(dst, info): with open(dst, 'w') as f: f.write(data) - for dist in sorted(info['_dists']): + + all_dists = info["_dists"].copy() + for env_info in info.get("_extra_envs_info", {}).values(): + all_dists += env_info["_dists"] + all_dists = list({dist: None for dist in all_dists}) # de-duplicate + + # TODO: Split output by env name + for dist in sorted(all_dists): if dist.startswith('_'): continue f.write("{\\listtext\t\n\\f1 \\uc0\\u8259 \n\\f0 \t}%s %s\\\n" % @@ -344,8 +351,14 @@ def create(info, verbose=False): os.makedirs(pkgs_dir) preconda.write_files(info, pkgs_dir) preconda.copy_extra_files(info, prefix) - for dist in info['_dists']: + + all_dists = info["_dists"].copy() + for env_info in info.get("_extra_envs_info", {}).values(): + all_dists += env_info["_dists"] + all_dists = list({dist: None for dist in all_dists}) # de-duplicate + for dist in all_dists: os.link(join(CACHE_DIR, dist), join(pkgs_dir, dist)) + shutil.copyfile(info['_conda_exe'], join(prefix, "conda.exe")) # Sign conda-standalone so it can pass notarization diff --git a/constructor/preconda.py b/constructor/preconda.py index 76eddd171..1491e389e 100644 --- a/constructor/preconda.py +++ b/constructor/preconda.py @@ -18,6 +18,7 @@ from .conda_interface import (CONDA_INTERFACE_VERSION, Dist, MatchSpec, default_prefix, PrefixData, write_repodata, get_repodata, all_channel_urls) from .conda_interface import distro as conda_distro +from .utils import get_final_channels try: import json @@ -34,18 +35,29 @@ def write_index_cache(info, dst_dir, used_packages): os.makedirs(cache_dir) _platforms = info['_platform'], 'noarch' + _remap_configs = list(info.get("channels_remap", [])) + _env_channels = [] + for env_info in info.get("extra_envs", {}).values(): + _remap_configs += env_info.get("channels_remap", []) + _env_channels += env_info.get("channels", []) + _remaps = {url['src'].rstrip('/'): url['dest'].rstrip('/') - for url in info.get('channels_remap', [])} + for url in _remap_configs} _channels = [ url.rstrip("/") for url in list(_remaps) + info.get("channels", []) + info.get("conda_default_channels", []) + + _env_channels ] _urls = all_channel_urls(_channels, subdirs=_platforms) repodatas = {url: get_repodata(url) for url in _urls if url is not None} - for url, _ in info['_urls']: + all_urls = info["_urls"].copy() + for env_info in info.get("_extra_envs_info", {}).values(): + all_urls += env_info["_urls"] + + for url, _ in all_urls: src, subdir, fn = url.rsplit('/', 2) dst = _remaps.get(src) if dst is not None: @@ -66,7 +78,8 @@ def write_index_cache(info, dst_dir, used_packages): del repodatas['%s/%s' % (src, subdir)] for url, repodata in repodatas.items(): - write_repodata(cache_dir, url, repodata, used_packages, info) + if repodata is not None: + write_repodata(cache_dir, url, repodata, used_packages, info) for cache_file in os.listdir(cache_dir): if not cache_file.endswith(".json"): @@ -103,30 +116,57 @@ def write_files(info, dst_dir): with open(join(dst_dir, '.constructor-build.info'), 'w') as fo: json.dump(system_info(), fo) - final_urls_md5s = tuple((get_final_url(info, url), md5) for url, md5 in info['_urls']) + all_urls = info["_urls"].copy() + for env_info in info.get("_extra_envs_info", {}).values(): + all_urls += env_info["_urls"] + + final_urls_md5s = tuple((get_final_url(info, url), md5) for url, md5 in info["_urls"]) + all_final_urls_md5s = tuple((get_final_url(info, url), md5) for url, md5 in all_urls) with open(join(dst_dir, 'urls'), 'w') as fo: - for url, md5 in final_urls_md5s: + for url, md5 in all_final_urls_md5s: fo.write('%s#%s\n' % (url, md5)) with open(join(dst_dir, 'urls.txt'), 'w') as fo: - for url, _ in final_urls_md5s: + for url, _ in all_final_urls_md5s: fo.write('%s\n' % url) - write_index_cache(info, dst_dir, info['_dists']) + all_dists = info["_dists"].copy() + for env_info in info.get("_extra_envs_info", {}).values(): + all_dists += env_info["_dists"] + all_dists = list({dist: None for dist in all_dists}) # de-duplicate + + write_index_cache(info, dst_dir, all_dists) + # base environment conda-meta write_conda_meta(info, dst_dir, final_urls_md5s) write_repodata_record(info, dst_dir) - write_env_txt(info, dst_dir) + # base environment file used with conda install --file + # (list of specs/dists to install) + write_env_txt(info, dst_dir, info["_dists"]) for fn in files: os.chmod(join(dst_dir, fn), 0o664) + for env_name, env_info in info.get("_extra_envs_info", {}).items(): + env_config = info["extra_envs"][env_name] + env_dst_dir = os.path.join(dst_dir, "envs", env_name) + # environment conda-meta + env_urls_md5 = tuple((get_final_url(info, url), md5) for url, md5 in env_info["_urls"]) + user_requested_specs = env_config.get('user_requested_specs', env_config.get('specs', ())) + write_conda_meta(info, env_dst_dir, env_urls_md5, user_requested_specs) + # environment installation list + write_env_txt(info, env_dst_dir, env_info["_dists"]) + # channels + write_channels_txt(info, env_dst_dir, env_config) + + +def write_conda_meta(info, dst_dir, final_urls_md5s, user_requested_specs=None): + if user_requested_specs is None: + user_requested_specs = info.get('user_requested_specs', info.get('specs', ())) -def write_conda_meta(info, dst_dir, final_urls_md5s): - user_requested_specs = info.get('user_requested_specs', info.get('specs', ())) cmd = path_split(sys.argv[0])[-1] if len(sys.argv) > 1: cmd = "%s %s" % (cmd, " ".join(sys.argv[1:])) @@ -150,7 +190,10 @@ def write_conda_meta(info, dst_dir, final_urls_md5s): def write_repodata_record(info, dst_dir): - for dist in info['_dists']: + all_dists = info["_dists"].copy() + for env_data in info.get("_extra_envs_info", {}).values(): + all_dists += env_data["_dists"] + for dist in all_dists: if filename_dist(dist).endswith(".conda"): _dist = filename_dist(dist)[:-6] elif filename_dist(dist).endswith(".tar.bz2"): @@ -173,9 +216,11 @@ def write_repodata_record(info, dst_dir): json.dump(rr_json, rf, indent=2, sort_keys=True) -def write_env_txt(info, dst_dir): +def write_env_txt(info, dst_dir, dists=None): + if dists is None: + dists = info["_dists"] dists_san_extn = [] - for dist in info['_dists']: + for dist in dists: if filename_dist(dist).endswith('.conda'): dists_san_extn.append(filename_dist(dist)[:-6]) elif filename_dist(dist).endswith('.tar.bz2'): @@ -184,6 +229,16 @@ def write_env_txt(info, dst_dir): with open(join(dst_dir, "env.txt"), "w") as envf: envf.write('\n'.join(specs)) +def write_channels_txt(info, dst_dir, env_config): + env_config = env_config.copy() + if "channels" not in env_config: + env_config["channels"] = info.get("channels", ()) + if "channels_remap" not in env_config: + env_config["channels_remap"] = info.get("channels_remap", ()) + + with open(join(dst_dir, "channels.txt"), "w") as f: + f.write(",".join(get_final_channels(env_config))) + def copy_extra_files(info, workdir): extra_files = info.get('extra_files') diff --git a/constructor/shar.py b/constructor/shar.py index 6b81ed1f2..aa8929814 100644 --- a/constructor/shar.py +++ b/constructor/shar.py @@ -100,6 +100,11 @@ def create(info, verbose=False): for dist in preconda_files: fn = filename_dist(dist) pre_t.add(join(tmp_dir, fn), 'pkgs/' + fn) + + for env_name in info.get("_extra_envs_info", ()): + pre_t.add(join(tmp_dir, "envs", env_name, "env.txt"), + f"pkgs/envs/{env_name}/env.txt") + for key in 'pre_install', 'post_install': if key in info: pre_t.add(info[key], 'pkgs/%s.sh' % key, @@ -109,7 +114,13 @@ def create(info, verbose=False): for cf in os.listdir(cache_dir): if cf.endswith(".json"): pre_t.add(join(cache_dir, cf), 'pkgs/cache/' + cf) - for dist in info['_dists']: + + all_dists = info["_dists"].copy() + for env_data in info.get("_extra_envs_info", {}).values(): + all_dists += env_data["_dists"] + all_dists = list({dist: None for dist in all_dists}) # de-duplicate + + for dist in all_dists: if filename_dist(dist).endswith(".conda"): _dist = filename_dist(dist)[:-6] elif filename_dist(dist).endswith(".tar.bz2"): @@ -120,6 +131,11 @@ def create(info, verbose=False): pre_t.add(record_file_src, record_file_dest) pre_t.addfile(tarinfo=tarfile.TarInfo("conda-meta/history")) post_t.add(join(tmp_dir, 'conda-meta', 'history'), 'conda-meta/history') + + for env_name in info.get("_extra_envs_info", {}): + pre_t.addfile(tarinfo=tarfile.TarInfo(f"envs/{env_name}/conda-meta/history")) + post_t.add(join(tmp_dir, 'envs', env_name, 'conda-meta', 'history'), + f"envs/{env_name}/conda-meta/history") extra_files = copy_extra_files(info, tmp_dir) for path in extra_files: @@ -134,7 +150,7 @@ def create(info, verbose=False): t.add(postconda_tarball, basename(postconda_tarball)) if 'license_file' in info: t.add(info['license_file'], 'LICENSE.txt') - for dist in info['_dists']: + for dist in all_dists: fn = filename_dist(dist) t.add(join(info['_download_dir'], fn), 'pkgs/' + fn) t.close() diff --git a/constructor/winexe.py b/constructor/winexe.py index 262677b31..9dd90b7b2 100644 --- a/constructor/winexe.py +++ b/constructor/winexe.py @@ -58,11 +58,79 @@ def extra_files_commands(paths, common_parent): return lines +def setup_envs_commands(info, dir_path): + template = """ + # Set up {name} env + SetDetailsPrint TextOnly + DetailPrint "Setting up the {name} environment ..." + SetDetailsPrint both + # List of packages to install + SetOutPath "{env_txt_dir}" + File {env_txt_abspath} + # A conda-meta\history file is required for a valid conda prefix + SetOutPath "{conda_meta}" + FileOpen $0 "history" w + FileClose $0 + # Set channels + System::Call 'kernel32::SetEnvironmentVariable(t,t)i("CONDA_CHANNELS", "{channels}").r0' + # Run conda + SetDetailsPrint TextOnly + nsExec::ExecToLog '"$INSTDIR\_conda.exe" install --offline -yp "{prefix}" --file "{env_txt}" {shortcuts}' + Pop $0 + SetDetailsPrint both + # Cleanup {name} env.txt + SetOutPath "$INSTDIR" + Delete "{env_txt}" + # Restore shipped conda-meta\history for remapped + # channels and retain only the first transaction + SetOutPath "{conda_meta}" + File {history_abspath} + """ + + lines = template.format( # this one block is for the base environment + name="base", + prefix=r"$INSTDIR", + env_txt=r"$INSTDIR\pkgs\env.txt", # env.txt as seen by the running installer + env_txt_dir=r"$INSTDIR\pkgs", # env.txt location in the installer filesystem + env_txt_abspath=join(dir_path, "env.txt"), # env.txt location while building the installer + conda_meta=r"$INSTDIR\conda-meta", + history_abspath=join(dir_path, "conda-meta", "history"), + channels=','.join(get_final_channels(info)), + shortcuts="--no-shortcuts" + ).splitlines() + # now we generate one more block per extra env, if present + for env_name in info.get("_extra_envs_info", {}): + lines += ["", ""] + env_info = info["extra_envs"][env_name] + channel_info = { + "channels": env_info.get("channels", info.get("channels", ())), + "channels_remap": env_info.get("channels_remap", info.get("channels_remap", ())) + } + lines += template.format( + name=env_name, + prefix=join("$INSTDIR", "envs", env_name), + env_txt=join("$INSTDIR", "pkgs", "envs", env_name, "env.txt"), + env_txt_dir=join("$INSTDIR", "pkgs", "envs", env_name), + env_txt_abspath=join(dir_path, "envs", env_name, "env.txt"), + conda_meta=join("$INSTDIR", "envs", env_name, "conda-meta"), + history_abspath=join(dir_path, "envs", env_name, "conda-meta", "history"), + channels=",".join(get_final_channels(channel_info)), + shortcuts="", + ).splitlines() + + return [line.strip() for line in lines] + + def make_nsi(info, dir_path, extra_files=()): "Creates the tmp/main.nsi from the template file" name = info['name'] download_dir = info['_download_dir'] - dists = info['_dists'] + + dists = info['_dists'].copy() + for env_info in info["_extra_envs_info"].values(): + dists += env_info["_dists"] + dists = list({dist: None for dist in dists}) # de-duplicate + py_name, py_version, unused_build = filename_dist(dists[0]).rsplit('-', 2) assert py_name == 'python' arch = int(info['_platform'].split('-')[1]) @@ -100,7 +168,6 @@ def make_nsi(info, dir_path, extra_files=()): 'PRE_UNINSTALL': '@pre_uninstall.bat', 'INDEX_CACHE': '@cache', 'REPODATA_RECORD': '@repodata_record.json', - 'CHANNELS': ','.join(get_final_channels(info)) } for key, value in replace.items(): if value.startswith('@'): @@ -134,6 +201,7 @@ def make_nsi(info, dir_path, extra_files=()): ('@NSIS_DIR@', NSIS_DIR), ('@BITS@', str(arch)), ('@PKG_COMMANDS@', '\n '.join(pkg_commands(download_dir, dists))), + ('@SETUP_ENVS@', '\n '.join(setup_envs_commands(info, dir_path))), ('@WRITE_CONDARC@', '\n '.join(add_condarc(info))), ('@MENU_PKGS@', ' '.join(info.get('menu_packages', []))), ('@SIZE@', str(approx_pkgs_size_kb)), diff --git a/examples/extra_envs/construct.yaml b/examples/extra_envs/construct.yaml new file mode 100644 index 000000000..e717775e6 --- /dev/null +++ b/examples/extra_envs/construct.yaml @@ -0,0 +1,20 @@ +name: ExtraEnvs +version: X +installer_type: all +channels: + - http://repo.anaconda.com/pkgs/main/ +specs: + - python=3.7 + - conda # conda is required for extra_envs + - console_shortcut # [win] +extra_envs: + py38: + specs: + - python=3.8 + channels: + - conda-forge + dav1d: + environment_file: dav1d_env.yaml + +post_install: test_install.sh # [unix] +post_install: test_install.bat # [win] diff --git a/examples/extra_envs/dav1d_env.yaml b/examples/extra_envs/dav1d_env.yaml new file mode 100644 index 000000000..743a5fea6 --- /dev/null +++ b/examples/extra_envs/dav1d_env.yaml @@ -0,0 +1,5 @@ +name: NOT_USED +channels: + - conda-forge +dependencies: + - dav1d diff --git a/examples/extra_envs/test_install.bat b/examples/extra_envs/test_install.bat new file mode 100644 index 000000000..2d70efb1a --- /dev/null +++ b/examples/extra_envs/test_install.bat @@ -0,0 +1,18 @@ +:: base env +if not exist "%PREFIX%\conda-meta\history" exit 1 +"%PREFIX%\python.exe" -c "from sys import version_info; assert version_info[:2] == (3, 7)" || goto :error + +:: extra env named 'py38' +if not exist "%PREFIX%\envs\py38\conda-meta\history" exit 1 +"%PREFIX%\envs\py38\python.exe" -c "from sys import version_info; assert version_info[:2] == (3, 8)" || goto :error + +:: extra env named 'dav1d' only contains dav1d, no python +if not exist "%PREFIX%\envs\dav1d\conda-meta\history" exit 1 +if exist "%PREFIX%\envs\dav1d\python.exe" exit 1 +"%PREFIX%\envs\dav1d\Library\bin\dav1d.exe" --version || goto :error + +goto :EOF + +:error +echo Failed with error #%errorlevel%. +exit %errorlevel% diff --git a/examples/extra_envs/test_install.sh b/examples/extra_envs/test_install.sh new file mode 100644 index 000000000..6ccccfecb --- /dev/null +++ b/examples/extra_envs/test_install.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -ex + +# if PREFIX is not defined, then this was called from an OSX PKG installer +# $2 is the install location, ($HOME by default) +if [ xxx$PREFIX = 'xxx' ]; then + PREFIX=$(cd "$2/__NAME_LOWER__"; pwd) +fi + +# tests +# base environment uses python 3.7 +test -f "$PREFIX/conda-meta/history" +"$PREFIX/bin/python" -c "from sys import version_info; assert version_info[:2] == (3, 7)" +"$PREFIX/bin/pip" -V + + +# extra env named 'py38' uses python 3.8 +test -f "$PREFIX/envs/py38/conda-meta/history" +"$PREFIX/envs/py38/bin/python" -c "from sys import version_info; assert version_info[:2] == (3, 8)" +"$PREFIX/envs/py38/bin/pip" -V + +# this env only contains dav1d, no python; it should have been created with no errors, +# even if we had excluded tk from the package list +test -f "$PREFIX/envs/dav1d/conda-meta/history" +test ! -f "$PREFIX/envs/dav1d/bin/python" +"$PREFIX/envs/dav1d/bin/dav1d" --version