From 44bd1db7a6237784dfe42c5471ff13ad336a9c58 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Sat, 9 Nov 2024 13:31:29 +0100 Subject: [PATCH] Use Jinja for SH installers --- constructor/construct.py | 4 +- constructor/header.sh | 249 +++++++++++++++++---------------------- constructor/jinja.py | 19 ++- constructor/shar.py | 81 ++++++------- dev/environment.yml | 1 + pyproject.toml | 1 + tests/test_header.py | 118 ++++--------------- 7 files changed, 189 insertions(+), 284 deletions(-) diff --git a/constructor/construct.py b/constructor/construct.py index f556f566f..bee91f7d2 100644 --- a/constructor/construct.py +++ b/constructor/construct.py @@ -744,10 +744,10 @@ def yamlize(data, directory, content_filter): if ('{{' not in data) and ('{%' not in data): raise UnableToParse(original=e) try: - from constructor.jinja import render_jinja + from constructor.jinja import render_jinja_for_input_file except ImportError as ex: raise UnableToParseMissingJinja2(original=ex) - data = render_jinja(data, directory, content_filter) + data = render_jinja_for_input_file(data, directory, content_filter) return yaml.load(data) diff --git a/constructor/header.sh b/constructor/header.sh index 923efe62a..fd669d416 100644 --- a/constructor/header.sh +++ b/constructor/header.sh @@ -1,28 +1,28 @@ #!/bin/sh # -# Created by constructor __CONSTRUCTOR_VERSION__ +# Created by constructor {{ constructor_version }} # -# NAME: __NAME__ -# VER: __VERSION__ -# PLAT: __PLAT__ -# MD5: __MD5__ +# NAME: {{ installer_name }} +# VER: {{ installer_version }} +# PLAT: {{ installer_platform }} +# MD5: {{ installer_md5 }} set -eu -#if osx +{%- if osx %} unset DYLD_LIBRARY_PATH DYLD_FALLBACK_LIBRARY_PATH -#else +{%- else %} export OLD_LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}" unset LD_LIBRARY_PATH -#endif +{%- endif %} if ! echo "$0" | grep '\.sh$' > /dev/null; then printf 'Please run using "bash"/"dash"/"sh"/"zsh", but not "." or "source".\n' >&2 return 1 fi -#if osx and min_osx_version -min_osx_version="__MIN_OSX_VERSION__" +{%- if osx and min_osx_version %} +min_osx_version="{{ min_osx_version }}" system_osx_version=$(SYSTEM_VERSION_COMPAT=0 sw_vers -productVersion) # shellcheck disable=SC2183 disable=SC2046 int_min_osx_version="$(printf "%02d%02d%02d" $(echo "$min_osx_version" | sed 's/\./ /g'))" @@ -32,9 +32,8 @@ if [ "$int_system_osx_version" -lt "$int_min_osx_version" ]; then echo "Installer requires macOS >=${min_osx_version}, but system has ${system_osx_version}." exit 1 fi -#endif -#if linux and min_glibc_version -min_glibc_version="__MIN_GLIBC_VERSION__" +{%- elif linux and min_glibc_version %} +min_glibc_version="{{ min_glibc_version }}" case "$(ldd --version 2>&1)" in *musl*) # musl ldd will report musl version; call libc.so directly @@ -64,36 +63,36 @@ if [ "$int_system_glibc_version" -lt "$int_min_glibc_version" ]; then echo "Installer requires GLIBC >=${min_glibc_version}, but system has ${system_glibc_version}." exit 1 fi -#endif +{%- endif %} # Export variables to make installer metadata available to pre/post install scripts # NOTE: If more vars are added, make sure to update the examples/scripts tests too -_SCRIPT_ENV_VARIABLES_='' # Templated extra environment variable(s) -export INSTALLER_NAME='__NAME__' -export INSTALLER_VER='__VERSION__' -export INSTALLER_PLAT='__PLAT__' +{{ script_env_variables }} +export INSTALLER_NAME='{{ installer_name }}' +export INSTALLER_VER='{{ installer_version }}' +export INSTALLER_PLAT='{{ installer_platform }}' export INSTALLER_TYPE="SH" THIS_DIR=$(DIRNAME=$(dirname "$0"); cd "$DIRNAME"; pwd) THIS_FILE=$(basename "$0") THIS_PATH="$THIS_DIR/$THIS_FILE" -PREFIX="__DEFAULT_PREFIX__" -#if batch_mode +PREFIX="{{ default_prefix }}" +{%- if batch_mode %} BATCH=1 -#else +{%- else %} BATCH=0 -#endif +{%- endif %} FORCE=0 -#if keep_pkgs +{%- if keep_pkgs %} KEEP_PKGS=1 -#else +{%- else %} KEEP_PKGS=0 -#endif +{%- endif %} SKIP_SCRIPTS=0 -#if enable_shortcuts == "true" +{%- if enable_shortcuts == "true" %} SKIP_SHORTCUTS=0 -#endif +{%- endif %} TEST=0 REINSTALL=0 USAGE=" @@ -101,41 +100,41 @@ usage: $0 [options] Installs ${INSTALLER_NAME} ${INSTALLER_VER} -#if batch_mode +{%- if batch_mode %} -i run install in interactive mode -#else +{%- else %} -b run install in batch mode (without manual intervention), it is expected the license terms (if any) are agreed upon -#endif +{%- endif %} -f no error if install prefix already exists -h print this help message and exit -#if not keep_pkgs +{%- if not keep_pkgs %} -k do not clear the package cache after installation -#endif -#if check_path_spaces +{%- endif %} +{%- if check_path_spaces %} -p PREFIX install prefix, defaults to $PREFIX, must not contain spaces. -#else +{%- else %} -p PREFIX install prefix, defaults to $PREFIX -#endif +{%- endif %} -s skip running pre/post-link/install scripts -#if enable_shortcuts == 'true' +{%- if enable_shortcuts == 'true' %} -m disable the creation of menu items / shortcuts -#endif +{%- endif %} -u update an existing installation -#if has_conda +{%- if has_conda %} -t run package tests after installation (may install conda-build) -#endif +{%- endif %} " # We used to have a getopt version here, falling back to getopts if needed # However getopt is not standardized and the version on Mac has different # behaviour. getopts is good enough for what we need :) # More info: https://unix.stackexchange.com/questions/62950/ -#if enable_shortcuts == "true" +{%- if enable_shortcuts == "true" %} while getopts "bifhkp:smut" x; do -#else +{%- else %} while getopts "bifhkp:sut" x; do -#endif +{%- endif %} case "$x" in h) printf "%s\\n" "$USAGE" @@ -159,19 +158,19 @@ while getopts "bifhkp:sut" x; do s) SKIP_SCRIPTS=1 ;; -#if enable_shortcuts == "true" +{%- if enable_shortcuts == "true" %} m) SKIP_SHORTCUTS=1 ;; -#endif +{%- endif %} u) FORCE=1 ;; -#if has_conda +{%- if has_conda %} t) TEST=1 ;; -#endif +{%- endif %} ?) printf "ERROR: did not recognize option '%s', please try -h\\n" "$x" exit 1 @@ -191,7 +190,7 @@ fi if [ "$BATCH" = "0" ] # interactive mode then -#if x86 and not x86_64 +{%- if x86 and not x86_64 %} if [ "$(uname -m)" = "x86_64" ]; then printf "WARNING:\\n" printf " Your system is x86_64, but you are trying to install an x86 (32-bit)\\n" @@ -209,9 +208,7 @@ then exit 2 fi fi -#endif - -#if x86_64 +{%- elif x86_64 %} if [ "$(uname -m)" != "x86_64" ]; then printf "WARNING:\\n" printf " Your operating system appears not to be 64-bit, but you are trying to\\n" @@ -226,9 +223,7 @@ then exit 2 fi fi -#endif - -#if ppc64le +{%- elif ppc64le %} if [ "$(uname -m)" != "ppc64le" ]; then printf "WARNING:\\n" printf " Your machine hardware does not appear to be Power8 (little endian), \\n" @@ -243,9 +238,7 @@ then exit 2 fi fi -#endif - -#if s390x +{%- elif s390x %} if [ "$(uname -m)" != "s390x" ]; then printf "WARNING:\\n" printf " Your machine hardware does not appear to be s390x (big endian), \\n" @@ -260,9 +253,7 @@ then exit 2 fi fi -#endif - -#if aarch64 +{%- elif aarch64 %} if [ "$(uname -m)" != "aarch64" ]; then printf "WARNING:\\n" printf " Your machine hardware does not appear to be aarch64, \\n" @@ -277,9 +268,9 @@ then exit 2 fi fi -#endif +{%- endif %} -#if osx +{%- if osx %} if [ "$(uname)" != "Darwin" ]; then printf "WARNING:\\n" printf " Your operating system does not appear to be macOS, \\n" @@ -294,9 +285,7 @@ then exit 2 fi fi -#endif - -#if linux +{%- elif linux %} if [ "$(uname)" != "Linux" ]; then printf "WARNING:\\n" printf " Your operating system does not appear to be Linux, \\n" @@ -311,11 +300,11 @@ then exit 2 fi fi -#endif +{%- endif %} printf "\\n" printf "Welcome to %s %s\\n" "${INSTALLER_NAME}" "${INSTALLER_VER}" -#if has_license +{%- if has_license %} printf "\\n" printf "In order to continue the installation process, please review the license\\n" printf "agreement.\\n" @@ -327,7 +316,7 @@ then pager="more" fi "$pager" <<'EOF' -__LICENSE__ +{{ license }} EOF printf "\\n" printf "Do you accept the license terms? [yes|no]\\n" @@ -346,7 +335,7 @@ EOF printf "The license agreement wasn't approved, aborting installation.\\n" exit 2 fi -#endif +{%- endif %} printf "\\n" printf "%s will now be installed into this location:\\n" "${INSTALLER_NAME}" @@ -359,7 +348,7 @@ EOF printf "[%s] >>> " "$PREFIX" read -r user_prefix if [ "$user_prefix" != "" ]; then -#if check_path_spaces is True +{%- if check_path_spaces %} case "$user_prefix" in *\ * ) printf "ERROR: Cannot install into directories with spaces\\n" >&2 @@ -369,20 +358,20 @@ EOF eval PREFIX="$user_prefix" ;; esac -#else +{%- else %} PREFIX="$user_prefix" -#endif +{%- endif %} fi fi # !BATCH -#if check_path_spaces is True +{%- if check_path_spaces %} case "$PREFIX" in *\ * ) printf "ERROR: Cannot install into directories with spaces\\n" >&2 exit 1 ;; esac -#endif +{%- endif %} if [ "$FORCE" = "0" ] && [ -e "$PREFIX" ]; then printf "ERROR: File or directory already exists: '%s'\\n" "$PREFIX" >&2 @@ -392,7 +381,7 @@ elif [ "$FORCE" = "1" ] && [ -e "$PREFIX" ]; then REINSTALL=1 fi -total_installation_size_kb="__TOTAL_INSTALLATION_SIZE_KB__" +total_installation_size_kb="{{ total_installation_size_kb }}" total_installation_size_mb="$(( total_installation_size_kb / 1024 ))" if ! mkdir -p "$PREFIX"; then printf "ERROR: Could not create directory: '%s'.\\n" "$PREFIX" >&2 @@ -442,20 +431,20 @@ last_line=$(grep -anm 1 '^@@END_HEADER@@' "$THIS_PATH" | sed 's/:.*//') # the start of the first payload, in bytes, indexed from zero boundary0=$(head -n "${last_line}" "${THIS_PATH}" | wc -c | sed 's/ //g') # the start of the second payload / the end of the first payload, plus one -boundary1=$(( boundary0 + __FIRST_PAYLOAD_SIZE__ )) +boundary1=$(( boundary0 + {{ first_payload_size }} )) # the end of the second payload, plus one -boundary2=$(( boundary1 + __SECOND_PAYLOAD_SIZE__ )) +boundary2=$(( boundary1 + {{ second_payload_size }} )) # verify the MD5 sum of the tarball appended to this header -#if osx +{%- if osx %} MD5=$(extract_range "${boundary0}" "${boundary2}" | md5) -#else +{%- else %} MD5=$(extract_range "${boundary0}" "${boundary2}" | md5sum -) -#endif +{%- endif %} -if ! echo "$MD5" | grep __MD5__ >/dev/null; then +if ! echo "$MD5" | grep {{ installer_md5 }} >/dev/null; then printf "WARNING: md5sum mismatch of tar archive\\n" >&2 - printf "expected: __MD5__\\n" >&2 + printf "expected: {{ installer_md5 }}\\n" >&2 printf " got: %s\\n" "$MD5" >&2 fi @@ -480,13 +469,13 @@ mkdir -p "$TMP" # micromamba needs an existing pkgs_dir to operate even offline, # but we haven't created $PREFIX/pkgs yet... give it a temp location # shellcheck disable=SC2050 -if [ "__VIRTUAL_SPECS__" != "" ]; then - echo 'Checking virtual specs compatibility: __VIRTUAL_SPECS__' +{%- if virtual_specs %} + echo 'Checking virtual specs compatibility: {{ virtual_specs }}' CONDA_QUIET="$BATCH" \ CONDA_SOLVER="classic" \ CONDA_PKGS_DIRS="$(mktemp -d)" \ - "$CONDA_EXEC" create --dry-run --prefix "$PREFIX/envs/_virtual_specs_checks" --offline __VIRTUAL_SPECS__ __NO_RCS_ARG__ -fi + "$CONDA_EXEC" create --dry-run --prefix "$PREFIX/envs/_virtual_specs_checks" --offline {{ virtual_specs }} {{ no_rcs_arg }} +{%- endif %} # Create $PREFIX/.nonadmin if the installation didn't require superuser permissions if [ "$(id -u)" -ne 0 ]; then @@ -495,7 +484,7 @@ fi # the second binary payload: the tarball of packages printf "Unpacking payload ...\n" -extract_range $boundary1 $boundary2 | \ +extract_range "$boundary1" "$boundary2" | \ CONDA_QUIET="$BATCH" "$CONDA_EXEC" constructor --extract-tarball --prefix "$PREFIX" PRECONDA="$PREFIX/preconda.tar.bz2" @@ -506,26 +495,22 @@ rm -f "$PRECONDA" CONDA_QUIET="$BATCH" \ "$CONDA_EXEC" constructor --prefix "$PREFIX" --extract-conda-pkgs || exit 1 -#The templating doesn't support nested if statements -#if has_pre_install +{%- if has_pre_install %} if [ "$SKIP_SCRIPTS" = "1" ]; then export INST_OPT='--skip-scripts' printf "WARNING: skipping pre_install.sh by user request\\n" >&2 else export INST_OPT='' -#endif -#if has_pre_install and direct_execute_pre_install +{%- if direct_execute_pre_install %} if ! "$PREFIX/pkgs/pre_install.sh"; then -#endif -#if has_pre_install and not direct_execute_pre_install +{%- else %} if ! sh "$PREFIX/pkgs/pre_install.sh"; then -#endif -#if has_pre_install +{%- endif %} printf "ERROR: executing pre_install.sh failed\\n" >&2 exit 1 fi fi -#endif +{%- endif %} MSGS="$PREFIX/.messages.txt" touch "$MSGS" @@ -541,33 +526,30 @@ test -d ~/.conda || mkdir -p ~/.conda >/dev/null 2>/dev/null || test -d ~/.conda printf "\nInstalling base environment...\n\n" -#if enable_shortcuts == "true" +{%- if enable_shortcuts == "true" %} if [ "$SKIP_SHORTCUTS" = "1" ]; then shortcuts="--no-shortcuts" else - shortcuts="__SHORTCUTS__" + shortcuts="{{ shortcuts }}" fi -#endif -#if enable_shortcuts == "false" +{%- elif enable_shortcuts == "false" %} shortcuts="--no-shortcuts" -#endif -#if enable_shortcuts == "incompatible" +{%- elif enable_shortcuts == "incompatible" %} shortcuts="" -#endif +{%- endif %} # shellcheck disable=SC2086 CONDA_ROOT_PREFIX="$PREFIX" \ -CONDA_REGISTER_ENVS="__REGISTER_ENVS__" \ +CONDA_REGISTER_ENVS="{{ register_envs }}" \ CONDA_SAFETY_CHECKS=disabled \ CONDA_EXTRA_SAFETY_CHECKS=no \ -CONDA_CHANNELS="__CHANNELS__" \ +CONDA_CHANNELS="{{ channels }}" \ CONDA_PKGS_DIRS="$PREFIX/pkgs" \ CONDA_QUIET="$BATCH" \ -"$CONDA_EXEC" install --offline --file "$PREFIX/pkgs/env.txt" -yp "$PREFIX" $shortcuts __NO_RCS_ARG__ || exit 1 +"$CONDA_EXEC" install --offline --file "$PREFIX/pkgs/env.txt" -yp "$PREFIX" $shortcuts {{ no_rcs_arg }} || exit 1 rm -f "$PREFIX/pkgs/env.txt" -#The templating doesn't support nested if statements -#if has_conda +{%- if has_conda %} mkdir -p "$PREFIX/envs" for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do env_name=$(basename "${env_pkgs}") @@ -581,10 +563,9 @@ for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do env_channels=$(cat "${env_pkgs}channels.txt") rm -f "${env_pkgs}channels.txt" else - env_channels="__CHANNELS__" + env_channels="{{ channels }}" fi -#endif -#if has_conda and enable_shortcuts == "true" +{%- if enable_shortcuts == "true" %} if [ "$SKIP_SHORTCUTS" = "1" ]; then env_shortcuts="--no-shortcuts" else @@ -592,28 +573,25 @@ for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do env_shortcuts=$(cat "${env_pkgs}shortcuts.txt") rm -f "${env_pkgs}shortcuts.txt" fi -#endif -#if has_conda and enable_shortcuts == "false" +{%- elif enable_shortcuts == "false" %} env_shortcuts="--no-shortcuts" -#endif -#if has_conda and enable_shortcuts == "incompatible" +{%- elif enable_shortcuts == "incompatible" %} env_shortcuts="" -#endif -#if has_conda +{%- endif %} # shellcheck disable=SC2086 CONDA_ROOT_PREFIX="$PREFIX" \ - CONDA_REGISTER_ENVS="__REGISTER_ENVS__" \ + CONDA_REGISTER_ENVS="{{ register_envs }}" \ CONDA_SAFETY_CHECKS=disabled \ CONDA_EXTRA_SAFETY_CHECKS=no \ CONDA_CHANNELS="$env_channels" \ CONDA_PKGS_DIRS="$PREFIX/pkgs" \ CONDA_QUIET="$BATCH" \ - "$CONDA_EXEC" install --offline --file "${env_pkgs}env.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts __NO_RCS_ARG__ || exit 1 + "$CONDA_EXEC" install --offline --file "${env_pkgs}env.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts {{ no_rcs_arg }} || exit 1 rm -f "${env_pkgs}env.txt" done -#endif +{%- endif %} -__INSTALL_COMMANDS__ +{{ install_commands }} POSTCONDA="$PREFIX/postconda.tar.bz2" CONDA_QUIET="$BATCH" \ @@ -624,23 +602,20 @@ export TMP="$TMP_BACKUP" #The templating doesn't support nested if statements -#if has_post_install +{%- if has_post_install %} if [ "$SKIP_SCRIPTS" = "1" ]; then printf "WARNING: skipping post_install.sh by user request\\n" >&2 else -#endif -#if has_post_install and direct_execute_post_install +{%- if direct_execute_post_install %} if ! "$PREFIX/pkgs/post_install.sh"; then -#endif -#if has_post_install and not direct_execute_post_install +{%- else %} if ! sh "$PREFIX/pkgs/post_install.sh"; then -#endif -#if has_post_install +{%- endif %} printf "ERROR: executing post_install.sh failed\\n" >&2 exit 1 fi fi -#endif +{%- endif %} if [ -f "$MSGS" ]; then cat "$MSGS" @@ -655,7 +630,7 @@ else fi cat <<'EOF' -__CONCLUSION_TEXT__ +{{ conclusion_text }} EOF if [ "${PYTHONPATH:-}" != "" ]; then @@ -668,14 +643,12 @@ if [ "${PYTHONPATH:-}" != "" ]; then fi if [ "$BATCH" = "0" ]; then -#if initialize_conda is True and initialize_by_default is True +{%- if has_conda and initialize_conda %} +{%- if initialize_by_default %} DEFAULT=yes -#endif -#if initialize_conda is True and initialize_by_default is False +{%- else %} DEFAULT=no -#endif - -#if has_conda and initialize_conda is True +{%- endif %} # Interactive mode. printf "Do you wish to update your shell profile to automatically initialize conda?\\n" @@ -718,13 +691,13 @@ if [ "$BATCH" = "0" ]; then esac fi fi -#endif +{%- endif %} printf "Thank you for installing %s!\\n" "${INSTALLER_NAME}" fi # !BATCH -#if has_conda +{%- if has_conda %} if [ "$TEST" = "1" ]; then printf "INFO: Running package tests in a subshell\\n" NFAILS=0 @@ -757,7 +730,7 @@ if [ "$TEST" = "1" ]; then exit $NFAILS fi fi -#endif +{%- endif %} exit 0 # shellcheck disable=SC2317 diff --git a/constructor/jinja.py b/constructor/jinja.py index 41f65ee2a..40ad1c45b 100644 --- a/constructor/jinja.py +++ b/constructor/jinja.py @@ -1,8 +1,9 @@ import os -from jinja2 import BaseLoader, Environment, FileSystemLoader, TemplateError +from jinja2 import BaseLoader, Environment, FileSystemLoader, TemplateError, StrictUndefined -from constructor.exceptions import UnableToParse +from . import __version__ +from .exceptions import UnableToParse # adapted from conda-build @@ -25,7 +26,7 @@ def get_source(self, environment, template): # adapted from conda-build -def render_jinja(data, directory, content_filter): +def render_jinja_for_input_file(data, directory, content_filter): loader = FilteredLoader(FileSystemLoader(directory), content_filter) env = Environment(loader=loader) env.globals['environ'] = os.environ.copy() @@ -36,3 +37,15 @@ def render_jinja(data, directory, content_filter): except TemplateError as ex: raise UnableToParse(original=ex) return rendered + + +def render_template(text, **kwargs): + env = Environment() + env.globals["constructor_version"] = __version__ + env.undefined = StrictUndefined + try: + template = env.from_string(text) + return template.render(**kwargs) + except TemplateError as ex: + raise + raise UnableToParse(original=ex) from ex diff --git a/constructor/shar.py b/constructor/shar.py index 768e5a4b1..ffafd3257 100644 --- a/constructor/shar.py +++ b/constructor/shar.py @@ -14,6 +14,7 @@ from os.path import basename, dirname, getsize, isdir, join, relpath from .construct import ns_platform +from .jinja import render_template from .preconda import copy_extra_files from .preconda import files as preconda_files from .preconda import write_files as preconda_write_files @@ -21,11 +22,9 @@ add_condarc, approx_size_kb, filename_dist, - fill_template, get_final_channels, hash_files, parse_virtual_specs, - preprocess, read_ascii_only, shortcuts_flags, ) @@ -56,64 +55,54 @@ def get_header(conda_exec, tarball, info): name = info['name'] has_license = bool(info.get('license_file')) - ppd = ns_platform(info['_platform']) - ppd['keep_pkgs'] = bool(info.get('keep_pkgs', False)) - ppd['batch_mode'] = bool(info.get('batch_mode', False)) - ppd['has_license'] = has_license - if ppd['batch_mode'] and has_license: + variables = ns_platform(info['_platform']) + variables['keep_pkgs'] = bool(info.get('keep_pkgs', False)) + variables['batch_mode'] = bool(info.get('batch_mode', False)) + variables['has_license'] = has_license + if variables['batch_mode'] and has_license: raise Exception( "It is not possible to use both the 'batch_mode' and " "'license_file' options together." ) for key in 'pre_install', 'post_install', 'pre_uninstall': - ppd['has_%s' % key] = bool(key in info) + variables['has_%s' % key] = bool(key in info) if key in info: - ppd['direct_execute_%s' % key] = has_shebang(info[key]) - ppd['initialize_conda'] = info.get('initialize_conda', True) - ppd['initialize_by_default'] = info.get('initialize_by_default', False) - ppd['has_conda'] = info['_has_conda'] - ppd['enable_shortcuts'] = str(info['_enable_shortcuts']).lower() - ppd['check_path_spaces'] = info.get("check_path_spaces", True) + variables['direct_execute_%s' % key] = has_shebang(info[key]) + variables['initialize_conda'] = info.get('initialize_conda', True) + variables['initialize_by_default'] = info.get('initialize_by_default', False) + variables['has_conda'] = info['_has_conda'] + variables['enable_shortcuts'] = str(info['_enable_shortcuts']).lower() + variables['check_path_spaces'] = info.get("check_path_spaces", True) install_lines = list(add_condarc(info)) - # Needs to happen first -- can be templated - replace = { - 'CONSTRUCTOR_VERSION': info['CONSTRUCTOR_VERSION'], - 'NAME': name, - 'name': name.lower(), - 'VERSION': info['version'], - 'PLAT': info['_platform'], - 'DEFAULT_PREFIX': info.get('default_prefix', - '${HOME:-/opt}/%s' % name.lower()), - 'MD5': hash_files([conda_exec, tarball]), - 'FIRST_PAYLOAD_SIZE': str(getsize(conda_exec)), - 'SECOND_PAYLOAD_SIZE': str(getsize(tarball)), - 'INSTALL_COMMANDS': '\n'.join(install_lines), - 'CHANNELS': ','.join(get_final_channels(info)), - 'CONCLUSION_TEXT': info.get("conclusion_text", "installation finished."), - 'pycache': '__pycache__', - 'SHORTCUTS': shortcuts_flags(info), - 'REGISTER_ENVS': str(info.get("register_envs", True)).lower(), - 'TOTAL_INSTALLATION_SIZE_KB': str(approx_size_kb(info, "total")), - 'VIRTUAL_SPECS': shlex.join(info.get("virtual_specs", ())), - 'NO_RCS_ARG': info.get('_ignore_condarcs_arg', ''), - } + variables['installer_name'] = name + variables['installer_version'] = info['version'] + variables['installer_platform'] = info['_platform'] + variables['installer_md5'] = hash_files([conda_exec, tarball]) + variables['default_prefix'] = info.get('default_prefix', '${HOME:-/opt}/%s' % name.lower()) + variables['first_payload_size'] = str(getsize(conda_exec)) + variables['second_payload_size'] = str(getsize(tarball)) + variables['install_commands'] = '\n'.join(install_lines) + variables['channels'] = ','.join(get_final_channels(info)) + variables['conclusion_text'] = info.get("conclusion_text", "installation finished.") + variables['pycache'] = '__pycache__' + variables['shortcuts'] = shortcuts_flags(info) + variables['register_envs'] = str(info.get("register_envs", True)).lower() + variables['total_installation_size_kb'] = str(approx_size_kb(info, "total")) + variables['virtual_specs'] = shlex.join(info.get("virtual_specs", ())) + variables['no_rcs_arg'] = info.get('_ignore_condarcs_arg', '') if has_license: - replace['LICENSE'] = read_ascii_only(info['license_file']) + variables['license'] = read_ascii_only(info['license_file']) virtual_specs = parse_virtual_specs(info) min_osx_version = virtual_specs.get("__osx", {}).get("min") or "" - replace['MIN_OSX_VERSION'] = ppd['min_osx_version'] = min_osx_version + variables['min_osx_version'] = min_osx_version min_glibc_version = virtual_specs.get("__glibc", {}).get("min") or "" - replace['MIN_GLIBC_VERSION'] = ppd['min_glibc_version'] = min_glibc_version + variables['min_glibc_version'] = min_glibc_version - data = read_header_template() - data = preprocess(data, ppd) - custom_variables = info.get('script_env_variables', {}) - data = fill_template(data, replace) + variables['script_env_variables'] = '\n'.join( + [f"export {key}='{value}'" for key, value in info.get('script_env_variables', {}).items()]) - data = data.replace("_SCRIPT_ENV_VARIABLES_=''", '\n'.join( - [f"export {key}='{value}'" for key, value in custom_variables.items()])) - return data + return render_template(read_header_template(), **variables) def create(info, verbose=False): diff --git a/dev/environment.yml b/dev/environment.yml index 50a2b25d1..721e3419c 100644 --- a/dev/environment.yml +++ b/dev/environment.yml @@ -7,3 +7,4 @@ dependencies: - ruamel.yaml >=0.11.14,<0.19 - conda-standalone # >=23.11.0 - pillow >=3.1 # [osx or win] + - jinja2 diff --git a/pyproject.toml b/pyproject.toml index f1bb954a9..6c80b2811 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "conda >=4.6", "ruamel.yaml >=0.11.14,<0.19", "pillow >=3.1 ; platform_system=='Windows' or platform_system=='Darwin'", + "jinja2", ] [project.scripts] diff --git a/tests/test_header.py b/tests/test_header.py index 539d01e0a..4ebd0b875 100644 --- a/tests/test_header.py +++ b/tests/test_header.py @@ -1,4 +1,3 @@ -import itertools import subprocess import sys import tempfile @@ -13,8 +12,9 @@ else: # Tests with OSX_DIR are skipped, but a placeholder is needed for pytest OSX_DIR = "" +from constructor import __version__ +from constructor.jinja import render_template from constructor.shar import read_header_template -from constructor.utils import preprocess @lru_cache @@ -39,90 +39,6 @@ def run_shellcheck(script): return findings, p.returncode -def test_linux_template_processing(): - template = read_header_template() - errors = [] - for ( - osx, - direct_execute_post_install, - direct_execute_pre_install, - batch_mode, - keep_pkgs, - has_conda, - has_license, - initialize_conda, - initialize_by_default, - has_post_install, - has_pre_install, - enable_shortcuts, - check_path_spaces, - arch, - min_glibc_version, - min_osx_version, - ) in itertools.product( - [False, True], - [False, True], - [False, True], - [False, True], - [False, True], - [False, True], - [False, True], - [False, True], - [False, True], - [False, True], - [False, True], - [False, True], - [False, True], - ["x86", "x86_64", " ppc64le", "s390x", "aarch64"], - [None, "2.17"], - [None, "10.13"], - ): - params = { - "has_license": has_license, - "osx": osx, - "batch_mode": batch_mode, - "keep_pkgs": keep_pkgs, - "has_conda": has_conda, - "x86": arch == "x86", - "x86_64": arch == "x86_64", - "ppc64le": arch == "ppc64le", - "s390x": arch == "s390x", - "aarch64": arch == "aarch64", - "linux": not osx, - "has_pre_install": has_pre_install, - "direct_execute_pre_install": direct_execute_pre_install, - "has_post_install": has_post_install, - "direct_execute_post_install": direct_execute_post_install, - "initialize_conda": initialize_conda, - "initialize_by_default": initialize_by_default, - "enable_shortcuts": enable_shortcuts, - "check_path_spaces": check_path_spaces, - "min_glibc_version": min_glibc_version, - "min_osx_version": min_osx_version, - } - processed = preprocess(template, params) - for template_string in ["#if", "#else", "#endif"]: - if template_string in processed: - errors.append( - f"Found '{template_string}' after processing header.sh with '{params}'." - ) - - assert not errors - - -@pytest.mark.skipif(sys.platform != "darwin", reason="Only on MacOS") -@pytest.mark.parametrize("arch", ["x86_64", "arm64"]) -@pytest.mark.parametrize("check_path_spaces", [False, True]) -@pytest.mark.parametrize("script", sorted(Path(OSX_DIR).glob("*.sh"))) -def test_osxpkg_scripts_template_processing(arch, check_path_spaces, script): - with script.open() as f: - data = f.read() - processed = preprocess(data, {"arch": arch, "check_path_spaces": check_path_spaces}) - assert "#if" not in processed - assert "#else" not in processed - assert "#endif" not in processed - - @pytest.mark.skipif(sys.platform != "darwin", reason="Only on MacOS") @pytest.mark.skipif(available_command("shellcheck") is False, reason="requires shellcheck") @pytest.mark.parametrize("arch", ["x86_64", "arm64"]) @@ -133,13 +49,7 @@ def test_osxpkg_scripts_template_processing(arch, check_path_spaces, script): def test_osxpkg_scripts_shellcheck(arch, check_path_spaces, script): with script.open() as f: data = f.read() - processed = preprocess( - data, - { - "arch": arch, - "check_path_spaces": check_path_spaces, - }, - ) + processed = render_template(data, arch=arch, check_path_spaces= check_path_spaces) findings, returncode = run_shellcheck(processed) print(*findings, sep="\n") @@ -183,9 +93,9 @@ def test_template_shellcheck( min_osx_version, ): template = read_header_template() - processed = preprocess( + processed = render_template( template, - { + **{ "has_license": has_license, "osx": osx, "batch_mode": batch_mode, @@ -207,6 +117,24 @@ def test_template_shellcheck( "enable_shortcuts": enable_shortcuts, "min_glibc_version": min_glibc_version, "min_osx_version": min_osx_version, + "first_payload_size": "1024", + "second_payload_size": "512", + "constructor_version": __version__, + "installer_name": "Example", + "installer_version": "1.2.3", + "installer_platform": "linux-64", + "installer_md5": "a0098a2c837f4cf50180cfc0a041b2af", + "script_env_variables": "", # TODO: Fill this in with actual value + "default_prefix": "/opt/Example", + "license": "Some text", + "total_installation_size_kb": "1024", + "virtual_specs": "__glibc>=2.17", + "shortcuts": "", + "register_envs": "1", + "channels": "conda-forge", + "no_rcs_arg": "", + "install_commands": "", # TODO: Fill this in with actual value + "conclusion_text": "Something", }, )