From b11239500e40bfc3f35638aa7de4fc91dcdb7ceb Mon Sep 17 00:00:00 2001 From: Althea Denlinger Date: Tue, 17 Sep 2024 15:33:44 -0700 Subject: [PATCH 1/3] Add flags to `pip install` in quick start --- docs/developers_guide/quick_start.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/developers_guide/quick_start.rst b/docs/developers_guide/quick_start.rst index d48ad1cec..2397e84f3 100644 --- a/docs/developers_guide/quick_start.rst +++ b/docs/developers_guide/quick_start.rst @@ -382,7 +382,7 @@ switch branches. .. code-block:: bash - python -m pip install -e . + python -m pip install --no-deps --no-build-isolation -e . The activation script will do this automatically when you source it in the root directory of your compass branch. The activation script will also @@ -446,7 +446,7 @@ to the current directory. .. code-block:: bash - python -m pip install -e . + python -m pip install --no-deps --no-build-isolation -e . This will be substantially faster than rerunning ``./conda/configure_compass_env.py ...`` but at the risk that dependencies are From 3d67afb794057b07e42939df7e94ba8626f51a3f Mon Sep 17 00:00:00 2001 From: Althea Denlinger Date: Mon, 23 Sep 2024 09:35:42 -0700 Subject: [PATCH 2/3] Update `pip install` flags in conda files --- conda/bootstrap.py | 11 ++++++----- conda/configure_compass_env.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index ff2f74209..0e0d5403e 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -336,7 +336,7 @@ def build_conda_env(env_type, recreate, mpi, conda_mpi, version, f'git submodule update --init alphaBetaLab &&' \ f'cd {source_path}/alphaBetaLab&& ' \ f'conda install -y --file dev-spec.txt && ' \ - f'python -m pip install --no-deps -e .' + f'python -m pip install --no-deps --no-build-isolation -e .' check_call(commands, logger=logger) if recreate or update_jigsaw: @@ -348,7 +348,7 @@ def build_conda_env(env_type, recreate, mpi, conda_mpi, version, f'{activate_env} && ' \ f'cd {source_path} && ' \ f'rm -rf compass.egg-info && ' \ - f'python -m pip install --no-deps -e .' + f'python -m pip install --no-deps --no-build-isolation -e .' check_call(commands, logger=logger) print('Installing pre-commit\n') @@ -402,7 +402,7 @@ def build_jigsaw(activate_env, source_path, env_path, logger): commands = \ f'{activate_env} && ' \ f'cd {source_path}/jigsaw-python && ' \ - f'python -m pip install --no-deps -e . && ' \ + f'python -m pip install --no-deps --no-build-isolation -e . && ' \ f'cp jigsawpy/_bin/* ${{CONDA_PREFIX}}/bin' check_call(commands, logger=logger) @@ -688,7 +688,8 @@ def write_load_compass(template_path, activ_path, conda_base, env_type, mkdir -p conda/logs echo Reinstalling compass package in edit mode... rm -rf compass.egg-info - python -m pip install --no-deps -e . &> conda/logs/install_compass.log + python -m pip install --no-deps --no-build-isolation -e . \\ + &> conda/logs/install_compass.log echo Done. echo fi @@ -1093,7 +1094,7 @@ def main(): # noqa: C901 f'conda activate {conda_env_name} && ' \ f'cd ../build_mache/mache && ' \ f'conda install -y --file spec-file.txt && ' \ - f'python -m pip install --no-deps .' + f'python -m pip install --no-deps --no-build-isolation .' check_call(commands, logger=logger) previous_conda_env = conda_env_name diff --git a/conda/configure_compass_env.py b/conda/configure_compass_env.py index bb2eb51f8..233ac8d0f 100755 --- a/conda/configure_compass_env.py +++ b/conda/configure_compass_env.py @@ -115,7 +115,7 @@ def main(): f'git@github.com:{args.mache_fork}.git mache && ' \ f'cd mache && ' \ f'conda install -y --file spec-file.txt && ' \ - f'python -m pip install --no-deps .' + f'python -m pip install --no-deps --no-build-isolation .' check_call(commands, logger=logger) From d32d67e61ddb29ec759d875a26a3a53ade5d8c43 Mon Sep 17 00:00:00 2001 From: Althea Denlinger Date: Mon, 23 Sep 2024 10:18:47 -0700 Subject: [PATCH 3/3] Satisfy linter --- conda/bootstrap.py | 108 +++++++++++++++++++-------------- conda/configure_compass_env.py | 29 +++++---- 2 files changed, 78 insertions(+), 59 deletions(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index 0e0d5403e..1e42a3b5d 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -312,8 +312,9 @@ def build_conda_env(env_type, recreate, mpi, conda_mpi, version, # conda packages don't like dashes version_conda = version.replace('-', '') packages = f'{packages} "compass={version_conda}={mpi_prefix}_*"' - commands = f'{activate_base} && ' \ - f'conda create -y -n {env_name} {channels} {packages}' + commands = \ + f'{activate_base} && ' \ + f'conda create -y -n {env_name} {channels} {packages}' check_call(commands, logger=logger) else: if env_type == 'dev': @@ -408,7 +409,7 @@ def build_jigsaw(activate_env, source_path, env_path, logger): t1 = time.time() total = int(t1 - t0 + 0.5) - message = f'JIGSAW install took {total:.1f} s.' + message = f'JIGSAW install took {total:.1f} s.' # noqa: 231 if logger is None: print(message) else: @@ -424,11 +425,12 @@ def get_env_vars(machine, compiler, mpilib): env_vars = 'export MPAS_EXTERNAL_LIBS=""\n' if 'intel' in compiler and machine == 'anvil': - env_vars = f'{env_vars}' \ - f'export I_MPI_CC=icc\n' \ - f'export I_MPI_CXX=icpc\n' \ - f'export I_MPI_F77=ifort\n' \ - f'export I_MPI_F90=ifort\n' + env_vars = \ + f'{env_vars}' \ + f'export I_MPI_CC=icc\n' \ + f'export I_MPI_CXX=icpc\n' \ + f'export I_MPI_F77=ifort\n' \ + f'export I_MPI_F90=ifort\n' if machine.startswith('conda'): # we're using parallelio so we don't have ADIOS support @@ -442,9 +444,10 @@ def get_env_vars(machine, compiler, mpilib): f'export MPAS_EXTERNAL_LIBS="${{MPAS_EXTERNAL_LIBS}} -lgomp"\n' if mpilib == 'mvapich': - env_vars = f'{env_vars}' \ - f'export MV2_ENABLE_AFFINITY=0\n' \ - f'export MV2_SHOW_CPU_BINDING=1\n' + env_vars = \ + f'{env_vars}' \ + f'export MV2_ENABLE_AFFINITY=0\n' \ + f'export MV2_SHOW_CPU_BINDING=1\n' if machine.startswith('chicoma') or machine.startswith('pm'): env_vars = \ @@ -525,8 +528,9 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901 f'"scorpio' f'@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc"') # make sure scorpio, not esmf, libraries are linked - lib_path = f'{spack_branch_base}/var/spack/environments/' \ - f'{spack_env}/.spack-env/view/lib' + lib_path = \ + f'{spack_branch_base}/var/spack/environments/' \ + f'{spack_env}/.spack-env/view/lib' scorpio_lib_path = '$(spack find --format "{prefix}" scorpio)' custom_spack = \ f'{custom_spack}' \ @@ -572,8 +576,9 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901 custom_spack=custom_spack, spack_mirror=spack_mirror) # remove ESMC/ESMF include files that interfere with MPAS time keeping - include_path = f'{spack_branch_base}/var/spack/environments/' \ - f'{spack_env}/.spack-env/view/include' + include_path = \ + f'{spack_branch_base}/var/spack/environments/' \ + f'{spack_env}/.spack-env/view/include' for prefix in ['ESMC', 'esmf']: files = glob.glob(os.path.join(include_path, f'{prefix}*')) for filename in files: @@ -587,10 +592,12 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901 include_e3sm_hdf5_netcdf=e3sm_hdf5_netcdf, yaml_template=yaml_template) - spack_view = f'{spack_branch_base}/var/spack/environments/' \ - f'{spack_env}/.spack-env/view' - env_vars = f'{env_vars}' \ - f'export PIO={spack_view}\n' + spack_view = \ + f'{spack_branch_base}/var/spack/environments/' \ + f'{spack_env}/.spack-env/view' + env_vars = \ + f'{env_vars}' \ + f'export PIO={spack_view}\n' if parallelio != 'None': env_vars = \ f'{env_vars}' \ @@ -619,14 +626,16 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901 f'${{ALBANY_LINK_LIBS}} {stdcxx} {mpicxx}"\n' if lapack != 'None': - env_vars = f'{env_vars}' \ - f'export LAPACK={spack_view}\n' \ - f'export USE_LAPACK=true\n' + env_vars = \ + f'{env_vars}' \ + f'export LAPACK={spack_view}\n' \ + f'export USE_LAPACK=true\n' if petsc != 'None': - env_vars = f'{env_vars}' \ - f'export PETSC={spack_view}\n' \ - f'export USE_PETSC=true\n' + env_vars = \ + f'{env_vars}' \ + f'export PETSC={spack_view}\n' \ + f'export USE_PETSC=true\n' return spack_branch_base, spack_script, env_vars @@ -636,7 +645,7 @@ def set_ld_library_path(spack_branch_base, spack_env, logger): f'source {spack_branch_base}/share/spack/setup-env.sh && ' \ f'spack env activate {spack_env} && ' \ f'spack config add modules:prefix_inspections:lib:[LD_LIBRARY_PATH] && ' \ - f'spack config add modules:prefix_inspections:lib64:[LD_LIBRARY_PATH]' # noqa: E501 + f'spack config add modules:prefix_inspections:lib64:[LD_LIBRARY_PATH]' # noqa: E501,E231 check_call(commands, logger=logger) @@ -658,21 +667,26 @@ def write_load_compass(template_path, activ_path, conda_base, env_type, script_filename = '{}/{}{}.sh'.format(activ_path, prefix, activ_suffix) if not env_only: - env_vars = f'{env_vars}\n' \ - f'export USE_PIO2=true' + env_vars = \ + f'{env_vars}\n' \ + f'export USE_PIO2=true' if without_openmp: - env_vars = f'{env_vars}\n' \ - f'export OPENMP=false' + env_vars = \ + f'{env_vars}\n' \ + f'export OPENMP=false' else: - env_vars = f'{env_vars}\n' \ - f'export OPENMP=true' + env_vars = \ + f'{env_vars}\n' \ + f'export OPENMP=true' - env_vars = f'{env_vars}\n' \ - f'export HDF5_USE_FILE_LOCKING=FALSE\n' \ - f'export LOAD_COMPASS_ENV={script_filename}' + env_vars = \ + f'{env_vars}\n' \ + f'export HDF5_USE_FILE_LOCKING=FALSE\n' \ + f'export LOAD_COMPASS_ENV={script_filename}' if machine is not None and not machine.startswith('conda'): - env_vars = f'{env_vars}\n' \ - f'export COMPASS_MACHINE={machine}' + env_vars = \ + f'{env_vars}\n' \ + f'export COMPASS_MACHINE={machine}' filename = f'{template_path}/load_compass.template' with open(filename, 'r') as f: @@ -1090,11 +1104,12 @@ def main(): # noqa: C901 if local_mache: print('Install local mache\n') - commands = f'source {conda_base}/etc/profile.d/conda.sh && ' \ - f'conda activate {conda_env_name} && ' \ - f'cd ../build_mache/mache && ' \ - f'conda install -y --file spec-file.txt && ' \ - f'python -m pip install --no-deps --no-build-isolation .' + commands = \ + f'source {conda_base}/etc/profile.d/conda.sh && ' \ + f'conda activate {conda_env_name} && ' \ + f'cd ../build_mache/mache && ' \ + f'conda install -y --file spec-file.txt && ' \ + f'python -m pip install --no-deps --no-build-isolation .' check_call(commands, logger=logger) previous_conda_env = conda_env_name @@ -1110,10 +1125,11 @@ def main(): # noqa: C901 config, args.update_spack, machine, compiler, mpi, spack_env, spack_base, spack_template_path, env_vars, args.tmpdir, logger) - spack_script = f'echo Loading Spack environment...\n' \ - f'{spack_script}\n' \ - f'echo Done.\n' \ - f'echo\n' + spack_script = \ + f'echo Loading Spack environment...\n' \ + f'{spack_script}\n' \ + f'echo Done.\n' \ + f'echo\n' else: env_vars = \ f'{env_vars}' \ diff --git a/conda/configure_compass_env.py b/conda/configure_compass_env.py index 233ac8d0f..e2db4855b 100755 --- a/conda/configure_compass_env.py +++ b/conda/configure_compass_env.py @@ -30,8 +30,9 @@ def bootstrap(activate_install_env, source_path, local_conda_build): print('Creating the compass conda environment\n') bootstrap_command = f'{source_path}/conda/bootstrap.py' - command = f'{activate_install_env} && ' \ - f'{bootstrap_command} {" ".join(sys.argv[1:])}' + command = \ + f'{activate_install_env} && ' \ + f'{bootstrap_command} {" ".join(sys.argv[1:])}' if local_conda_build is not None: command = f'{command} --local_conda_build {local_conda_build}' check_call(command) @@ -51,8 +52,9 @@ def setup_install_env(env_name, activate_base, use_local, logger, recreate, f'conda create -y -n {env_name} {channels} {packages}' else: print('Updating conda environment for installing compass\n') - commands = f'{activate_base} && ' \ - f'conda install -y -n {env_name} {channels} {packages}' + commands = \ + f'{activate_base} && ' \ + f'conda install -y -n {env_name} {channels} {packages}' check_call(commands, logger=logger) @@ -107,15 +109,16 @@ def main(): if local_mache: print('Clone and install local mache\n') - commands = f'{activate_install_env} && ' \ - f'rm -rf conda/build_mache && ' \ - f'mkdir -p conda/build_mache && ' \ - f'cd conda/build_mache && ' \ - f'git clone -b {args.mache_branch} ' \ - f'git@github.com:{args.mache_fork}.git mache && ' \ - f'cd mache && ' \ - f'conda install -y --file spec-file.txt && ' \ - f'python -m pip install --no-deps --no-build-isolation .' + commands = \ + f'{activate_install_env} && ' \ + f'rm -rf conda/build_mache && ' \ + f'mkdir -p conda/build_mache && ' \ + f'cd conda/build_mache && ' \ + f'git clone -b {args.mache_branch} ' \ + f'git@github.com:{args.mache_fork}.git mache && ' \ + f'cd mache && ' \ + f'conda install -y --file spec-file.txt && ' \ + f'python -m pip install --no-deps --no-build-isolation .' # noqa: E231,E501 check_call(commands, logger=logger)