diff --git a/crates/pixi_config/src/lib.rs b/crates/pixi_config/src/lib.rs index 5157d3de1..a2f881098 100644 --- a/crates/pixi_config/src/lib.rs +++ b/crates/pixi_config/src/lib.rs @@ -20,6 +20,8 @@ use reqwest_middleware::ClientWithMiddleware; use serde::{de::IntoDeserializer, Deserialize, Serialize}; use url::Url; +const EXPERIMENTAL: &str = "experimental"; + pub fn default_channel_config() -> ChannelConfig { ChannelConfig::default_with_root_dir( std::env::current_dir().expect("Could not retrieve the current directory"), @@ -122,9 +124,17 @@ pub struct ConfigCliPrompt { #[arg(long)] change_ps1: Option, } +impl From for Config { + fn from(cli: ConfigCliPrompt) -> Self { + Self { + change_ps1: cli.change_ps1, + ..Default::default() + } + } +} impl ConfigCliPrompt { - pub fn merge_with_config(self, config: Config) -> Config { + pub fn merge_config(self, config: Config) -> Config { let mut config = config; config.change_ps1 = self.change_ps1.or(config.change_ps1); config @@ -168,6 +178,29 @@ impl RepodataConfig { } } +#[derive(Parser, Debug, Default, Clone)] +pub struct ConfigCliActivation { + /// Do not use the environment activation cache. (default: true except in experimental mode) + #[arg(long)] + force_activate: bool, +} + +impl ConfigCliActivation { + pub fn merge_config(self, config: Config) -> Config { + let mut config = config; + config.force_activate = Some(self.force_activate); + config + } +} + +impl From for Config { + fn from(cli: ConfigCliActivation) -> Self { + Self { + force_activate: Some(cli.force_activate), + ..Default::default() + } + } +} #[derive(Clone, Default, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] pub struct RepodataChannelConfig { @@ -273,6 +306,33 @@ impl Default for DetachedEnvironments { } } +#[derive(Default, Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub struct ExperimentalConfig { + /// The option to opt into the environment activation cache feature. + /// This is an experimental feature and may be removed in the future or made default. + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub use_environment_activation_cache: Option, +} + +impl ExperimentalConfig { + pub fn merge(self, other: Self) -> Self { + Self { + use_environment_activation_cache: other + .use_environment_activation_cache + .or(self.use_environment_activation_cache), + } + } + pub fn use_environment_activation_cache(&self) -> bool { + self.use_environment_activation_cache.unwrap_or(false) + } + + pub fn is_default(&self) -> bool { + self.use_environment_activation_cache.is_none() + } +} + impl PyPIConfig { /// Merge the given PyPIConfig into the current one. pub fn merge(self, other: Self) -> Self { @@ -479,6 +539,16 @@ pub struct Config { /// it back to the .pixi folder. #[serde(skip_serializing_if = "Option::is_none")] pub detached_environments: Option, + + /// The option to disable the environment activation cache + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub force_activate: Option, + + /// Experimental features that can be enabled. + #[serde(default)] + #[serde(skip_serializing_if = "ExperimentalConfig::is_default")] + pub experimental: ExperimentalConfig, } impl Default for Config { @@ -495,6 +565,8 @@ impl Default for Config { pypi_config: PyPIConfig::default(), detached_environments: Some(DetachedEnvironments::default()), pinning_strategy: Default::default(), + force_activate: None, + experimental: Default::default(), } } } @@ -735,6 +807,7 @@ impl Config { "pypi-config.index-url", "pypi-config.extra-index-urls", "pypi-config.keyring-provider", + "experimental.use-environment-activation-cache", ] } @@ -763,6 +836,8 @@ impl Config { pypi_config: other.pypi_config.merge(self.pypi_config), detached_environments: other.detached_environments.or(self.detached_environments), pinning_strategy: other.pinning_strategy.or(self.pinning_strategy), + force_activate: other.force_activate, + experimental: other.experimental.merge(self.experimental), } } @@ -820,6 +895,14 @@ impl Config { self.detached_environments.clone().unwrap_or_default() } + pub fn force_activate(&self) -> bool { + self.force_activate.unwrap_or(false) + } + + pub fn experimental_activation_cache_usage(&self) -> bool { + self.experimental.use_environment_activation_cache() + } + /// Modify this config with the given key and value /// /// # Note @@ -943,6 +1026,29 @@ impl Config { _ => return Err(err), } } + key if key.starts_with(EXPERIMENTAL) => { + if key == EXPERIMENTAL { + if let Some(value) = value { + self.experimental = serde_json::de::from_str(&value).into_diagnostic()?; + } else { + self.experimental = ExperimentalConfig::default(); + } + return Ok(()); + } else if !key.starts_with(format!("{EXPERIMENTAL}.").as_str()) { + return Err(err); + } + + let subkey = key + .strip_prefix(format!("{EXPERIMENTAL}.").as_str()) + .unwrap(); + match subkey { + "use-environment-activation-cache" => { + self.experimental.use_environment_activation_cache = + value.map(|v| v.parse()).transpose().into_diagnostic()?; + } + _ => return Err(err), + } + } _ => return Err(err), } @@ -1091,6 +1197,7 @@ UNUSED = "unused" config.authentication_override_file, Some(PathBuf::from("path.json")) ); + assert!(!config.experimental.use_environment_activation_cache()); } #[test] diff --git a/crates/pixi_config/src/snapshots/pixi_config__tests__config_merge.snap b/crates/pixi_config/src/snapshots/pixi_config__tests__config_merge.snap index 8ed30d9b7..91d5b26d6 100644 --- a/crates/pixi_config/src/snapshots/pixi_config__tests__config_merge.snap +++ b/crates/pixi_config/src/snapshots/pixi_config__tests__config_merge.snap @@ -68,4 +68,8 @@ Config { true, ), ), + force_activate: None, + experimental: ExperimentalConfig { + use_environment_activation_cache: None, + }, } diff --git a/crates/pixi_consts/src/consts.rs b/crates/pixi_consts/src/consts.rs index f4010c292..6f0751f78 100644 --- a/crates/pixi_consts/src/consts.rs +++ b/crates/pixi_consts/src/consts.rs @@ -22,6 +22,7 @@ pub const SOLVE_GROUP_ENVIRONMENTS_DIR: &str = "solve-group-envs"; pub const PYPI_DEPENDENCIES: &str = "pypi-dependencies"; pub const DEPENDENCIES: &str = "dependencies"; pub const TASK_CACHE_DIR: &str = "task-cache-v0"; +pub const ACTIVATION_ENV_CACHE_DIR: &str = "activation-env-v0"; pub const PIXI_UV_INSTALLER: &str = "uv-pixi"; pub const CONDA_PACKAGE_CACHE_DIR: &str = rattler_cache::PACKAGE_CACHE_DIR; pub const CONDA_REPODATA_CACHE_DIR: &str = rattler_cache::REPODATA_CACHE_DIR; diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ba4c3d855..a8a78e747 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -250,6 +250,7 @@ You cannot run `pixi run source setup.bash` as `source` is not available in the - `--locked`: only install if the `pixi.lock` is up-to-date with the [manifest file](project_configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. - `--environment (-e)`: The environment to run the task in, if none are provided the default environment will be used or a selector will be given to select the right environment. - `--clean-env`: Run the task in a clean environment, this will remove all environment variables of the shell environment except for the ones pixi sets. THIS DOESN't WORK ON `Windows`. +- `--force-activate`: (default, except in _experimental_ mode) Force the activation of the environment, even if the environment is already activated. - `--revalidate`: Revalidate the full environment, instead of checking the lock file hash. [more info](../features/environment.md#environment-installation-metadata) ```shell @@ -509,6 +510,8 @@ List project's packages. Highlighted packages are explicit dependencies. - `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](project_configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). - `--locked`: Only install if the `pixi.lock` is up-to-date with the [manifest file](project_configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. - `--no-install`: Don't install the environment for pypi solving, only update the lock-file if it can solve without installing. (Implied by `--frozen` and `--locked`) +- `--no-lockfile-update`: Don't update the lock-file, implies the `--no-install` flag. +- `--no-progress`: Hide all progress bars, always turned on if stderr is not a terminal [env: PIXI_NO_PROGRESS=] ```shell pixi list @@ -570,6 +573,8 @@ The package tree can also be inverted (`-i`), to see which packages require a sp - `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](project_configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). - `--locked`: Only install if the `pixi.lock` is up-to-date with the [manifest file](project_configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. - `--no-install`: Don't install the environment for pypi solving, only update the lock-file if it can solve without installing. (Implied by `--frozen` and `--locked`) +- `--no-lockfile-update`: Don't update the lock-file, implies the `--no-install` flag. +- `--no-progress`: Hide all progress bars, always turned on if stderr is not a terminal [env: PIXI_NO_PROGRESS=] ```shell pixi tree @@ -680,7 +685,11 @@ To exit the pixi shell, simply run `exit`. - `--manifest-path `: the path to [manifest file](project_configuration.md), by default it searches for one in the parent directories. - `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](project_configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). - `--locked`: only install if the `pixi.lock` is up-to-date with the [manifest file](project_configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. +- `--no-install`: Don't install the environment, only activate the environment. +- `--no-lockfile-update`: Don't update the lock-file, implies the `--no-install` flag. - `--environment (-e)`: The environment to activate the shell in, if none are provided the default environment will be used or a selector will be given to select the right environment. +- `--no-progress`: Hide all progress bars, always turned on if stderr is not a terminal [env: PIXI_NO_PROGRESS=] +- `--force-activate`: (default, except in _experimental_ mode) Force the activation of the environment, even if the environment is already activated. - `--revalidate`: Revalidate the full environment, instead of checking lock file hash. [more info](../features/environment.md#environment-installation-metadata) ```shell @@ -710,6 +719,7 @@ This command prints the activation script of an environment. - `--environment (-e)`: The environment to activate, if none are provided the default environment will be used or a selector will be given to select the right environment. - `--json`: Print all environment variables that are exported by running the activation script as JSON. When specifying this option, `--shell` is ignored. +- `--force-activate`: (default, except in _experimental_ mode) Force the activation of the environment, even if the environment is already activated. - `--revalidate`: Revalidate the full environment, instead of checking lock file hash. [more info](../features/environment.md#environment-installation-metadata) ```shell @@ -876,6 +886,7 @@ Use this command to manage the configuration. - `--system (-s)`: Specify management scope to system configuration. - `--global (-g)`: Specify management scope to global configuration. - `--local (-l)`: Specify management scope to local configuration. +- `--manifest-path `: the path to [manifest file](project_configuration.md), by default it searches for one in the parent directories. Checkout the [pixi configuration](./pixi_configuration.md) for more information about the locations. diff --git a/docs/reference/pixi_configuration.md b/docs/reference/pixi_configuration.md index b6901b9f5..7c73a536a 100644 --- a/docs/reference/pixi_configuration.md +++ b/docs/reference/pixi_configuration.md @@ -44,7 +44,7 @@ The configuration is loaded in the following order: To find the locations where `pixi` looks for configuration files, run `pixi` with `-vv`. -## Reference +## Configuration options ??? info "Casing In Configuration" In versions of pixi `0.20.1` and older the global configuration used snake_case @@ -222,6 +222,55 @@ keyring-provider = "subprocess" Unlike pip, these settings, with the exception of `keyring-provider` will only modify the `pixi.toml`/`pyproject.toml` file and are not globally interpreted when not present in the manifest. This is because we want to keep the manifest file as complete and reproducible as possible. +## Experimental +This allows the user to set specific experimental features that are not yet stable. + +Please write a GitHub issue and add the flag `experimental` to the issue if you find issues with the feature you activated. + + +### Caching environment activations +Turn this feature on from configuration with the following command: +```shell +# For all your projects +pixi config set experimental.use-environment-activation-cache true --global + +# For a specific project +pixi config set experimental.use-environment-activation-cache true --local +``` + +This will cache the environment activation in the `.pixi/activation-env-v0` folder in the project root. +It will create a json file for each environment that is activated, and it will be used to activate the environment in the future. +```bash +> tree .pixi/activation-env-v0/ +.pixi/activation-env-v0/ +├── activation_default.json +└── activation_lint.json + +> cat .pixi/activation-env-v0/activation_lint.json +{"hash":"8d8344e0751d377a","environment_variables":{}} +``` + +- The `hash` is a hash of the data on that environment in the `pixi.lock`, plus some important information on the environment activation. + Like `[activation.scripts]` and `[activation.env]` from the manifest file. +- The `environment_variables` are the environment variables that are set when activating the environment. + +You can ignore the cache by running: +``` +pixi run/shell/shell-hook --force-activate +``` + +Set the configuration with: +```toml title="config.toml" +[experimental] +# Enable the use of the environment activation cache +use-environment-activation-cache = true +``` + +!!! note "Why is this experimental?" +This feature is experimental because the cache invalidation is very tricky, +and we don't want to disturb users that are not affected by activation times. + + ## Mirror configuration You can configure mirrors for conda channels. We expect that mirrors are exact diff --git a/src/activation.rs b/src/activation.rs index 121372226..451c9aab1 100644 --- a/src/activation.rs +++ b/src/activation.rs @@ -1,9 +1,13 @@ +use crate::{project::Environment, Project}; +use crate::{project::HasProjectRef, task::EnvironmentHash}; +use fs_err::tokio as tokio_fs; use indexmap::IndexMap; -use std::collections::HashMap; - use itertools::Itertools; use miette::IntoDiagnostic; +use pixi_manifest::EnvironmentName; +use pixi_manifest::FeaturesExt; use rattler_conda_types::Platform; +use rattler_lock::LockFile; use rattler_shell::{ activation::{ ActivationError, ActivationError::FailedToRunActivationScript, ActivationVariables, @@ -11,11 +15,9 @@ use rattler_shell::{ }, shell::ShellEnum, }; - -use crate::project::HasProjectRef; -use crate::{project::Environment, Project}; -use pixi_manifest::EnvironmentName; -use pixi_manifest::FeaturesExt; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::PathBuf; // Setting a base prefix for the pixi package const PROJECT_PREFIX: &str = "PIXI_PROJECT_"; @@ -30,6 +32,14 @@ pub enum CurrentEnvVarBehavior { Exclude, } +#[derive(Serialize, Deserialize)] +struct ActivationCache { + /// The hash of the environment which produced the activation's environment variables. + hash: EnvironmentHash, + /// The environment variables set by the activation. + environment_variables: HashMap, +} + impl Project { /// Returns environment variables and their values that should be injected when running a command. pub(crate) fn get_metadata_env(&self) -> HashMap { @@ -142,11 +152,102 @@ pub(crate) fn get_activator<'p>( Ok(activator) } +/// Get the environment variables from the shell environment. +/// This method retrieves the specified environment variables from the shell and returns them as a HashMap. +/// If the variable is not set, its value will be `None`. +fn get_environment_variable_from_shell_environment( + names: Vec<&str>, +) -> HashMap> { + names + .into_iter() + .map(|name| { + let value = std::env::var(name).ok(); + (name.to_string(), value) + }) + .collect() +} + +/// Try to get the activation cache from the cache file. +/// If it can get the cache, it will validate it with the lock file and the current environment. +/// If the cache is valid, it will return the environment variables from the cache. +/// +/// Without a lock file it will not use the cache, as it indicates the cache is not interesting +async fn try_get_valid_activation_cache( + lock_file: &LockFile, + environment: &Environment<'_>, + cache_file: PathBuf, +) -> Option> { + // Find cache file + if !cache_file.exists() { + return None; + } + // Read the cache file + let cache_content = match tokio_fs::read_to_string(&cache_file).await { + Ok(content) => content, + Err(e) => { + tracing::debug!("Failed to read activation cache file, reactivating. Error: {e}"); + return None; + } + }; + // Parse the cache file + let cache: ActivationCache = match serde_json::from_str(&cache_content) { + Ok(parsed) => parsed, + Err(e) => { + tracing::debug!("Failed to parse cache file, reactivating. Error: {e}"); + return None; + } + }; + + // Get the current environment variables + let current_input_env_vars = get_environment_variable_from_shell_environment( + cache + .environment_variables + .keys() + .map(String::as_str) + .collect(), + ); + + // Hash the current state + let hash = EnvironmentHash::from_environment(environment, ¤t_input_env_vars, lock_file); + + // Check if the hash matches + if cache.hash == hash { + Some(cache.environment_variables) + } else { + None + } +} + /// Runs and caches the activation script. pub async fn run_activation( environment: &Environment<'_>, env_var_behavior: &CurrentEnvVarBehavior, + lock_file: Option<&LockFile>, + force_activate: bool, + experimental: bool, ) -> miette::Result> { + // If the user requested to use the cache and the lockfile is provided, we can try to use the cache. + if !force_activate && experimental { + let cache_file = environment + .project() + .activation_env_cache_folder() + .join(environment.activation_cache_name()); + if let Some(lock_file) = lock_file { + if let Some(env_vars) = + try_get_valid_activation_cache(lock_file, environment, cache_file).await + { + tracing::debug!("Using activation cache for {:?}", environment.name()); + return Ok(env_vars); + } + } else { + tracing::debug!( + "No lock file provided for activation, not using activation cache for {:?}", + environment.name() + ); + } + } + tracing::debug!("Running activation script for {:?}", environment.name()); + let activator = get_activator(environment, ShellEnum::default()).map_err(|e| { miette::miette!(format!( "failed to create activator for {:?}\n{}", @@ -210,6 +311,39 @@ pub async fn run_activation( } }; + // If the lock file is provided, and we can compute the environment hash, let's rewrite the + // cache file. + if experimental { + if let Some(lock_file) = lock_file { + // Get the current environment variables from the shell to be part of the hash + let current_input_env_vars = get_environment_variable_from_shell_environment( + activator_result.keys().map(String::as_str).collect(), + ); + let cache_file = environment.activation_cache_file_path(); + let cache = ActivationCache { + hash: EnvironmentHash::from_environment( + environment, + ¤t_input_env_vars, + lock_file, + ), + environment_variables: activator_result.clone(), + }; + let cache = serde_json::to_string(&cache).into_diagnostic()?; + + tokio_fs::create_dir_all(environment.project().activation_env_cache_folder()) + .await + .into_diagnostic()?; + tokio_fs::write(&cache_file, cache) + .await + .into_diagnostic()?; + tracing::debug!( + "Wrote activation cache for {} to {}", + environment.name(), + cache_file.display() + ); + } + } + Ok(activator_result) } @@ -286,11 +420,23 @@ pub(crate) fn get_clean_environment_variables() -> HashMap { /// the environment and stores the environment variables it added, finally it adds environment /// variables from the project and based on the clean_env setting it will also add in the current /// shell environment variables. +/// +/// If a lock file is given this will also create/use an activated environment cache when possible. pub(crate) async fn initialize_env_variables( environment: &Environment<'_>, env_var_behavior: CurrentEnvVarBehavior, + lock_file: Option<&LockFile>, + force_activate: bool, + experimental: bool, ) -> miette::Result> { - let activation_env = run_activation(environment, &env_var_behavior).await?; + let activation_env = run_activation( + environment, + &env_var_behavior, + lock_file, + force_activate, + experimental, + ) + .await?; // Get environment variables from the currently activated shell. let current_shell_env_vars = match env_var_behavior { @@ -314,9 +460,9 @@ pub(crate) async fn initialize_env_variables( #[cfg(test)] mod tests { - use std::path::Path; - use super::*; + use std::path::Path; + use std::str::FromStr; #[test] fn test_metadata_env() { @@ -420,4 +566,284 @@ mod tests { std::env::var("USER").as_ref().unwrap() ); } + + /// Test that the activation cache is created and used correctly based on the lockfile. + /// + /// This test will validate the cache usages by running the activation script and checking if the cache is created. + /// - It will then modify the cache and check if the cache is used. + /// - It will then modify the lock file and check if the cache is not used and recreated. + /// - It will then modify the cache again and check if the cache is used again. + #[tokio::test] + async fn test_run_activation_cache_based_on_lockfile() { + let temp_dir = tempfile::tempdir().unwrap(); + let project = r#" + [project] + name = "pixi" + channels = [] + platforms = [] + + [activation.env] + TEST = "ACTIVATION123" + "#; + let project = + Project::from_str(temp_dir.path().join("pixi.toml").as_path(), project).unwrap(); + let default_env = project.default_environment(); + + // Don't create cache, by not giving it a lockfile + let env = run_activation( + &default_env, + &CurrentEnvVarBehavior::Include, + None, + false, + true, + ) + .await + .unwrap(); + assert!(!project.activation_env_cache_folder().exists()); + assert!(env.contains_key("CONDA_PREFIX")); + + // Create cache + let lock_file = LockFile::default(); + let _env = run_activation( + &default_env, + &CurrentEnvVarBehavior::Include, + Some(&lock_file), + false, + true, + ) + .await + .unwrap(); + assert!(project.activation_env_cache_folder().exists()); + assert!(project + .activation_env_cache_folder() + .join(project.default_environment().activation_cache_name()) + .exists()); + + // Verify that the cache is used, by overwriting the cache and checking if that persisted + let cache_file = project.default_environment().activation_cache_file_path(); + let contents = tokio_fs::read_to_string(&cache_file).await.unwrap(); + let modified = contents.replace("ACTIVATION123", "ACTIVATION456"); + tokio_fs::write(&cache_file, modified).await.unwrap(); + + let env = run_activation( + &default_env, + &CurrentEnvVarBehavior::Include, + Some(&lock_file), + false, + true, + ) + .await + .unwrap(); + assert_eq!(env.get("TEST").unwrap(), "ACTIVATION456"); + + // Verify that the cache is not used when the hash is different + let mock_lock = r#" +version: 5 +environments: + default: + channels: + - url: https://fast.prefix.dev/conda-forge/ + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_r-mutex-1.0.1-anacondar_1.tar.bz2 + osx-64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_r-mutex-1.0.1-anacondar_1.tar.bz2 + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_r-mutex-1.0.1-anacondar_1.tar.bz2 + win-64: + - conda: https://conda.anaconda.org/conda-forge/noarch/_r-mutex-1.0.1-anacondar_1.tar.bz2 +packages: +- kind: conda + name: _r-mutex + version: 1.0.1 + build: anacondar_1 + build_number: 1 + subdir: noarch + noarch: generic + url: https://conda.anaconda.org/conda-forge/noarch/_r-mutex-1.0.1-anacondar_1.tar.bz2 + sha256: e58f9eeb416b92b550e824bcb1b9fb1958dee69abfe3089dfd1a9173e3a0528a + md5: 19f9db5f4f1b7f5ef5f6d67207f25f38 + license: BSD + size: 3566 + timestamp: 1562343890778 +- kind: conda + name: _r-mutex + version: 1.0.1 + build: anacondar_1 + build_number: 1 + subdir: noarch + noarch: generic + url: https://conda.anaconda.org/conda-forge/noarch/_r-mutex-1.0.1-anacondar_1.tar.bz2 + sha256: e58f9eeb416b92b550e824bcb1b9fb1958dee69abfe3089dfd1a9173e3a0528a + md5: 19f9db5f4f1b7f5ef5f6d67207f25f38 + license: BSD + size: 3566 + timestamp: 1562343890778 +"#; + let lock_file = LockFile::from_str(mock_lock).unwrap(); + let env = run_activation( + &default_env, + &CurrentEnvVarBehavior::Include, + Some(&lock_file), + false, + true, + ) + .await + .unwrap(); + assert_eq!(env.get("TEST").unwrap(), "ACTIVATION123"); + + // Verify that the cache is used again after the hash is the same + let contents = tokio_fs::read_to_string(&cache_file).await.unwrap(); + let modified = contents.replace("ACTIVATION123", "ACTIVATION456"); + tokio_fs::write(&cache_file, modified).await.unwrap(); + + let env = run_activation( + &default_env, + &CurrentEnvVarBehavior::Include, + Some(&lock_file), + false, + true, + ); + assert_eq!(env.await.unwrap().get("TEST").unwrap(), "ACTIVATION456"); + } + + #[tokio::test] + async fn test_run_activation_cache_based_on_activation_env() { + let temp_dir = tempfile::tempdir().unwrap(); + let project = r#" + [project] + name = "pixi" + channels = [] + platforms = [] + + [activation.env] + TEST = "ACTIVATION123" + "#; + let project = + Project::from_str(temp_dir.path().join("pixi.toml").as_path(), project).unwrap(); + let default_env = project.default_environment(); + let env = run_activation( + &default_env, + &CurrentEnvVarBehavior::Include, + Some(&LockFile::default()), + false, + true, + ) + .await + .unwrap(); + assert_eq!(env.get("TEST").unwrap(), "ACTIVATION123",); + + // Modify the variable in cache + let cache_file = project.default_environment().activation_cache_file_path(); + let contents = tokio_fs::read_to_string(&cache_file).await.unwrap(); + let modified = contents.replace("ACTIVATION123", "ACTIVATION456"); + tokio_fs::write(&cache_file, modified).await.unwrap(); + + // Check that the cache is invalidated when the activation.env changes. + let project = r#" + [project] + name = "pixi" + channels = [] + platforms = [] + + [activation.env] + TEST = "ACTIVATION123" + TEST2 = "ACTIVATION1234" + "#; + let project = + Project::from_str(temp_dir.path().join("pixi.toml").as_path(), project).unwrap(); + let default_env = project.default_environment(); + let env = run_activation( + &default_env, + &CurrentEnvVarBehavior::Include, + Some(&LockFile::default()), + false, + true, + ) + .await + .unwrap(); + assert_eq!( + env.get("TEST").unwrap(), + "ACTIVATION123", + "The old variable should be reset" + ); + assert_eq!( + env.get("TEST2").unwrap(), + "ACTIVATION1234", + "The new variable should be set" + ); + } + + // This test works, most of the times.., so this is a good test to run locally. + // But it is to flaky for CI unfortunately! + // #[tokio::test] + // async fn test_run_activation_based_on_existing_env(){ + // let temp_dir = tempfile::tempdir().unwrap(); + // let project = r#" + // [project] + // name = "pixi" + // channels = [] + // platforms = ["linux-64", "osx-64", "win-64", "osx-arm64"] + // + // [target.unix.activation.env] + // TEST_ENV_VAR = "${TEST_ENV_VAR}_and_some_more" + // + // [target.win.activation.env] + // TEST_ENV_VAR = "%TEST_ENV_VAR%_and_some_more" + // "#; + // let project = + // Project::from_str(temp_dir.path().join("pixi.toml").as_path(), project).unwrap(); + // let default_env = project.default_environment(); + // + // // Set the environment variable + // std::env::set_var("TEST_ENV_VAR", "test_value"); + // + // // Run the activation script + // let env = run_activation( + // &default_env, + // &CurrentEnvVarBehavior::Include, + // Some(&LockFile::default()), + // false, + // true, + // ).await.unwrap(); + // + // // Check that the environment variable is set correctly + // assert_eq!(env.get("TEST_ENV_VAR").unwrap(), "test_value_and_some_more"); + // + // // Modify the environment variable + // let cache_file = project.default_environment().activation_cache_file_path(); + // let contents = tokio_fs::read_to_string(&cache_file).await.unwrap(); + // let modified = contents.replace("test_value_and_some_more", "modified_cache"); + // tokio_fs::write(&cache_file, modified).await.unwrap(); + // + // // Run the activation script + // let env = run_activation( + // &default_env, + // &CurrentEnvVarBehavior::Include, + // Some(&LockFile::default()), + // false, + // true, + // ).await.unwrap(); + // + // // Check that the environment variable is taken from cache + // assert_eq!(env.get("TEST_ENV_VAR").unwrap(), "modified_cache"); + // + // // Reset the environment variable + // std::env::set_var("TEST_ENV_VAR", "different_test_value"); + // + // // Run the activation script + // let env = run_activation( + // &default_env, + // &CurrentEnvVarBehavior::Include, + // Some(&LockFile::default()), + // false, + // true, + // ).await.unwrap(); + // + // // Check that the environment variable reset, thus the cache was invalidated. + // assert_eq!(env.get("TEST_ENV_VAR").unwrap(), "different_test_value_and_some_more"); + // + // // Unset the environment variable + // std::env::remove_var("TEST_ENV_VAR"); + // } } diff --git a/src/cli/clean.rs b/src/cli/clean.rs index f522167de..483c1849d 100644 --- a/src/cli/clean.rs +++ b/src/cli/clean.rs @@ -8,6 +8,8 @@ use std::time::Duration; use crate::cli::cli_config::ProjectConfig; use clap::Parser; +use fancy_display::FancyDisplay; +use fs_err::tokio as tokio_fs; use indicatif::ProgressBar; use miette::IntoDiagnostic; use pixi_progress::{global_multi_progress, long_running_progress_style}; @@ -34,6 +36,10 @@ pub struct Args { /// The environment directory to remove. #[arg(long, short, conflicts_with = "command")] pub environment: Option, + + /// Only remove the activation cache + #[arg(long)] + pub activation_cache: bool, } /// Clean the cache of your system which are touched by pixi. @@ -93,8 +99,17 @@ pub async fn execute(args: Args) -> miette::Result<()> { .transpose()?; if let Some(explicit_env) = explicit_environment { - remove_folder_with_progress(explicit_env.dir(), true).await?; - tracing::info!("Skipping removal of task cache and solve group environments for explicit environment '{:?}'", explicit_env.name()); + if args.activation_cache { + remove_file(explicit_env.activation_cache_file_path(), false).await?; + tracing::info!( + "Only removing activation cache for explicit environment '{}'", + explicit_env.name().fancy_display() + ); + } else { + remove_folder_with_progress(explicit_env.dir(), true).await?; + remove_file(explicit_env.activation_cache_file_path(), false).await?; + tracing::info!("Skipping removal of task cache and solve group environments for explicit environment '{}'", explicit_env.name().fancy_display()); + } } else { // Remove all pixi related work from the project. if !project.environments_dir().starts_with(project.pixi_dir()) @@ -106,11 +121,11 @@ pub async fn execute(args: Args) -> miette::Result<()> { false, ) .await?; - remove_folder_with_progress(project.task_cache_folder(), false).await?; } remove_folder_with_progress(project.environments_dir(), true).await?; remove_folder_with_progress(project.solve_group_environments_dir(), false).await?; remove_folder_with_progress(project.task_cache_folder(), false).await?; + remove_folder_with_progress(project.activation_env_cache_folder(), false).await?; } Project::warn_on_discovered_from_env(args.project_config.manifest_path.as_deref()) @@ -182,7 +197,7 @@ async fn remove_folder_with_progress( )); // Ignore errors - let result = tokio::fs::remove_dir_all(&folder).await; + let result = tokio_fs::remove_dir_all(&folder).await; if let Err(e) = result { tracing::info!("Failed to remove folder {:?}: {}", folder, e); } @@ -194,3 +209,24 @@ async fn remove_folder_with_progress( )); Ok(()) } + +async fn remove_file(file: PathBuf, warning_non_existent: bool) -> miette::Result<()> { + if !file.exists() { + if warning_non_existent { + eprintln!( + "{}", + console::style(format!("File {:?} was not found.", &file)).yellow() + ); + } + return Ok(()); + } + + // Ignore errors + let result = tokio_fs::remove_file(&file).await; + if let Err(e) = result { + tracing::info!("Failed to remove file {:?}: {}", file, e); + } else { + eprintln!("{} {}", console::style("removed").green(), file.display()); + } + Ok(()) +} diff --git a/src/cli/cli_config.rs b/src/cli/cli_config.rs index 9fcd00eb4..3eddf04f0 100644 --- a/src/cli/cli_config.rs +++ b/src/cli/cli_config.rs @@ -17,7 +17,7 @@ use std::collections::HashMap; use std::path::PathBuf; /// Project configuration -#[derive(Parser, Debug, Default)] +#[derive(Parser, Debug, Default, Clone)] pub struct ProjectConfig { /// The path to `pixi.toml` or `pyproject.toml` #[arg(long)] diff --git a/src/cli/config.rs b/src/cli/config.rs index 7735d14f7..618fd1b8c 100644 --- a/src/cli/config.rs +++ b/src/cli/config.rs @@ -1,13 +1,12 @@ -use std::{path::PathBuf, str::FromStr}; - +use crate::cli::cli_config::ProjectConfig; +use crate::Project; use clap::Parser; use miette::{IntoDiagnostic, WrapErr}; use pixi_config; use pixi_config::Config; use pixi_consts::consts; use rattler_conda_types::NamedChannelOrUrl; - -use crate::project; +use std::{path::PathBuf, str::FromStr}; #[derive(Parser, Debug)] enum Subcommand { @@ -60,6 +59,9 @@ struct CommonArgs { /// Operation on system configuration #[arg(long, short, conflicts_with_all = &["local", "global"])] system: bool, + + #[clap(flatten)] + pub project_config: ProjectConfig, } #[derive(Parser, Debug, Clone)] @@ -195,25 +197,16 @@ pub async fn execute(args: Args) -> miette::Result<()> { } fn determine_project_root(common_args: &CommonArgs) -> miette::Result> { - match project::find_project_manifest(std::env::current_dir().into_diagnostic()?) { - None => { + match Project::load_or_else_discover(common_args.project_config.manifest_path.as_deref()) { + Err(e) => { if common_args.local { return Err(miette::miette!( - "--local flag can only be used inside a pixi project" + "--local flag can only be used inside a pixi project: '{e}'", )); } Ok(None) } - Some(manifest_file) => { - let full_path = dunce::canonicalize(&manifest_file).into_diagnostic()?; - let root = full_path - .parent() - .ok_or_else(|| { - miette::miette!("can not find parent of {}", manifest_file.display()) - })? - .to_path_buf(); - Ok(Some(root)) - } + Ok(project) => Ok(Some(project.root().to_path_buf())), } } diff --git a/src/cli/install.rs b/src/cli/install.rs index 62c1aa020..b3299d424 100644 --- a/src/cli/install.rs +++ b/src/cli/install.rs @@ -1,5 +1,5 @@ use crate::cli::cli_config::ProjectConfig; -use crate::environment::update_prefix; +use crate::environment::get_update_lock_file_and_prefix; use crate::lock_file::UpdateMode; use crate::Project; use clap::Parser; @@ -53,7 +53,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { let environment = project.environment_from_name_or_env_var(Some(env))?; // Update the prefix by installing all packages - update_prefix( + get_update_lock_file_and_prefix( &environment, args.lock_file_usage.into(), false, diff --git a/src/cli/project/channel/add.rs b/src/cli/project/channel/add.rs index ceb18fbaf..e27cff66a 100644 --- a/src/cli/project/channel/add.rs +++ b/src/cli/project/channel/add.rs @@ -1,5 +1,5 @@ use crate::{ - environment::{update_prefix, LockFileUsage}, + environment::{get_update_lock_file_and_prefix, LockFileUsage}, lock_file::UpdateMode, Project, }; @@ -15,7 +15,7 @@ pub async fn execute(mut project: Project, args: AddRemoveArgs) -> miette::Resul )?; // TODO: Update all environments touched by the features defined. - update_prefix( + get_update_lock_file_and_prefix( &project.default_environment(), LockFileUsage::Update, args.no_install, diff --git a/src/cli/project/channel/remove.rs b/src/cli/project/channel/remove.rs index b78306981..901a98d90 100644 --- a/src/cli/project/channel/remove.rs +++ b/src/cli/project/channel/remove.rs @@ -1,6 +1,6 @@ use crate::lock_file::UpdateMode; use crate::{ - environment::{update_prefix, LockFileUsage}, + environment::{get_update_lock_file_and_prefix, LockFileUsage}, Project, }; @@ -13,7 +13,7 @@ pub async fn execute(mut project: Project, args: AddRemoveArgs) -> miette::Resul .remove_channels(args.prioritized_channels(), &args.feature_name())?; // Try to update the lock-file without the removed channels - update_prefix( + get_update_lock_file_and_prefix( &project.default_environment(), LockFileUsage::Update, args.no_install, diff --git a/src/cli/project/platform/add.rs b/src/cli/project/platform/add.rs index 79382e2eb..71ffaa892 100644 --- a/src/cli/project/platform/add.rs +++ b/src/cli/project/platform/add.rs @@ -1,7 +1,7 @@ use std::str::FromStr; use crate::{ - environment::{update_prefix, LockFileUsage}, + environment::{get_update_lock_file_and_prefix, LockFileUsage}, lock_file::UpdateMode, Project, }; @@ -45,7 +45,7 @@ pub async fn execute(mut project: Project, args: Args) -> miette::Result<()> { .add_platforms(platforms.iter(), &feature_name)?; // Try to update the lock-file with the new channels - update_prefix( + get_update_lock_file_and_prefix( &project.default_environment(), LockFileUsage::Update, args.no_install, diff --git a/src/cli/project/platform/remove.rs b/src/cli/project/platform/remove.rs index 0cc22137f..a5d24c9ea 100644 --- a/src/cli/project/platform/remove.rs +++ b/src/cli/project/platform/remove.rs @@ -2,7 +2,7 @@ use std::str::FromStr; use crate::lock_file::UpdateMode; use crate::{ - environment::{update_prefix, LockFileUsage}, + environment::{get_update_lock_file_and_prefix, LockFileUsage}, Project, }; use clap::Parser; @@ -44,7 +44,7 @@ pub async fn execute(mut project: Project, args: Args) -> miette::Result<()> { .manifest .remove_platforms(platforms.clone(), &feature_name)?; - update_prefix( + get_update_lock_file_and_prefix( &project.default_environment(), LockFileUsage::Update, args.no_install, diff --git a/src/cli/remove.rs b/src/cli/remove.rs index 7dde9f224..189057e3b 100644 --- a/src/cli/remove.rs +++ b/src/cli/remove.rs @@ -1,7 +1,7 @@ use clap::Parser; use miette::Context; -use crate::environment::update_prefix; +use crate::environment::get_update_lock_file_and_prefix; use crate::DependencyType; use crate::Project; @@ -79,7 +79,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { // TODO: update all environments touched by this feature defined. // updating prefix after removing from toml if !prefix_update_config.no_lockfile_update { - update_prefix( + get_update_lock_file_and_prefix( &project.default_environment(), prefix_update_config.lock_file_usage(), prefix_update_config.no_install, diff --git a/src/cli/run.rs b/src/cli/run.rs index 725eea267..5bfee8eb7 100644 --- a/src/cli/run.rs +++ b/src/cli/run.rs @@ -19,6 +19,7 @@ use crate::task::{ }; use crate::Project; use fancy_display::FancyDisplay; +use pixi_config::ConfigCliActivation; use pixi_manifest::TaskName; use thiserror::Error; use tracing::Level; @@ -36,6 +37,9 @@ pub struct Args { #[clap(flatten)] pub prefix_update_config: PrefixUpdateConfig, + #[clap(flatten)] + pub activation_config: ConfigCliActivation, + /// The environment to run the task in. #[arg(long, short)] pub environment: Option, @@ -50,9 +54,13 @@ pub struct Args { /// CLI entry point for `pixi run` /// When running the sigints are ignored and child can react to them. As it pleases. pub async fn execute(args: Args) -> miette::Result<()> { + let cli_config = args + .activation_config + .merge_config(args.prefix_update_config.config.clone().into()); + // Load the project let project = Project::load_or_else_discover(args.project_config.manifest_path.as_deref())? - .with_cli_config(args.prefix_update_config.config.clone()); + .with_cli_config(cli_config); // Extract the passed in environment name. let environment = project.environment_from_name_or_env_var(args.environment.clone())?; @@ -147,7 +155,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { // check task cache let task_cache = match executable_task - .can_skip(&lock_file) + .can_skip(&lock_file.lock_file) .await .into_diagnostic()? { @@ -178,6 +186,9 @@ pub async fn execute(args: Args) -> miette::Result<()> { let command_env = get_task_env( &executable_task.run_environment, args.clean_env || executable_task.task().clean_env(), + Some(&lock_file.lock_file), + project.config().force_activate(), + project.config().experimental_activation_cache_usage(), ) .await?; entry.insert(command_env) diff --git a/src/cli/shell.rs b/src/cli/shell.rs index 1b8d7cbb0..7f8f22496 100644 --- a/src/cli/shell.rs +++ b/src/cli/shell.rs @@ -11,11 +11,11 @@ use rattler_shell::{ use crate::cli::cli_config::{PrefixUpdateConfig, ProjectConfig}; use crate::lock_file::UpdateMode; use crate::{ - activation::CurrentEnvVarBehavior, environment::update_prefix, + activation::CurrentEnvVarBehavior, environment::get_update_lock_file_and_prefix, project::virtual_packages::verify_current_platform_has_required_virtual_packages, prompt, Project, }; -use pixi_config::ConfigCliPrompt; +use pixi_config::{ConfigCliActivation, ConfigCliPrompt}; use pixi_manifest::EnvironmentName; #[cfg(target_family = "unix")] use pixi_pty::unix::PtySession; @@ -35,6 +35,9 @@ pub struct Args { #[clap(flatten)] prompt_config: ConfigCliPrompt, + + #[clap(flatten)] + activation_config: ConfigCliActivation, } /// Set up Ctrl-C handler to ignore it (the child process should react on CTRL-C) @@ -224,10 +227,13 @@ async fn start_nu_shell( pub async fn execute(args: Args) -> miette::Result<()> { let config = args - .prompt_config - .merge_with_config(args.prefix_update_config.config.clone().into()); + .activation_config + .merge_config(args.prompt_config.into()) + .merge_config(args.prefix_update_config.config.clone().into()); + let project = Project::load_or_else_discover(args.project_config.manifest_path.as_deref())? .with_cli_config(config); + let environment = project.environment_from_name_or_env_var(args.environment)?; verify_current_platform_has_required_virtual_packages(&environment).into_diagnostic()?; @@ -238,7 +244,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { }; // Make sure environment is up-to-date, default to install, users can avoid this with frozen or locked. - update_prefix( + let (lock_file_data, _prefix) = get_update_lock_file_and_prefix( &environment, args.prefix_update_config.lock_file_usage(), false, @@ -248,7 +254,13 @@ pub async fn execute(args: Args) -> miette::Result<()> { // Get the environment variables we need to set activate the environment in the shell. let env = project - .get_activated_environment_variables(&environment, CurrentEnvVarBehavior::Exclude) + .get_activated_environment_variables( + &environment, + CurrentEnvVarBehavior::Exclude, + Some(&lock_file_data.lock_file), + project.config().force_activate(), + project.config().experimental_activation_cache_usage(), + ) .await?; tracing::debug!("Pixi environment activation:\n{:?}", env); diff --git a/src/cli/shell_hook.rs b/src/cli/shell_hook.rs index 6236451ff..0f663981c 100644 --- a/src/cli/shell_hook.rs +++ b/src/cli/shell_hook.rs @@ -2,7 +2,8 @@ use std::{collections::HashMap, default::Default}; use clap::Parser; use miette::IntoDiagnostic; -use pixi_config::ConfigCliPrompt; +use pixi_config::{ConfigCliActivation, ConfigCliPrompt}; +use rattler_lock::LockFile; use rattler_shell::{ activation::{ActivationVariables, PathModificationBehavior}, shell::ShellEnum, @@ -11,7 +12,7 @@ use serde::Serialize; use serde_json; use crate::activation::CurrentEnvVarBehavior; -use crate::environment::update_prefix; +use crate::environment::get_update_lock_file_and_prefix; use crate::{ activation::get_activator, cli::cli_config::{PrefixUpdateConfig, ProjectConfig}, @@ -36,6 +37,9 @@ pub struct Args { #[clap(flatten)] pub prefix_update_config: PrefixUpdateConfig, + #[clap(flatten)] + activation_config: ConfigCliActivation, + /// The environment to activate in the script #[arg(long, short)] environment: Option, @@ -87,10 +91,21 @@ async fn generate_activation_script( /// Generates a JSON object describing the changes to the shell environment when /// activating the provided pixi environment. -async fn generate_environment_json(environment: &Environment<'_>) -> miette::Result { +async fn generate_environment_json( + environment: &Environment<'_>, + lock_file: &LockFile, + force_activate: bool, + experimental_cache: bool, +) -> miette::Result { let environment_variables = environment .project() - .get_activated_environment_variables(environment, CurrentEnvVarBehavior::Exclude) + .get_activated_environment_variables( + environment, + CurrentEnvVarBehavior::Exclude, + Some(lock_file), + force_activate, + experimental_cache, + ) .await?; let shell_env = ShellEnv { @@ -104,12 +119,13 @@ async fn generate_environment_json(environment: &Environment<'_>) -> miette::Res pub async fn execute(args: Args) -> miette::Result<()> { let config = args .prompt_config - .merge_with_config(args.prefix_update_config.config.clone().into()); + .merge_config(args.activation_config.into()) + .merge_config(args.prefix_update_config.config.clone().into()); let project = Project::load_or_else_discover(args.project_config.manifest_path.as_deref())? .with_cli_config(config); let environment = project.environment_from_name_or_env_var(args.environment)?; - update_prefix( + let (lock_file_data, _prefix) = get_update_lock_file_and_prefix( &environment, args.prefix_update_config.lock_file_usage(), false, @@ -118,7 +134,17 @@ pub async fn execute(args: Args) -> miette::Result<()> { .await?; let output = match args.json { - true => generate_environment_json(&environment).await?, + true => { + generate_environment_json( + &environment, + &lock_file_data.lock_file, + project.config().force_activate(), + project.config().experimental_activation_cache_usage(), + ) + .await? + } + // Skipping the activated environment caching for the script. + // As it can still run scripts. false => generate_activation_script(args.shell, &environment).await?, }; diff --git a/src/environment.rs b/src/environment.rs index 4f5e35240..2fa369a28 100644 --- a/src/environment.rs +++ b/src/environment.rs @@ -34,6 +34,7 @@ use std::{ use tokio::sync::Semaphore; use uv_distribution_types::{InstalledDist, Name}; +use crate::lock_file::LockFileDerivedData; use xxhash_rust::xxh3::Xxh3; /// Verify the location of the prefix folder is not changed so the applied @@ -388,12 +389,12 @@ impl LockFileUsage { /// takes up a lot of memory and takes a while to load. If `sparse_repo_data` is /// `None` it will be downloaded. If the lock-file is not updated, the /// `sparse_repo_data` is ignored. -pub async fn update_prefix( - environment: &Environment<'_>, +pub async fn get_update_lock_file_and_prefix<'env>( + environment: &Environment<'env>, lock_file_usage: LockFileUsage, mut no_install: bool, update_mode: UpdateMode, -) -> miette::Result<()> { +) -> miette::Result<(LockFileDerivedData<'env>, Prefix)> { let current_platform = environment.best_platform(); let project = environment.project(); @@ -415,11 +416,14 @@ pub async fn update_prefix( }) .await?; - // Get the locked environment from the lock-file. - if !no_install { - lock_file.prefix(environment, update_mode).await?; - } - Ok(()) + // Get the prefix from the lock-file. + let prefix = if no_install { + Prefix::new(environment.dir()) + } else { + lock_file.prefix(environment, update_mode).await? + }; + + Ok((lock_file, prefix)) } #[allow(clippy::too_many_arguments)] diff --git a/src/lock_file/update.rs b/src/lock_file/update.rs index 5cdb996dd..dc3e28efa 100644 --- a/src/lock_file/update.rs +++ b/src/lock_file/update.rs @@ -72,6 +72,11 @@ impl Project { ) -> miette::Result> { update::update_lock_file(self, options).await } + + /// Get lockfile without checking + pub async fn get_lock_file(&self) -> miette::Result { + load_lock_file(self).await + } } #[derive(Debug, Error, Diagnostic)] @@ -240,7 +245,15 @@ impl<'p> LockFileDerivedData<'p> { // TODO: This can be really slow (~200ms for pixi on @ruben-arts machine). let env_variables = self .project - .get_activated_environment_variables(environment, CurrentEnvVarBehavior::Exclude) + // Not providing a lock-file as the cache will be invalidated directly anyway, + // by it changing the lockfile with pypi records. + .get_activated_environment_variables( + environment, + CurrentEnvVarBehavior::Exclude, + None, + false, + false, + ) .await?; let non_isolated_packages = environment.pypi_options().no_build_isolation; @@ -1157,7 +1170,13 @@ impl<'p> UpdateContext<'p> { // Get environment variables from the activation let env_variables = project - .get_activated_environment_variables(environment, CurrentEnvVarBehavior::Exclude) + .get_activated_environment_variables( + environment, + CurrentEnvVarBehavior::Exclude, + None, + false, + false, + ) .await?; // Construct a future that will resolve when we have the repodata available diff --git a/src/project/environment.rs b/src/project/environment.rs index a18969a67..1243206b9 100644 --- a/src/project/environment.rs +++ b/src/project/environment.rs @@ -98,6 +98,19 @@ impl<'p> Environment<'p> { .join(self.environment.name.as_str()) } + /// We store a hash of the lockfile and all activation env variables in a file + /// in the cache. The current name is `activation_environment-name.json`. + pub(crate) fn activation_cache_name(&self) -> String { + format!("activation_{}.json", self.name()) + } + + /// Returns the activation cache file path. + pub(crate) fn activation_cache_file_path(&self) -> std::path::PathBuf { + self.project + .activation_env_cache_folder() + .join(self.activation_cache_name()) + } + /// Returns the best platform for the current platform & environment. pub fn best_platform(&self) -> Platform { let current = Platform::current(); diff --git a/src/project/mod.rs b/src/project/mod.rs index d79167e77..25522051a 100644 --- a/src/project/mod.rs +++ b/src/project/mod.rs @@ -436,6 +436,9 @@ impl Project { &self, environment: &Environment<'_>, current_env_var_behavior: CurrentEnvVarBehavior, + lock_file: Option<&LockFile>, + force_activate: bool, + experimental_cache: bool, ) -> miette::Result<&HashMap> { let vars = self.env_vars.get(environment.name()).ok_or_else(|| { miette::miette!( @@ -447,26 +450,48 @@ impl Project { CurrentEnvVarBehavior::Clean => { vars.clean() .get_or_try_init(async { - initialize_env_variables(environment, current_env_var_behavior).await + initialize_env_variables( + environment, + current_env_var_behavior, + lock_file, + force_activate, + experimental_cache, + ) + .await }) .await } CurrentEnvVarBehavior::Exclude => { vars.pixi_only() .get_or_try_init(async { - initialize_env_variables(environment, current_env_var_behavior).await + initialize_env_variables( + environment, + current_env_var_behavior, + lock_file, + force_activate, + experimental_cache, + ) + .await }) .await } CurrentEnvVarBehavior::Include => { vars.full() .get_or_try_init(async { - initialize_env_variables(environment, current_env_var_behavior).await + initialize_env_variables( + environment, + current_env_var_behavior, + lock_file, + force_activate, + experimental_cache, + ) + .await }) .await } } } + /// Returns all the solve groups in the project. pub(crate) fn solve_groups(&self) -> Vec { self.manifest @@ -526,6 +551,10 @@ impl Project { self.pixi_dir().join(consts::TASK_CACHE_DIR) } + pub(crate) fn activation_env_cache_folder(&self) -> PathBuf { + self.pixi_dir().join(consts::ACTIVATION_ENV_CACHE_DIR) + } + /// Returns what pypi mapping configuration we should use. /// It can be a custom one in following format : conda_name: pypi_name /// Or we can use our self-hosted diff --git a/src/task/executable_task.rs b/src/task/executable_task.rs index 7850c6f0d..e28001bf8 100644 --- a/src/task/executable_task.rs +++ b/src/task/executable_task.rs @@ -10,6 +10,7 @@ use deno_task_shell::{ }; use itertools::Itertools; use miette::{Context, Diagnostic, IntoDiagnostic}; +use rattler_lock::LockFile; use thiserror::Error; use tokio::task::JoinHandle; @@ -246,17 +247,14 @@ impl<'p> ExecutableTask<'p> { /// `CanSkip::No` and includes the hash of the task that caused the task /// to not be skipped - we can use this later to update the cache file /// quickly. - pub(crate) async fn can_skip( - &self, - lock_file: &LockFileDerivedData<'p>, - ) -> Result { + pub(crate) async fn can_skip(&self, lock_file: &LockFile) -> Result { tracing::info!("Checking if task can be skipped"); let cache_name = self.cache_name(); let cache_file = self.project().task_cache_folder().join(cache_name); if cache_file.exists() { let cache = tokio::fs::read_to_string(&cache_file).await?; let cache: TaskCache = serde_json::from_str(&cache)?; - let hash = TaskHash::from_task(self, &lock_file.lock_file).await; + let hash = TaskHash::from_task(self, lock_file).await; if let Ok(Some(hash)) = hash { if hash.computation_hash() != cache.hash { return Ok(CanSkip::No(Some(hash))); @@ -353,6 +351,9 @@ fn get_export_specific_task_env(task: &Task) -> String { pub async fn get_task_env<'p>( environment: &Environment<'p>, clean_env: bool, + lock_file: Option<&LockFile>, + force_activate: bool, + experimental_cache: bool, ) -> miette::Result> { // Make sure the system requirements are met verify_current_platform_has_required_virtual_packages(environment).into_diagnostic()?; @@ -364,9 +365,13 @@ pub async fn get_task_env<'p>( CurrentEnvVarBehavior::Include }; let mut activation_env = await_in_progress("activating environment", |_| { - environment - .project() - .get_activated_environment_variables(environment, env_var_behavior) + environment.project().get_activated_environment_variables( + environment, + env_var_behavior, + lock_file, + force_activate, + experimental_cache, + ) }) .await .wrap_err("failed to activate environment")? @@ -471,7 +476,9 @@ mod tests { let project = Project::from_manifest(manifest); let environment = project.default_environment(); - let env = get_task_env(&environment, false).await.unwrap(); + let env = get_task_env(&environment, false, None, false, false) + .await + .unwrap(); assert_eq!( env.get("INIT_CWD").unwrap(), &std::env::current_dir() diff --git a/src/task/mod.rs b/src/task/mod.rs index f729e05c3..dce93537b 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -7,7 +7,7 @@ mod task_hash; pub use file_hashes::{FileHashes, FileHashesError}; pub use pixi_manifest::{Task, TaskName}; -pub use task_hash::{ComputationHash, InputHashes, TaskHash}; +pub use task_hash::{ComputationHash, EnvironmentHash, InputHashes, TaskHash}; pub use executable_task::{ get_task_env, CanSkip, ExecutableTask, FailedToParseShellScript, InvalidWorkingDirectory, diff --git a/src/task/task_hash.rs b/src/task/task_hash.rs index 8a0631b15..e5a455dab 100644 --- a/src/task/task_hash.rs +++ b/src/task/task_hash.rs @@ -3,6 +3,7 @@ use crate::task::{ExecutableTask, FileHashes, FileHashesError, InvalidWorkingDir use miette::Diagnostic; use rattler_lock::LockFile; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use std::fmt::{Display, Formatter}; use std::hash::{Hash, Hasher}; use thiserror::Error; @@ -33,21 +34,46 @@ pub struct TaskCache { pub hash: ComputationHash, } -#[derive(Debug, Hash)] +#[derive(Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub struct EnvironmentHash(String); impl EnvironmentHash { - fn from_environment(run_environment: &project::Environment<'_>, lock_file: &LockFile) -> Self { + pub(crate) fn from_environment( + run_environment: &project::Environment<'_>, + input_environment_variables: &HashMap>, + lock_file: &LockFile, + ) -> Self { let mut hasher = Xxh3::new(); + + // Hash the environment variables + let mut sorted_input_environment_variables: Vec<_> = + input_environment_variables.iter().collect(); + sorted_input_environment_variables.sort_by_key(|(key, _)| *key); + for (key, value) in sorted_input_environment_variables { + key.hash(&mut hasher); + value.hash(&mut hasher); + } + + // Hash the activation scripts let activation_scripts = run_environment.activation_scripts(Some(run_environment.best_platform())); - for script in activation_scripts { script.hash(&mut hasher); } - let mut urls = Vec::new(); + // Hash the environment variables + let project_activation_env = + run_environment.activation_env(Some(run_environment.best_platform())); + let mut env_vars: Vec<_> = project_activation_env.iter().collect(); + env_vars.sort_by_key(|(key, _)| *key); + + for (key, value) in env_vars { + key.hash(&mut hasher); + value.hash(&mut hasher); + } + // Hash the packages + let mut urls = Vec::new(); if let Some(env) = lock_file.environment(run_environment.name().as_str()) { if let Some(packages) = env.packages(run_environment.best_platform()) { for package in packages { @@ -55,14 +81,19 @@ impl EnvironmentHash { } } } - urls.sort(); - urls.hash(&mut hasher); + EnvironmentHash(format!("{:x}", hasher.finish())) } } +impl Display for EnvironmentHash { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + /// The [`TaskHash`] group all the hashes of a task. It can be converted to a [`ComputationHash`] /// with the [`TaskHash::computation_hash`] method. #[derive(Debug)] @@ -90,7 +121,12 @@ impl TaskHash { command: task.full_command(), outputs: output_hashes, inputs: input_hashes, - environment: EnvironmentHash::from_environment(&task.run_environment, lock_file), + // Skipping environment variables used for caching the task + environment: EnvironmentHash::from_environment( + &task.run_environment, + &HashMap::new(), + lock_file, + ), })) } diff --git a/tests/integration_python/test_run_cli.py b/tests/integration_python/test_run_cli.py index 7fbc3d848..82f611c49 100644 --- a/tests/integration_python/test_run_cli.py +++ b/tests/integration_python/test_run_cli.py @@ -1,3 +1,4 @@ +import json from pathlib import Path from .common import verify_cli_command, ExitCode, default_env_path @@ -153,3 +154,68 @@ def test_prefix_revalidation(pixi: Path, tmp_path: Path, dummy_channel_1: str) - # Validate that the dummy-a files are reinstalled for file in dummy_a_meta_files: assert Path(file).exists() + + +def test_run_with_activation(pixi: Path, tmp_path: Path) -> None: + manifest = tmp_path.joinpath("pixi.toml") + toml = f""" + {EMPTY_BOILERPLATE_PROJECT} + [activation.env] + TEST_ENV_VAR_FOR_ACTIVATION_TEST = "test123" + [tasks] + task = "echo $TEST_ENV_VAR_FOR_ACTIVATION_TEST" + """ + manifest.write_text(toml) + + # Run the default task + verify_cli_command( + [pixi, "run", "--manifest-path", manifest, "task"], + ExitCode.SUCCESS, + stdout_contains="test123", + ) + + # Validate that without experimental it does not use the cache + assert not tmp_path.joinpath(".pixi/activation-env-v0").exists() + + # Enable the experimental cache config + verify_cli_command( + [ + pixi, + "config", + "set", + "--manifest-path", + manifest, + "--local", + "experimental.use-environment-activation-cache", + "true", + ], + ExitCode.SUCCESS, + ) + + # Run the default task and create cache + verify_cli_command( + [pixi, "run", "--manifest-path", manifest, "task"], + ExitCode.SUCCESS, + stdout_contains="test123", + ) + + # Modify the environment variable in cache + cache_path = tmp_path.joinpath(".pixi", "activation-env-v0", "activation_default.json") + data = json.loads(cache_path.read_text()) + data["environment_variables"]["TEST_ENV_VAR_FOR_ACTIVATION_TEST"] = "test456" + cache_path.write_text(json.dumps(data, indent=4)) + + verify_cli_command( + [pixi, "run", "--manifest-path", manifest, "task"], + ExitCode.SUCCESS, + # Contain overwritten value + stdout_contains="test456", + stdout_excludes="test123", + ) + + # Ignore activation cache + verify_cli_command( + [pixi, "run", "--manifest-path", manifest, "--force-activate", "task", "-vvv"], + ExitCode.SUCCESS, + stdout_contains="test123", + ) diff --git a/tests/integration_rust/common/mod.rs b/tests/integration_rust/common/mod.rs index 464f78a02..1fb6ee71d 100644 --- a/tests/integration_rust/common/mod.rs +++ b/tests/integration_rust/common/mod.rs @@ -446,7 +446,9 @@ impl PixiControl { lock_file .prefix(&task.run_environment, UpdateMode::Revalidate) .await?; - let env = get_task_env(&task.run_environment, args.clean_env).await?; + let env = + get_task_env(&task.run_environment, args.clean_env, None, false, false) + .await?; task_env.insert(env) } Some(task_env) => task_env, diff --git a/tests/integration_rust/install_tests.rs b/tests/integration_rust/install_tests.rs index 9cf48484e..76b369a08 100644 --- a/tests/integration_rust/install_tests.rs +++ b/tests/integration_rust/install_tests.rs @@ -569,7 +569,7 @@ async fn test_old_lock_install() { "tests/data/satisfiability/old_lock_file/pyproject.toml", )) .unwrap(); - pixi::environment::update_prefix( + pixi::environment::get_update_lock_file_and_prefix( &project.default_environment(), LockFileUsage::Update, false, diff --git a/tests/integration_rust/test_activation.rs b/tests/integration_rust/test_activation.rs index d1e9d633d..259ae1437 100644 --- a/tests/integration_rust/test_activation.rs +++ b/tests/integration_rust/test_activation.rs @@ -15,7 +15,13 @@ async fn test_pixi_only_env_activation() { let default_env = project.default_environment(); let pixi_only_env = project - .get_activated_environment_variables(&default_env, CurrentEnvVarBehavior::Exclude) + .get_activated_environment_variables( + &default_env, + CurrentEnvVarBehavior::Exclude, + None, + false, + false, + ) .await .unwrap(); @@ -38,7 +44,13 @@ async fn test_full_env_activation() { std::env::set_var("DIRTY_VAR", "Dookie"); let full_env = project - .get_activated_environment_variables(&default_env, CurrentEnvVarBehavior::Include) + .get_activated_environment_variables( + &default_env, + CurrentEnvVarBehavior::Include, + None, + false, + false, + ) .await .unwrap(); assert!(full_env.get("CONDA_PREFIX").is_some()); @@ -58,7 +70,13 @@ async fn test_clean_env_activation() { std::env::set_var("DIRTY_VAR", "Dookie"); let clean_env = project - .get_activated_environment_variables(&default_env, CurrentEnvVarBehavior::Clean) + .get_activated_environment_variables( + &default_env, + CurrentEnvVarBehavior::Clean, + None, + false, + false, + ) .await .unwrap(); assert!(clean_env.get("CONDA_PREFIX").is_some());