perf: Experimental environment activation cache (prefix-dev#2367)

Co-authored-by: Hofer-Julian <[email protected]>
Co-authored-by: Julian Hofer <[email protected]>

crates/pixi_config/src/lib.rs

@@ -20,6 +20,8 @@ use reqwest_middleware::ClientWithMiddleware;
 use serde::{de::IntoDeserializer, Deserialize, Serialize};
 use url::Url;
 
+const EXPERIMENTAL: &str = "experimental";
+
 pub fn default_channel_config() -> ChannelConfig {
     ChannelConfig::default_with_root_dir(
         std::env::current_dir().expect("Could not retrieve the current directory"),
@@ -122,9 +124,17 @@ pub struct ConfigCliPrompt {
     #[arg(long)]
     change_ps1: Option<bool>,
 }
+impl From<ConfigCliPrompt> for Config {
+    fn from(cli: ConfigCliPrompt) -> Self {
+        Self {
+            change_ps1: cli.change_ps1,
+            ..Default::default()
+        }
+    }
+}
 
 impl ConfigCliPrompt {
-    pub fn merge_with_config(self, config: Config) -> Config {
+    pub fn merge_config(self, config: Config) -> Config {
         let mut config = config;
         config.change_ps1 = self.change_ps1.or(config.change_ps1);
         config
@@ -168,6 +178,29 @@ impl RepodataConfig {
     }
 }
 
+#[derive(Parser, Debug, Default, Clone)]
+pub struct ConfigCliActivation {
+    /// Do not use the environment activation cache. (default: true except in experimental mode)
+    #[arg(long)]
+    force_activate: bool,
+}
+
+impl ConfigCliActivation {
+    pub fn merge_config(self, config: Config) -> Config {
+        let mut config = config;
+        config.force_activate = Some(self.force_activate);
+        config
+    }
+}
+
+impl From<ConfigCliActivation> for Config {
+    fn from(cli: ConfigCliActivation) -> Self {
+        Self {
+            force_activate: Some(cli.force_activate),
+            ..Default::default()
+        }
+    }
+}
 #[derive(Clone, Default, Debug, Deserialize, Serialize)]
 #[serde(deny_unknown_fields, rename_all = "kebab-case")]
 pub struct RepodataChannelConfig {
@@ -273,6 +306,33 @@ impl Default for DetachedEnvironments {
     }
 }
 
+#[derive(Default, Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
+#[serde(rename_all = "kebab-case")]
+pub struct ExperimentalConfig {
+    /// The option to opt into the environment activation cache feature.
+    /// This is an experimental feature and may be removed in the future or made default.
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub use_environment_activation_cache: Option<bool>,
+}
+
+impl ExperimentalConfig {
+    pub fn merge(self, other: Self) -> Self {
+        Self {
+            use_environment_activation_cache: other
+                .use_environment_activation_cache
+                .or(self.use_environment_activation_cache),
+        }
+    }
+    pub fn use_environment_activation_cache(&self) -> bool {
+        self.use_environment_activation_cache.unwrap_or(false)
+    }
+
+    pub fn is_default(&self) -> bool {
+        self.use_environment_activation_cache.is_none()
+    }
+}
+
 impl PyPIConfig {
     /// Merge the given PyPIConfig into the current one.
     pub fn merge(self, other: Self) -> Self {
@@ -479,6 +539,16 @@ pub struct Config {
     /// it back to the .pixi folder.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub detached_environments: Option<DetachedEnvironments>,
+
+    /// The option to disable the environment activation cache
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub force_activate: Option<bool>,
+
+    /// Experimental features that can be enabled.
+    #[serde(default)]
+    #[serde(skip_serializing_if = "ExperimentalConfig::is_default")]
+    pub experimental: ExperimentalConfig,
 }
 
 impl Default for Config {
@@ -495,6 +565,8 @@ impl Default for Config {
             pypi_config: PyPIConfig::default(),
             detached_environments: Some(DetachedEnvironments::default()),
             pinning_strategy: Default::default(),
+            force_activate: None,
+            experimental: Default::default(),
         }
     }
 }
@@ -735,6 +807,7 @@ impl Config {
             "pypi-config.index-url",
             "pypi-config.extra-index-urls",
             "pypi-config.keyring-provider",
+            "experimental.use-environment-activation-cache",
         ]
     }
 
@@ -763,6 +836,8 @@ impl Config {
             pypi_config: other.pypi_config.merge(self.pypi_config),
             detached_environments: other.detached_environments.or(self.detached_environments),
             pinning_strategy: other.pinning_strategy.or(self.pinning_strategy),
+            force_activate: other.force_activate,
+            experimental: other.experimental.merge(self.experimental),
         }
     }
 
@@ -820,6 +895,14 @@ impl Config {
         self.detached_environments.clone().unwrap_or_default()
     }
 
+    pub fn force_activate(&self) -> bool {
+        self.force_activate.unwrap_or(false)
+    }
+
+    pub fn experimental_activation_cache_usage(&self) -> bool {
+        self.experimental.use_environment_activation_cache()
+    }
+
     /// Modify this config with the given key and value
     ///
     /// # Note
@@ -943,6 +1026,29 @@ impl Config {
                     _ => return Err(err),
                 }
             }
+            key if key.starts_with(EXPERIMENTAL) => {
+                if key == EXPERIMENTAL {
+                    if let Some(value) = value {
+                        self.experimental = serde_json::de::from_str(&value).into_diagnostic()?;
+                    } else {
+                        self.experimental = ExperimentalConfig::default();
+                    }
+                    return Ok(());
+                } else if !key.starts_with(format!("{EXPERIMENTAL}.").as_str()) {
+                    return Err(err);
+                }
+
+                let subkey = key
+                    .strip_prefix(format!("{EXPERIMENTAL}.").as_str())
+                    .unwrap();
+                match subkey {
+                    "use-environment-activation-cache" => {
+                        self.experimental.use_environment_activation_cache =
+                            value.map(|v| v.parse()).transpose().into_diagnostic()?;
+                    }
+                    _ => return Err(err),
+                }
+            }
             _ => return Err(err),
         }
 
@@ -1091,6 +1197,7 @@ UNUSED = "unused"
             config.authentication_override_file,
             Some(PathBuf::from("path.json"))
         );
+        assert!(!config.experimental.use_environment_activation_cache());
     }
 
     #[test]

crates/pixi_config/src/snapshots/pixi_config__tests__config_merge.snap

@@ -68,4 +68,8 @@ Config {
             true,
         ),
     ),
+    force_activate: None,
+    experimental: ExperimentalConfig {
+        use_environment_activation_cache: None,
+    },
 }

crates/pixi_consts/src/consts.rs

@@ -22,6 +22,7 @@ pub const SOLVE_GROUP_ENVIRONMENTS_DIR: &str = "solve-group-envs";
 pub const PYPI_DEPENDENCIES: &str = "pypi-dependencies";
 pub const DEPENDENCIES: &str = "dependencies";
 pub const TASK_CACHE_DIR: &str = "task-cache-v0";
+pub const ACTIVATION_ENV_CACHE_DIR: &str = "activation-env-v0";
 pub const PIXI_UV_INSTALLER: &str = "uv-pixi";
 pub const CONDA_PACKAGE_CACHE_DIR: &str = rattler_cache::PACKAGE_CACHE_DIR;
 pub const CONDA_REPODATA_CACHE_DIR: &str = rattler_cache::REPODATA_CACHE_DIR;

docs/reference/cli.md

@@ -250,6 +250,7 @@ You cannot run `pixi run source setup.bash` as `source` is not available in the
 - `--locked`: only install if the `pixi.lock` is up-to-date with the [manifest file](project_configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`.
 - `--environment <ENVIRONMENT> (-e)`: The environment to run the task in, if none are provided the default environment will be used or a selector will be given to select the right environment.
 - `--clean-env`: Run the task in a clean environment, this will remove all environment variables of the shell environment except for the ones pixi sets. THIS DOESN't WORK ON `Windows`.
+- `--force-activate`: (default, except in _experimental_ mode) Force the activation of the environment, even if the environment is already activated.
 - `--revalidate`: Revalidate the full environment, instead of checking the lock file hash. [more info](../features/environment.md#environment-installation-metadata)
 
 ```shell
@@ -509,6 +510,8 @@ List project's packages. Highlighted packages are explicit dependencies.
 - `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](project_configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`).
 - `--locked`: Only install if the `pixi.lock` is up-to-date with the [manifest file](project_configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`.
 - `--no-install`: Don't install the environment for pypi solving, only update the lock-file if it can solve without installing. (Implied by `--frozen` and `--locked`)
+- `--no-lockfile-update`: Don't update the lock-file, implies the `--no-install` flag.
+- `--no-progress`: Hide all progress bars, always turned on if stderr is not a terminal [env: PIXI_NO_PROGRESS=]
 
 ```shell
 pixi list
@@ -570,6 +573,8 @@ The package tree can also be inverted (`-i`), to see which packages require a sp
 - `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](project_configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`).
 - `--locked`: Only install if the `pixi.lock` is up-to-date with the [manifest file](project_configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`.
 - `--no-install`: Don't install the environment for pypi solving, only update the lock-file if it can solve without installing. (Implied by `--frozen` and `--locked`)
+- `--no-lockfile-update`: Don't update the lock-file, implies the `--no-install` flag.
+- `--no-progress`: Hide all progress bars, always turned on if stderr is not a terminal [env: PIXI_NO_PROGRESS=]
 
 ```shell
 pixi tree
@@ -680,7 +685,11 @@ To exit the pixi shell, simply run `exit`.
 - `--manifest-path <MANIFEST_PATH>`: the path to [manifest file](project_configuration.md), by default it searches for one in the parent directories.
 - `--frozen`: install the environment as defined in the lock file, doesn't update `pixi.lock` if it isn't up-to-date with [manifest file](project_configuration.md). It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`).
 - `--locked`: only install if the `pixi.lock` is up-to-date with the [manifest file](project_configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`.
+- `--no-install`: Don't install the environment, only activate the environment.
+- `--no-lockfile-update`: Don't update the lock-file, implies the `--no-install` flag.
 - `--environment <ENVIRONMENT> (-e)`: The environment to activate the shell in, if none are provided the default environment will be used or a selector will be given to select the right environment.
+- `--no-progress`: Hide all progress bars, always turned on if stderr is not a terminal [env: PIXI_NO_PROGRESS=]
+- `--force-activate`: (default, except in _experimental_ mode) Force the activation of the environment, even if the environment is already activated.
 - `--revalidate`: Revalidate the full environment, instead of checking lock file hash. [more info](../features/environment.md#environment-installation-metadata)
 
 ```shell
@@ -710,6 +719,7 @@ This command prints the activation script of an environment.
 - `--environment <ENVIRONMENT> (-e)`: The environment to activate, if none are provided the default environment will be used or a selector will be given to select the right environment.
 - `--json`: Print all environment variables that are exported by running the activation script as JSON. When specifying
   this option, `--shell` is ignored.
+- `--force-activate`: (default, except in _experimental_ mode) Force the activation of the environment, even if the environment is already activated.
 - `--revalidate`: Revalidate the full environment, instead of checking lock file hash. [more info](../features/environment.md#environment-installation-metadata)
 
 ```shell
@@ -876,6 +886,7 @@ Use this command to manage the configuration.
 - `--system (-s)`: Specify management scope to system configuration.
 - `--global (-g)`: Specify management scope to global configuration.
 - `--local (-l)`: Specify management scope to local configuration.
+- `--manifest-path <MANIFEST_PATH>`: the path to [manifest file](project_configuration.md), by default it searches for one in the parent directories.
 
 Checkout the [pixi configuration](./pixi_configuration.md) for more information about the locations.
 

docs/reference/pixi_configuration.md

@@ -44,7 +44,7 @@ The configuration is loaded in the following order:
     To find the locations where `pixi` looks for configuration files, run
     `pixi` with `-vv`.
 
-## Reference
+## Configuration options
 
 ??? info "Casing In Configuration"
     In versions of pixi `0.20.1` and older the global configuration used snake_case
@@ -222,6 +222,55 @@ keyring-provider = "subprocess"
     Unlike pip, these settings, with the exception of `keyring-provider` will only modify the `pixi.toml`/`pyproject.toml` file and are not globally interpreted when not present in the manifest.
     This is because we want to keep the manifest file as complete and reproducible as possible.
 
+## Experimental
+This allows the user to set specific experimental features that are not yet stable.
+
+Please write a GitHub issue and add the flag `experimental` to the issue if you find issues with the feature you activated.
+
+
+### Caching environment activations
+Turn this feature on from configuration with the following command:
+```shell
+# For all your projects
+pixi config set experimental.use-environment-activation-cache true --global
+
+# For a specific project
+pixi config set experimental.use-environment-activation-cache true --local
+```
+
+This will cache the environment activation in the `.pixi/activation-env-v0` folder in the project root.
+It will create a json file for each environment that is activated, and it will be used to activate the environment in the future.
+```bash
+> tree .pixi/activation-env-v0/
+.pixi/activation-env-v0/
+├── activation_default.json
+└── activation_lint.json
+
+> cat  .pixi/activation-env-v0/activation_lint.json
+{"hash":"8d8344e0751d377a","environment_variables":{<ENVIRONMENT_VARIABLES_USED_IN_ACTIVATION>}}
+```
+
+- The `hash` is a hash of the data on that environment in the `pixi.lock`, plus some important information on the environment activation.
+  Like `[activation.scripts]` and `[activation.env]` from the manifest file.
+- The `environment_variables` are the environment variables that are set when activating the environment.
+
+You can ignore the cache by running:
+```
+pixi run/shell/shell-hook --force-activate
+```
+
+Set the configuration with:
+```toml title="config.toml"
+[experimental]
+# Enable the use of the environment activation cache
+use-environment-activation-cache = true
+```
+
+!!! note "Why is this experimental?"
+This feature is experimental because the cache invalidation is very tricky,
+and we don't want to disturb users that are not affected by activation times.
+
+
 ## Mirror configuration
 
 You can configure mirrors for conda channels. We expect that mirrors are exact