diff --git a/Cargo.lock b/Cargo.lock index c3c21d51e..24b874ef4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3577,10 +3577,18 @@ dependencies = [ "jsonrpsee", "miette 7.2.0", "pixi_build_types", + "pixi_config", "pixi_consts", "pixi_manifest", + "pixi_utils", + "rattler", "rattler_conda_types", + "rattler_repodata_gateway", + "rattler_shell", + "rattler_solve", + "rattler_virtual_packages", "regex", + "reqwest-middleware", "rstest", "serde", "serde_json", @@ -3636,7 +3644,7 @@ version = "0.1.0" dependencies = [ "console", "lazy_static", - "rattler_cache 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rattler_cache", "url", ] @@ -4185,7 +4193,7 @@ dependencies = [ "memmap2 0.9.5", "once_cell", "parking_lot 0.12.3", - "rattler_cache 0.2.8 (git+https://github.com/conda/rattler?branch=feat/pixi-build)", + "rattler_cache", "rattler_conda_types", "rattler_digest", "rattler_networking", @@ -4195,7 +4203,7 @@ dependencies = [ "regex", "reqwest", "reqwest-middleware", - "simple_spawn_blocking 1.0.0 (git+https://github.com/conda/rattler?branch=feat/pixi-build)", + "simple_spawn_blocking", "smallvec", "tempfile", "thiserror", @@ -4205,34 +4213,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "rattler_cache" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e57222ac28f224f675de54452b239f532e4f55ea36836526ae75e5a73cce01" -dependencies = [ - "anyhow", - "dashmap", - "digest", - "dirs", - "fs4", - "futures", - "fxhash", - "itertools 0.13.0", - "parking_lot 0.12.3", - "rattler_conda_types", - "rattler_digest", - "rattler_networking", - "rattler_package_streaming", - "reqwest", - "reqwest-middleware", - "simple_spawn_blocking 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "thiserror", - "tokio", - "tracing", - "url", -] - [[package]] name = "rattler_cache" version = "0.2.8" @@ -4253,7 +4233,7 @@ dependencies = [ "rattler_package_streaming", "reqwest", "reqwest-middleware", - "simple_spawn_blocking 1.0.0 (git+https://github.com/conda/rattler?branch=feat/pixi-build)", + "simple_spawn_blocking", "thiserror", "tokio", "tracing", @@ -4440,7 +4420,7 @@ dependencies = [ "ouroboros", "parking_lot 0.12.3", "pin-project-lite", - "rattler_cache 0.2.8 (git+https://github.com/conda/rattler?branch=feat/pixi-build)", + "rattler_cache", "rattler_conda_types", "rattler_digest", "rattler_networking", @@ -4451,7 +4431,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "simple_spawn_blocking 1.0.0 (git+https://github.com/conda/rattler?branch=feat/pixi-build)", + "simple_spawn_blocking", "superslice", "tempfile", "thiserror", @@ -5505,15 +5485,6 @@ dependencies = [ "time", ] -[[package]] -name = "simple_spawn_blocking" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b31ed96d1593e129cc76cb7aca364fb5c173558bfda922c15aac4e2f2f5844e" -dependencies = [ - "tokio", -] - [[package]] name = "simple_spawn_blocking" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index 4aeb137b8..edf20e67f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -103,18 +103,30 @@ which = "6.0.3" # Rattler crates file_url = "0.1.4" -rattler = { version = "0.28.0", default-features = false } -rattler_cache = { version = "0.2.7", default-features = false } -rattler_conda_types = { version = "0.29.0", default-features = false } -rattler_digest = { version = "1.0.2", default-features = false } -rattler_lock = { version = "0.22.29", default-features = false } -rattler_networking = { version = "0.21.5", default-features = false, features = [ - "google-cloud-auth", -] } -rattler_repodata_gateway = { version = "0.21.18", default-features = false } -rattler_shell = { version = "0.22.4", default-features = false } -rattler_solve = { version = "1.1.0", default-features = false } -rattler_virtual_packages = { version = "1.1.7", default-features = false } +# rattler = { version = "0.28.0", default-features = false } +# rattler_cache = { version = "0.2.7", default-features = false } +# rattler_conda_types = { version = "0.29.0", default-features = false } +# rattler_digest = { version = "1.0.2", default-features = false } +# rattler_lock = { version = "0.22.29", default-features = false } +# rattler_networking = { version = "0.21.5", default-features = false, features = [ +# "google-cloud-auth", +# ] } +# rattler_repodata_gateway = { version = "0.21.18", default-features = false } +# rattler_shell = { version = "0.22.4", default-features = false } +# rattler_solve = { version = "1.1.0", default-features = false } +# rattler_virtual_packages = { version = "1.1.7", default-features = false } + +rattler = { git = "https://github.com/conda/rattler", branch = "feat/pixi-build" } +rattler_cache = { git = "https://github.com/conda/rattler", branch = "feat/pixi-build" } +rattler_conda_types = { git = "https://github.com/conda/rattler", branch = "feat/pixi-build" } +rattler_digest = { git = "https://github.com/conda/rattler", branch = "feat/pixi-build" } +rattler_lock = { git = "https://github.com/conda/rattler", branch = "feat/pixi-build" } +rattler_networking = { git = "https://github.com/conda/rattler", branch = "feat/pixi-build" } +rattler_package_streaming = { git = "https://github.com/conda/rattler", branch = "feat/pixi-build" } +rattler_repodata_gateway = { git = "https://github.com/conda/rattler", branch = "feat/pixi-build" } +rattler_shell = { git = "https://github.com/conda/rattler", branch = "feat/pixi-build" } +rattler_solve = { git = "https://github.com/conda/rattler", branch = "feat/pixi-build" } +rattler_virtual_packages = { git = "https://github.com/conda/rattler", branch = "feat/pixi-build" } # Bumping this to a higher version breaks the Windows path handling. url = "2.5.2" diff --git a/crates/pixi_build_frontend/Cargo.toml b/crates/pixi_build_frontend/Cargo.toml index d7430709e..0b08c7da8 100644 --- a/crates/pixi_build_frontend/Cargo.toml +++ b/crates/pixi_build_frontend/Cargo.toml @@ -15,10 +15,18 @@ futures = { workspace = true } itertools = { workspace = true } jsonrpsee = { workspace = true, features = ["client"] } miette = { workspace = true, features = ["fancy", "serde"] } +pixi_config = { workspace = true, features = ["rattler_repodata_gateway"] } pixi_consts = { workspace = true } pixi_manifest = { workspace = true } +pixi_utils = { workspace = true, features = ["rustls-tls"] } +rattler = { workspace = true } rattler_conda_types = { workspace = true } +rattler_repodata_gateway = { workspace = true, features = ["gateway"] } +rattler_shell = { workspace = true } +rattler_solve = { workspace = true, features = ["resolvo"] } +rattler_virtual_packages = { workspace = true } regex = { workspace = true } +reqwest-middleware = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } serde_with = { workspace = true } diff --git a/crates/pixi_build_frontend/src/build_frontend.rs b/crates/pixi_build_frontend/src/build_frontend.rs index 47eae1e53..44fa81002 100644 --- a/crates/pixi_build_frontend/src/build_frontend.rs +++ b/crates/pixi_build_frontend/src/build_frontend.rs @@ -4,12 +4,17 @@ use std::{path::PathBuf, sync::Arc}; use miette::Diagnostic; use rattler_conda_types::ChannelConfig; -use crate::{protocol, protocol_builder::ProtocolBuilder, tool::ToolCache, Protocol, SetupRequest}; +use crate::{ + protocol, + protocol_builder::ProtocolBuilder, + tool::{ToolCache, ToolContext}, + Protocol, SetupRequest, +}; /// The frontend for building packages. pub struct BuildFrontend { /// The cache for tools. This is used to avoid re-installing tools. - tool_cache: Arc, + pub tool_cache: Arc, /// The channel configuration used by the frontend channel_config: ChannelConfig, @@ -70,6 +75,19 @@ impl BuildFrontend { } } + /// Sets the tool configuration + pub fn with_tool_config(self, context: ToolContext) -> Self { + let tool_cache = ToolCache { + cache: self.tool_cache.cache.clone(), + context, + }; + + Self { + tool_cache: tool_cache.into(), + ..self + } + } + /// Constructs a new [`Protocol`] for the given request. This object can be /// used to build the package. pub async fn setup_protocol( diff --git a/crates/pixi_build_frontend/src/lib.rs b/crates/pixi_build_frontend/src/lib.rs index ff86b3526..6eef2ca61 100644 --- a/crates/pixi_build_frontend/src/lib.rs +++ b/crates/pixi_build_frontend/src/lib.rs @@ -18,7 +18,7 @@ use rattler_conda_types::MatchSpec; pub use reporters::{CondaBuildReporter, CondaMetadataReporter}; pub use reporters::{NoopCondaBuildReporter, NoopCondaMetadataReporter}; use tokio::io::{AsyncRead, AsyncWrite}; -pub use tool::{IsolatedToolSpec, SystemToolSpec, ToolSpec}; +pub use tool::{IsolatedToolSpec, SystemToolSpec, ToolContext, ToolSpec}; use url::Url; pub use crate::protocol::Protocol; diff --git a/crates/pixi_build_frontend/src/protocol_builder.rs b/crates/pixi_build_frontend/src/protocol_builder.rs index d79056058..f45817095 100644 --- a/crates/pixi_build_frontend/src/protocol_builder.rs +++ b/crates/pixi_build_frontend/src/protocol_builder.rs @@ -1,6 +1,7 @@ use std::path::{Path, PathBuf}; use rattler_conda_types::ChannelConfig; +use reqwest_middleware::ClientWithMiddleware; use crate::{ conda_build_protocol, pixi_protocol, @@ -107,6 +108,7 @@ impl ProtocolBuilder { Self::CondaBuild(protocol) => Ok(Protocol::CondaBuild( protocol .finish(tool_cache) + .await .map_err(FinishError::CondaBuild)?, )), } diff --git a/crates/pixi_build_frontend/src/protocols/conda_build/mod.rs b/crates/pixi_build_frontend/src/protocols/conda_build/mod.rs index 7380ed846..258d8a48b 100644 --- a/crates/pixi_build_frontend/src/protocols/conda_build/mod.rs +++ b/crates/pixi_build_frontend/src/protocols/conda_build/mod.rs @@ -8,6 +8,7 @@ use std::{ use miette::Diagnostic; pub use protocol::Protocol; use rattler_conda_types::{ChannelConfig, MatchSpec, ParseStrictness::Strict}; +use reqwest_middleware::ClientWithMiddleware; use thiserror::Error; use crate::{ @@ -96,8 +97,8 @@ impl ProtocolBuilder { } } - pub fn finish(self, tool: &ToolCache) -> Result { - let tool = tool.instantiate(self.backend_spec)?; + pub async fn finish(self, tool: &ToolCache) -> Result { + let tool = tool.instantiate(self.backend_spec).await?; Ok(Protocol { _channel_config: self.channel_config, tool, diff --git a/crates/pixi_build_frontend/src/protocols/conda_build/protocol.rs b/crates/pixi_build_frontend/src/protocols/conda_build/protocol.rs index 234158f14..764fa7c0b 100644 --- a/crates/pixi_build_frontend/src/protocols/conda_build/protocol.rs +++ b/crates/pixi_build_frontend/src/protocols/conda_build/protocol.rs @@ -50,12 +50,18 @@ impl Protocol { // Construct a new tool that can be used to invoke conda-render instead of the // original tool. - let conda_render_executable = tool.executable().with_file_name("conda-render"); - let conda_render_executable = if let Some(ext) = tool.executable().extension() { - conda_render_executable.with_extension(ext) + let conda_render_executable = String::from("conda-render"); + let conda_render_executable = if cfg!(windows) { + format!("{}.exe", conda_render_executable) } else { conda_render_executable }; + + // let conda_render_executable = if let Some(ext) = tool.executable().extension() { + // conda_render_executable.with_extension(ext) + // } else { + // conda_render_executable + // }; let conda_render_tool = tool.with_executable(conda_render_executable); // TODO: Properly pass channels diff --git a/crates/pixi_build_frontend/src/protocols/pixi/mod.rs b/crates/pixi_build_frontend/src/protocols/pixi/mod.rs index a5918491b..73cc8b24e 100644 --- a/crates/pixi_build_frontend/src/protocols/pixi/mod.rs +++ b/crates/pixi_build_frontend/src/protocols/pixi/mod.rs @@ -12,6 +12,7 @@ use pixi_consts::consts; use pixi_manifest::Manifest; pub use protocol::{InitializeError, Protocol}; use rattler_conda_types::ChannelConfig; +use reqwest_middleware::ClientWithMiddleware; pub(crate) use stderr::{stderr_null, stderr_stream}; use thiserror::Error; use which::Error; @@ -122,7 +123,12 @@ impl ProtocolBuilder { let tool_spec = self .backend_spec .ok_or(FinishError::NoBuildSection(self.manifest.path.clone()))?; - let tool = tool.instantiate(tool_spec).map_err(FinishError::Tool)?; + + let tool = tool + .instantiate(tool_spec) + .await + .map_err(FinishError::Tool)?; + Ok(Protocol::setup( self.source_dir, self.manifest.path, diff --git a/crates/pixi_build_frontend/src/protocols/pixi/protocol.rs b/crates/pixi_build_frontend/src/protocols/pixi/protocol.rs index c7e4b8431..e7c3f9462 100644 --- a/crates/pixi_build_frontend/src/protocols/pixi/protocol.rs +++ b/crates/pixi_build_frontend/src/protocols/pixi/protocol.rs @@ -117,6 +117,7 @@ impl Protocol { ) -> Result { match tool.try_into_executable() { Ok(tool) => { + eprintln!("Spawning tool: {:?}", tool.executable()); // Spawn the tool and capture stdin/stdout. let mut process = tokio::process::Command::from(tool.command()) .stdout(std::process::Stdio::piped()) @@ -124,11 +125,10 @@ impl Protocol { .stderr(std::process::Stdio::piped()) // TODO: Capture this? .spawn()?; - let backend_identifier = tool - .executable() - .file_stem() - .and_then(OsStr::to_str) - .map_or_else(|| "".to_string(), ToString::to_string); + let backend_identifier = tool.executable().clone(); + // .file_stem() + // .and_then(OsStr::to_str) + // .map_or_else(|| "".to_string(), ToString::to_string); // Acquire the stdin/stdout handles. let stdin = process diff --git a/crates/pixi_build_frontend/src/tool/cache.rs b/crates/pixi_build_frontend/src/tool/cache.rs index 2eb8da940..d25ed8351 100644 --- a/crates/pixi_build_frontend/src/tool/cache.rs +++ b/crates/pixi_build_frontend/src/tool/cache.rs @@ -1,6 +1,21 @@ -use std::path::PathBuf; +use std::{ + hash::{DefaultHasher, Hash, Hasher}, + path::PathBuf, +}; use dashmap::{DashMap, Entry}; +use pixi_consts::consts::{CACHED_BUILD_ENVS_DIR, CONDA_REPODATA_CACHE_DIR}; +use pixi_utils::reqwest::build_reqwest_clients; +use rattler::{install::Installer, package_cache::PackageCache}; +use rattler_conda_types::{Channel, GenericVirtualPackage, MatchSpec, Platform}; +use rattler_repodata_gateway::{ChannelConfig, Gateway}; +use rattler_shell::{ + activation::{ActivationVariables, Activator}, + shell::ShellEnum, +}; +use rattler_solve::{resolvo::Solver, SolverImpl, SolverTask}; +use rattler_virtual_packages::{VirtualPackage, VirtualPackageOverrides}; +use reqwest_middleware::{reqwest::Client, ClientWithMiddleware}; use super::IsolatedTool; use crate::{ @@ -8,12 +23,61 @@ use crate::{ IsolatedToolSpec, SystemToolSpec, }; +#[derive(Hash)] +pub struct EnvironmentHash { + pub command: String, + pub specs: Vec, + pub channels: Vec, +} + +impl EnvironmentHash { + pub(crate) fn new(command: String, specs: Vec, channels: Vec) -> Self { + Self { + command, + specs, + channels, + } + } + + /// Returns the name of the environment. + pub(crate) fn name(&self) -> String { + let mut hasher = DefaultHasher::new(); + self.hash(&mut hasher); + let hash = hasher.finish(); + format!("{}-{:x}", &self.command, hash) + } +} + +#[derive(Default, Debug)] +pub struct ToolContext { + pub gateway_config: ChannelConfig, + pub client: ClientWithMiddleware, + pub channels: Vec, +} + +impl ToolContext { + pub fn new( + gateway_config: ChannelConfig, + client: ClientWithMiddleware, + channels: Vec, + ) -> Self { + Self { + gateway_config, + client, + channels, + } + } +} + /// A [`ToolCache`] maintains a cache of environments for build tools. /// /// This is useful to ensure that if we need to build multiple packages that use /// the same tool, we can reuse their environments. +/// (nichita): it can also be seen as a way to create tools itself +#[derive(Default, Debug)] pub struct ToolCache { - cache: DashMap, + pub cache: DashMap, + pub context: ToolContext, } #[derive(thiserror::Error, Debug)] @@ -25,14 +89,14 @@ pub enum ToolCacheError { /// Describes the specification of the tool. This can be used to cache tool /// information. #[derive(Debug, Clone, Hash, Eq, PartialEq)] -enum CacheableToolSpec { +pub enum CacheableToolSpec { Isolated(IsolatedToolSpec), System(SystemToolSpec), } /// A tool that can be invoked. #[derive(Debug, Clone)] -enum CachedTool { +pub enum CachedTool { Isolated(IsolatedTool), System(SystemTool), } @@ -63,13 +127,18 @@ impl ToolCache { pub fn new() -> Self { Self { cache: DashMap::default(), + context: ToolContext::default(), } } + pub fn with_context(self, context: ToolContext) -> Self { + Self { context, ..self } + } + /// Instantiate a tool from a specification. /// /// If the tool is not already cached, it will be created and cached. - pub fn instantiate(&self, spec: ToolSpec) -> Result { + pub async fn instantiate(&self, spec: ToolSpec) -> Result { let spec = match spec { ToolSpec::Io(ipc) => return Ok(Tool::Io(ipc)), ToolSpec::Isolated(isolated) => CacheableToolSpec::Isolated(isolated), @@ -84,19 +153,93 @@ impl ToolCache { let tool: CachedTool = match spec { CacheableToolSpec::Isolated(spec) => { // Don't isolate yet we are just pretending - // TODO: add isolation - let found = which::which(&spec.command) - .map_err(|e| ToolCacheError::Instantiate(spec.command.clone().into(), e))?; - IsolatedTool::new(found, PathBuf::new()).into() + + let cache_dir = pixi_config::get_cache_dir().unwrap(); + + // collect existing dirs + // and check if matchspec can satisfy the existing cache + + // construct the gateway + // construct a new config + let config = ChannelConfig { + default: self.context.gateway_config.default.clone(), + per_channel: self.context.gateway_config.per_channel.clone(), + }; + + let gateway = Gateway::builder() + .with_client(self.context.client.clone()) + .with_cache_dir(cache_dir.join(CONDA_REPODATA_CACHE_DIR)) + .with_channel_config(config) + .finish(); + + let repodata = gateway + .query( + self.context.channels.clone(), + [Platform::current(), Platform::NoArch], + spec.specs.clone(), + ) + .recursive(true) + .execute() + .await + .unwrap(); + + // Determine virtual packages of the current platform + let virtual_packages = VirtualPackage::detect(&VirtualPackageOverrides::from_env()) + .unwrap() + .iter() + .cloned() + .map(GenericVirtualPackage::from) + .collect(); + + let solved_records = Solver + .solve(SolverTask { + specs: spec.specs.clone(), + virtual_packages, + ..SolverTask::from_iter(&repodata) + }) + .unwrap(); + + let cache = EnvironmentHash::new( + spec.command.clone(), + spec.specs, + self.context + .channels + .iter() + .map(|c| c.base_url().to_string()) + .collect(), + ); + + let cached_dir = cache_dir.join(CACHED_BUILD_ENVS_DIR).join(cache.name()); + + // Install the environment + Installer::new() + .with_download_client(self.context.client.clone()) + .with_package_cache(PackageCache::new( + cache_dir.join(pixi_consts::consts::CONDA_PACKAGE_CACHE_DIR), + )) + .install(&cached_dir, solved_records) + .await + .unwrap(); + + // get the activation scripts + let activator = + Activator::from_path(&cached_dir, ShellEnum::default(), Platform::current()) + .unwrap(); + + let activation_scripts = activator + .run_activation(ActivationVariables::from_env().unwrap_or_default(), None) + .unwrap(); + + IsolatedTool::new(spec.command, cached_dir, activation_scripts).into() } CacheableToolSpec::System(spec) => { - let exec = if spec.command.is_absolute() { - spec.command.clone() - } else { - which::which(&spec.command) - .map_err(|e| ToolCacheError::Instantiate(spec.command.clone(), e))? - }; - SystemTool::new(exec).into() + // let exec = if spec.command.is_absolute() { + // spec.command.clone() + // } else { + // which::which(&spec.command) + // .map_err(|e| ToolCacheError::Instantiate(spec.command.clone(), e))? + // }; + SystemTool::new(spec.command.to_string_lossy().to_string()).into() } }; @@ -104,3 +247,59 @@ impl ToolCache { Ok(tool.into()) } } + +#[cfg(test)] +mod tests { + use std::{path::PathBuf, str::FromStr}; + + use futures::channel; + use pixi_config::Config; + use rattler_conda_types::{ChannelConfig, MatchSpec, NamedChannelOrUrl, ParseStrictness}; + use reqwest_middleware::ClientWithMiddleware; + + use super::ToolCache; + use crate::{ + tool::{IsolatedTool, SystemTool, Tool, ToolContext, ToolSpec}, + IsolatedToolSpec, + }; + + #[tokio::test] + async fn test_tool_cache() { + let cache = ToolCache::new(); + let mut config = Config::default(); + config.default_channels = vec![NamedChannelOrUrl::Name("conda-forge".to_string())]; + + let auth_client = ClientWithMiddleware::default(); + let gateway_config = rattler_repodata_gateway::ChannelConfig::from(&config); + let channel_config = ChannelConfig::default_with_root_dir(PathBuf::new()); + + let channels = config + .default_channels + .iter() + .cloned() + .map(|c| c.into_channel(&channel_config).unwrap()) + .collect(); + + let tool_context = ToolContext::new(gateway_config, auth_client, channels); + + let cache = cache.with_context(tool_context); + + let tool_spec = IsolatedToolSpec { + specs: vec![MatchSpec::from_str("cowpy", ParseStrictness::Strict).unwrap()], + command: "cowpy".into(), + }; + + let tool = cache + .instantiate(ToolSpec::Isolated(tool_spec)) + .await + .unwrap(); + + let exec = tool.as_executable().unwrap(); + + eprintln!("{:?}", exec); + + let output = exec.command().arg("hello").spawn().unwrap(); + + eprintln!("{:?}", output); + } +} diff --git a/crates/pixi_build_frontend/src/tool/mod.rs b/crates/pixi_build_frontend/src/tool/mod.rs index 1ff0afe89..4180d626d 100644 --- a/crates/pixi_build_frontend/src/tool/mod.rs +++ b/crates/pixi_build_frontend/src/tool/mod.rs @@ -1,9 +1,12 @@ mod cache; mod spec; -use std::path::{Path, PathBuf}; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; -pub use cache::{ToolCache, ToolCacheError}; +pub use cache::{ToolCache, ToolCacheError, ToolContext}; pub use spec::{IsolatedToolSpec, SystemToolSpec, ToolSpec}; use crate::InProcessBackend; @@ -25,12 +28,12 @@ pub enum ExecutableTool { /// A tool that is pre-installed on the system. #[derive(Debug, Clone)] pub struct SystemTool { - command: PathBuf, + command: String, } impl SystemTool { /// Construct a new instance from a command. - pub(crate) fn new(command: impl Into) -> Self { + pub(crate) fn new(command: impl Into) -> Self { Self { command: command.into(), } @@ -46,16 +49,25 @@ impl From for Tool { /// A tool that is installed in its own isolated environment. #[derive(Debug, Clone)] pub struct IsolatedTool { - command: PathBuf, + /// The command to invoke. + command: String, + /// The prefix to use for the isolated environment. prefix: PathBuf, + /// Activation scripts + activation_scripts: HashMap, } impl IsolatedTool { /// Construct a new instance from a command and prefix. - pub(crate) fn new(command: impl Into, prefix: impl Into) -> Self { + pub(crate) fn new( + command: impl Into, + prefix: impl Into, + activation: HashMap, + ) -> Self { Self { command: command.into(), prefix: prefix.into(), + activation_scripts: activation, } } } @@ -86,7 +98,7 @@ impl Tool { impl ExecutableTool { /// Returns the full path to the executable to invoke. - pub fn executable(&self) -> &Path { + pub fn executable(&self) -> &String { match self { ExecutableTool::Isolated(tool) => &tool.command, ExecutableTool::System(tool) => &tool.command, @@ -94,11 +106,13 @@ impl ExecutableTool { } /// Construct a new tool that calls another executable. - pub fn with_executable(&self, executable: impl Into) -> Self { + pub fn with_executable(&self, executable: impl Into) -> Self { match self { - ExecutableTool::Isolated(tool) => { - ExecutableTool::Isolated(IsolatedTool::new(executable, tool.prefix.clone())) - } + ExecutableTool::Isolated(tool) => ExecutableTool::Isolated(IsolatedTool::new( + executable, + tool.prefix.clone(), + tool.activation_scripts.clone(), + )), ExecutableTool::System(_) => ExecutableTool::System(SystemTool::new(executable)), } } @@ -106,7 +120,12 @@ impl ExecutableTool { /// Construct a new command that enables invocation of the tool. pub fn command(&self) -> std::process::Command { match self { - ExecutableTool::Isolated(tool) => std::process::Command::new(&tool.command), + ExecutableTool::Isolated(tool) => { + let mut cmd = std::process::Command::new(&tool.command); + cmd.envs(tool.activation_scripts.clone()); + + cmd + } ExecutableTool::System(tool) => std::process::Command::new(&tool.command), } } diff --git a/crates/pixi_consts/src/consts.rs b/crates/pixi_consts/src/consts.rs index 9bd989dfa..1c8cec6b5 100644 --- a/crates/pixi_consts/src/consts.rs +++ b/crates/pixi_consts/src/consts.rs @@ -30,6 +30,7 @@ pub const CONDA_META_DIR: &str = "conda-meta"; pub const PYPI_CACHE_DIR: &str = "uv-cache"; pub const CONDA_PYPI_MAPPING_CACHE_DIR: &str = "conda-pypi-mapping"; pub const CACHED_ENVS_DIR: &str = "cached-envs-v0"; +pub const CACHED_BUILD_ENVS_DIR: &str = "cached-build-envs-v0"; pub const CONDA_INSTALLER: &str = "conda"; diff --git a/crates/pixi_manifest/src/build.rs b/crates/pixi_manifest/src/build.rs index c77af1af8..b7d9ae117 100644 --- a/crates/pixi_manifest/src/build.rs +++ b/crates/pixi_manifest/src/build.rs @@ -1,8 +1,13 @@ //! Defines the build section for the pixi manifest. +use rattler_conda_types::Channel; +use rattler_conda_types::ChannelConfig; use rattler_conda_types::MatchSpec; +use rattler_conda_types::NamedChannelOrUrl; +use rattler_conda_types::ParseChannelError; use serde::{Deserialize, Serialize}; use serde_with::serde_as; use serde_with::DisplayFromStr; +use url::Url; /// A build section in the pixi manifest. /// that defines what backend is used to build the project. @@ -17,6 +22,25 @@ pub struct BuildSection { /// The command to start the build backend pub build_backend: String, + + /// The channels to use for fetching build tools + pub channels: Vec, +} + +impl BuildSection { + pub fn channels_url(&self, config: &ChannelConfig) -> Result, ParseChannelError> { + self.channels + .iter() + .map(|c| c.clone().into_base_url(config)) + .collect() + } + + pub fn channels(&self, config: &ChannelConfig) -> Result, ParseChannelError> { + self.channels + .iter() + .map(|c| c.clone().into_channel(config)) + .collect() + } } #[cfg(test)] diff --git a/examples/cpp-sdl/pixi.toml b/examples/cpp-sdl/pixi.toml index 8838306c6..0ba1465a5 100644 --- a/examples/cpp-sdl/pixi.toml +++ b/examples/cpp-sdl/pixi.toml @@ -7,6 +7,7 @@ platforms = ["win-64", "linux-64", "osx-arm64", "osx-64"] [build] build-backend = "pixi-build-cmake" +channels = ["https://prefix.dev/graf", "https://fast.prefix.dev/conda-forge"] dependencies = ["pixi-build-cmake"] [tasks.start] diff --git a/examples/flask-hello-world-pyproject/pyproject.toml b/examples/flask-hello-world-pyproject/pyproject.toml index c989d7412..69f2daac3 100644 --- a/examples/flask-hello-world-pyproject/pyproject.toml +++ b/examples/flask-hello-world-pyproject/pyproject.toml @@ -21,7 +21,9 @@ platforms = ["linux-64", "osx-arm64", "osx-64", "win-64"] flask-hello-world-pyproject = { path = ".", editable = true } [tool.pixi.dependencies] -flask = "2.*" +# flask = "2.*" +flask-hello-world-pyproject = { path = "." } + [tool.pixi.environments] default = { solve-group = "default" } @@ -35,3 +37,12 @@ test = "pytest -v tests/*" [tool.pixi.host-dependencies] hatchling = "*" + + +[tool.pixi.build] +build-backend = "pixi-build-python" +channels = [ + "https://repo.prefix.dev/graf", + "https://fast.prefix.dev/conda-forge", +] +dependencies = ["pixi-build-python"] diff --git a/examples/flask-hello-world/pixi.toml b/examples/flask-hello-world/pixi.toml index 18a5e98b7..2f3cf2fe1 100644 --- a/examples/flask-hello-world/pixi.toml +++ b/examples/flask-hello-world/pixi.toml @@ -11,3 +11,11 @@ start = "python -m flask run --port=5050" [dependencies] flask = "2.*" + +[build] +build-backend = "pixi-build-python" +channels = [ + "file:///Users/graf/projects/oss/pixi-build-backends/output", + "https://fast.prefix.dev/conda-forge", +] +dependencies = ["pixi-build-python"] diff --git a/src/build/mod.rs b/src/build/mod.rs index 7a224e12d..e744a9b88 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -14,7 +14,7 @@ use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; use chrono::Utc; use itertools::Itertools; use miette::Diagnostic; -use pixi_build_frontend::SetupRequest; +use pixi_build_frontend::{SetupRequest, ToolContext}; use pixi_build_types::{ procedures::{ conda_build::{CondaBuildParams, CondaOutputIdentifier}, @@ -27,9 +27,10 @@ use pixi_glob::{GlobHashKey, GlobModificationTime, GlobModificationTimeError}; use pixi_record::{InputHash, PinnedPathSpec, PinnedSourceSpec, SourceRecord}; use pixi_spec::SourceSpec; use rattler_conda_types::{ - ChannelConfig, GenericVirtualPackage, PackageRecord, Platform, RepoDataRecord, + Channel, ChannelConfig, GenericVirtualPackage, PackageRecord, Platform, RepoDataRecord, }; use rattler_digest::Sha256; +use reqwest_middleware::ClientWithMiddleware; use thiserror::Error; use tracing::instrument; use typed_path::{Utf8TypedPath, Utf8TypedPathBuf}; @@ -132,24 +133,30 @@ impl BuildContext { &self, source_spec: &SourceSpec, channels: &[Url], + build_channels: Vec, host_platform: Platform, host_virtual_packages: Vec, build_platform: Platform, build_virtual_packages: Vec, metadata_reporter: Arc, build_id: usize, + gateway_config: rattler_repodata_gateway::ChannelConfig, + client: ClientWithMiddleware, ) -> Result { let source = self.fetch_source(source_spec).await?; let records = self .extract_records( &source, channels, + build_channels, host_platform, host_virtual_packages, build_platform, build_virtual_packages, metadata_reporter.clone(), build_id, + gateway_config, + client, ) .await?; @@ -163,16 +170,20 @@ impl BuildContext { &self, source_spec: &SourceRecord, channels: &[Url], + build_channels: Vec, host_platform: Platform, host_virtual_packages: Vec, build_virtual_packages: Vec, build_reporter: Arc, build_id: usize, + authenticated_client: ClientWithMiddleware, + gateway_config: rattler_repodata_gateway::ChannelConfig, ) -> Result { let source_checkout = SourceCheckout { path: self.fetch_pinned_source(&source_spec.source).await?, pinned: source_spec.source.clone(), }; + eprintln!("passed build channels {:?}", build_channels); let (cached_build, entry) = self .build_cache @@ -240,10 +251,17 @@ impl BuildContext { } } + let tool_config = ToolContext::new( + gateway_config, + authenticated_client, + build_channels.to_vec(), + ); + // Instantiate a protocol for the source directory. let protocol = pixi_build_frontend::BuildFrontend::default() .with_channel_config(self.channel_config.clone()) .with_cache_dir(self.cache_dir.clone()) + .with_tool_config(tool_config) .setup_protocol(SetupRequest { source_dir: source_checkout.path.clone(), build_tool_override: Default::default(), @@ -427,12 +445,15 @@ impl BuildContext { &self, source: &SourceCheckout, channels: &[Url], + build_channels: Vec, host_platform: Platform, host_virtual_packages: Vec, build_platform: Platform, build_virtual_packages: Vec, metadata_reporter: Arc, build_id: usize, + gateway_config: rattler_repodata_gateway::ChannelConfig, + client: ClientWithMiddleware, ) -> Result, BuildError> { let (cached_metadata, cache_entry) = self .source_metadata_cache @@ -478,10 +499,13 @@ impl BuildContext { )); } } + // tool config + let tool_config = ToolContext::new(gateway_config, client, build_channels); // Instantiate a protocol for the source directory. let protocol = pixi_build_frontend::BuildFrontend::default() .with_channel_config(self.channel_config.clone()) + .with_tool_config(tool_config) .setup_protocol(SetupRequest { source_dir: source.path.clone(), build_tool_override: Default::default(), diff --git a/src/cli/build.rs b/src/cli/build.rs index 4bb03957a..e7ac4e82d 100644 --- a/src/cli/build.rs +++ b/src/cli/build.rs @@ -84,8 +84,22 @@ pub async fn execute(args: Args) -> miette::Result<()> { // Instantiate a protocol for the source directory. let channel_config = project.channel_config(); + let channels = project + .manifest() + .build_section() + .ok_or_else(|| miette::miette!("no build section found in the manifest"))? + .channels(&channel_config) + .into_diagnostic()?; + + let tool_config = pixi_build_frontend::ToolContext { + gateway_config: project.config().into(), + client: project.authenticated_client().clone(), + channels, + }; + let protocol = pixi_build_frontend::BuildFrontend::default() .with_channel_config(channel_config.clone()) + .with_tool_config(tool_config) .setup_protocol(SetupRequest { source_dir: project.root().to_path_buf(), build_tool_override: Default::default(), @@ -94,6 +108,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { .await .into_diagnostic() .wrap_err("unable to setup the build-backend to build the project")?; + // Construct a temporary directory to build the package in. This path is also // automatically removed after the build finishes. let pixi_dir = &project.pixi_dir(); diff --git a/src/environment.rs b/src/environment.rs index fb144ac96..f8dfd0f6d 100644 --- a/src/environment.rs +++ b/src/environment.rs @@ -23,9 +23,10 @@ use rattler::{ install::{DefaultProgressFormatter, IndicatifReporter, Installer, PythonInfo, Transaction}, package_cache::PackageCache, }; -use rattler_conda_types::{GenericVirtualPackage, Platform, PrefixRecord, RepoDataRecord}; +use rattler_conda_types::{Channel, GenericVirtualPackage, Platform, PrefixRecord, RepoDataRecord}; use rattler_lock::Package::{Conda, Pypi}; use rattler_lock::{PypiIndexes, PypiPackageData, PypiPackageEnvironmentData}; +use rattler_repodata_gateway::ChannelConfig; use reqwest_middleware::ClientWithMiddleware; use serde::{Deserialize, Serialize}; use std::{ @@ -726,14 +727,17 @@ pub async fn update_prefix_conda( pixi_records: Vec, virtual_packages: Vec, channels: Vec, + build_channels: Vec, platform: Platform, progress_bar_message: &str, progress_bar_prefix: &str, io_concurrency_limit: Arc, build_context: BuildContext, + gateway_config: rattler_repodata_gateway::ChannelConfig, ) -> miette::Result { // Try to increase the rlimit to a sensible value for installation. try_increase_rlimit_to_sensible(); + eprintln!("IN UPDATE PREFIX CONDA"); let (mut repodata_records, source_records): (Vec<_>, Vec<_>) = pixi_records .into_iter() @@ -758,17 +762,27 @@ pub async fn update_prefix_conda( let build_id = progress_reporter.associate(record.package_record.name.as_source()); let build_context = &build_context; let channels = &channels; + let build_channels = &build_channels; let virtual_packages = &virtual_packages; + let client = authenticated_client.clone(); + // TODO(nichita): I think gateway_config should implement Clone + let gateway_config = ChannelConfig { + default: gateway_config.default.clone(), + per_channel: gateway_config.per_channel.clone(), + }; async move { build_context .build_source_record( &record, channels, + build_channels.to_vec(), platform, virtual_packages.clone(), virtual_packages.clone(), progress_reporter.clone(), build_id, + client, + gateway_config, ) .await } diff --git a/src/lock_file/update.rs b/src/lock_file/update.rs index 9e7669d39..f8e721928 100644 --- a/src/lock_file/update.rs +++ b/src/lock_file/update.rs @@ -23,8 +23,9 @@ use rattler_conda_types::{ Arch, Channel, GenericVirtualPackage, MatchSpec, ParseStrictness, Platform, RepoDataRecord, }; use rattler_lock::{LockFile, PypiIndexes, PypiPackageData, PypiPackageEnvironmentData}; -use rattler_repodata_gateway::{Gateway, RepoData}; +use rattler_repodata_gateway::{ChannelConfig, Gateway, RepoData}; use rattler_solve::ChannelPriority; +use reqwest_middleware::ClientWithMiddleware; use std::cmp::PartialEq; use std::{ collections::{HashMap, HashSet}, @@ -39,6 +40,7 @@ use tokio::sync::Semaphore; use tracing::Instrument; use uv_normalize::ExtraName; +use crate::cli::config; use crate::environment::{read_environment_file, LockedEnvironmentHash}; use crate::lock_file::reporter::{GatewayProgressReporter, SolveProgressBar}; use crate::lock_file::PypiRecord; @@ -364,6 +366,16 @@ impl<'p> LockFileDerivedData<'p> { .channel_urls(&self.project.channel_config()) .into_diagnostic()?; + let build_dep_channel_urls = environment + .project() + .manifest() + .build_section() + .ok_or_else(|| miette::miette!("No build section defined here"))? + .channels(&self.project.channel_config()) + .into_diagnostic()?; + + eprintln!("build dep channel urls {:?}", build_dep_channel_urls); + // Update the prefix with conda packages. let has_existing_packages = !installed_packages.is_empty(); let env_name = GroupedEnvironmentName::Environment(environment.name().clone()); @@ -379,6 +391,7 @@ impl<'p> LockFileDerivedData<'p> { .map(GenericVirtualPackage::from) .collect(), channel_urls, + build_dep_channel_urls, platform, &format!( "{} environment '{}'", @@ -392,6 +405,7 @@ impl<'p> LockFileDerivedData<'p> { "", self.io_concurrency_limit.clone().into(), self.build_context.clone(), + environment.project().config().into(), ) .await?; @@ -1125,7 +1139,7 @@ impl<'p> UpdateContext<'p> { project.repodata_gateway().clone(), platform, self.conda_solve_semaphore.clone(), - project.client().clone(), + project.authenticated_client().clone(), channel_priority, self.build_context.clone(), ) @@ -1654,7 +1668,7 @@ async fn spawn_solve_conda_environment_task( repodata_gateway: Gateway, platform: Platform, concurrency_semaphore: Arc, - client: reqwest::Client, + client: ClientWithMiddleware, channel_priority: ChannelPriority, build_context: BuildContext, ) -> miette::Result { @@ -1679,6 +1693,16 @@ async fn spawn_solve_conda_environment_task( // Get the channel configuration let channel_config = group.project().channel_config(); + let config = group.project().config().clone(); + + let build_channels = group + .project() + .manifest() + .build_section() + .ok_or_else(|| miette::miette!("build section not found"))? + .channels(&channel_config) + .into_diagnostic()?; + tokio::spawn( async move { // Acquire a permit before we are allowed to solve the environment. @@ -1713,6 +1737,9 @@ async fn spawn_solve_conda_environment_task( .collect::, _>>() .into_diagnostic()?; + let build_channels = &build_channels; + let config = &config; + let mut metadata_progress = None; let mut source_match_specs = Vec::new(); let source_futures = FuturesUnordered::new(); @@ -1729,12 +1756,15 @@ async fn spawn_solve_conda_environment_task( .extract_source_metadata( source_spec, &channel_urls, + build_channels.clone(), platform, virtual_packages.clone(), platform, virtual_packages.clone(), metadata_reporter.clone(), build_id, + config.into(), + client.clone(), ) .map_err(|e| { Report::new(e).wrap_err(format!( @@ -2182,11 +2212,20 @@ async fn spawn_create_prefix_task( ) -> miette::Result { let group_name = group.name().clone(); let prefix = group.prefix(); + let config = group.project().config(); let client = group.project().authenticated_client().clone(); let channels = group .channel_urls(&group.project().channel_config()) .into_diagnostic()?; + let build_channels = group + .project() + .manifest() + .build_section() + .ok_or_else(|| miette::miette!("build section is missing"))? + .channels(&group.project().channel_config()) + .into_diagnostic()?; + // Spawn a task to determine the currently installed packages. let installed_packages_future = tokio::spawn({ let prefix = prefix.clone(); @@ -2207,6 +2246,7 @@ async fn spawn_create_prefix_task( // Spawn a background task to update the prefix let (python_status, duration) = tokio::spawn({ let prefix = prefix.clone(); + let config = config.clone(); let group_name = group_name.clone(); async move { let start = Instant::now(); @@ -2219,6 +2259,7 @@ async fn spawn_create_prefix_task( pixi_records.records.clone(), build_virtual_packages, channels, + build_channels, Platform::current(), &format!( "{} python environment to solve pypi packages for '{}'", @@ -2232,6 +2273,7 @@ async fn spawn_create_prefix_task( " ", io_concurrency_limit.into(), build_context, + config.into(), ) .await?; let end = Instant::now();