Skip to content

Commit

Permalink
feat: Implement PEP735 support (prefix-dev#2448)
Browse files Browse the repository at this point in the history
  • Loading branch information
@olivier-lacroix
olivier-lacroix authored Nov 18, 2024
1 parent fc90191 commit 3cda60d
Show file tree
Hide file tree
Showing 14 changed files with 300 additions and 145 deletions.
9 changes: 9 additions & 0 deletions crates/pixi_manifest/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,15 @@ pub enum DependencyOverwriteBehavior {
Error,
}

pub enum PypiDependencyLocation {
// The [pypi-dependencies] or [tool.pixi.pypi-dependencies] table
Pixi,
// The [project.optional-dependencies] table in a 'pyproject.toml' manifest
OptionalDependencies,
// The [dependency-groups] table in a 'pyproject.toml' manifest
DependencyGroups,
}

/// Converts an array of Platforms to a non-empty Vec of Option<Platform>
fn to_options(platforms: &[Platform]) -> Vec<Option<Platform>> {
match platforms.is_empty() {
Expand Down
6 changes: 4 additions & 2 deletions crates/pixi_manifest/src/manifests/manifest.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ use crate::{
pypi::PyPiPackageName,
pyproject::PyProjectManifest,
to_options, DependencyOverwriteBehavior, Environment, EnvironmentName, Feature, FeatureName,
GetFeatureError, ParsedManifest, PrioritizedChannel, SpecType, Target, TargetSelector, Task,
TaskName,
GetFeatureError, ParsedManifest, PrioritizedChannel, PypiDependencyLocation, SpecType, Target,
TargetSelector, Task, TaskName,
};

#[derive(Debug, Clone)]
Expand Down Expand Up @@ -384,6 +384,7 @@ impl Manifest {
feature_name: &FeatureName,
editable: Option<bool>,
overwrite_behavior: DependencyOverwriteBehavior,
location: &Option<PypiDependencyLocation>,
) -> miette::Result<bool> {
let mut any_added = false;
for platform in crate::to_options(platforms) {
Expand All @@ -398,6 +399,7 @@ impl Manifest {
platform,
feature_name,
editable,
location,
)?;
any_added = true;
}
Expand Down
151 changes: 95 additions & 56 deletions crates/pixi_manifest/src/manifests/project.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use rattler_conda_types::{PackageName, Platform};
use toml_edit::{value, Array, Item, Table, Value};

use super::TomlManifest;
use crate::PypiDependencyLocation;
use crate::{consts, error::TomlError, pypi::PyPiPackageName, PyPiRequirement};
use crate::{consts::PYPROJECT_PIXI_PREFIX, FeatureName, SpecType, Task};

Expand Down Expand Up @@ -188,30 +189,43 @@ impl ManifestSource {
) -> Result<(), TomlError> {
// For 'pyproject.toml' manifest, try and remove the dependency from native
// arrays
let array = match self {
let remove_requirement =
|source: &mut ManifestSource, table, array_name| -> Result<(), TomlError> {
let array = source.manifest().get_toml_array(table, array_name)?;
if let Some(array) = array {
array.retain(|x| {
let req: pep508_rs::Requirement = x
.as_str()
.unwrap_or("")
.parse()
.expect("should be a valid pep508 dependency");
let name = PyPiPackageName::from_normalized(req.name);
name != *dep
});
if array.is_empty() {
source
.manifest()
.get_or_insert_nested_table(table)?
.remove(array_name);
}
}
Ok(())
};

match self {
ManifestSource::PyProjectToml(_) if feature_name.is_default() => {
self.manifest().get_toml_array("project", "dependencies")?
remove_requirement(self, "project", "dependencies")?;
}
ManifestSource::PyProjectToml(_) => {
let name = feature_name.to_string();
remove_requirement(self, "project.optional-dependencies", &name)?;
remove_requirement(self, "dependency-groups", &name)?;
}
ManifestSource::PyProjectToml(_) => self
.manifest()
.get_toml_array("project.optional-dependencies", &feature_name.to_string())?,
_ => None,
_ => (),
};
if let Some(array) = array {
array.retain(|x| {
let req: pep508_rs::Requirement = x
.as_str()
.unwrap_or("")
.parse()
.expect("should be a valid pep508 dependency");
let name = PyPiPackageName::from_normalized(req.name);
name != *dep
});
}

// For both 'pyproject.toml' and 'pixi.toml' manifest,
// try and remove the dependency from pixi native tables

let table_name = TableName::new()
.with_prefix(self.table_prefix())
.with_feature_name(Some(feature_name))
Expand Down Expand Up @@ -285,47 +299,72 @@ impl ManifestSource {
platform: Option<Platform>,
feature_name: &FeatureName,
editable: Option<bool>,
location: &Option<PypiDependencyLocation>,
) -> Result<(), TomlError> {
match self {
ManifestSource::PyProjectToml(_) => {
// Pypi dependencies can be stored in different places
// so we remove any potential dependency of the same name before adding it back
self.remove_pypi_dependency(
&PyPiPackageName::from_normalized(requirement.name.clone()),
platform,
feature_name,
)?;
if let FeatureName::Named(name) = feature_name {
self.manifest()
.get_or_insert_toml_array("project.optional-dependencies", name)?
.push(requirement.to_string())
} else {
self.manifest()
.get_or_insert_toml_array("project", "dependencies")?
.push(requirement.to_string())
}
}
ManifestSource::PixiToml(_) => {
let mut pypi_requirement =
PyPiRequirement::try_from(requirement.clone()).map_err(Box::new)?;
if let Some(editable) = editable {
pypi_requirement.set_editable(editable);
}
// Pypi dependencies can be stored in different places in pyproject.toml manifests
// so we remove any potential dependency of the same name before adding it back
if matches!(self, ManifestSource::PyProjectToml(_)) {
self.remove_pypi_dependency(
&PyPiPackageName::from_normalized(requirement.name.clone()),
platform,
feature_name,
)?;
}

let dependency_table = TableName::new()
.with_prefix(self.table_prefix())
.with_platform(platform.as_ref())
.with_feature_name(Some(feature_name))
.with_table(Some(consts::PYPI_DEPENDENCIES));

self.manifest()
.get_or_insert_nested_table(dependency_table.to_string().as_str())?
.insert(
requirement.name.as_ref(),
Item::Value(pypi_requirement.into()),
);
// The '[pypi-dependencies]' or '[tool.pixi.pypi-dependencies]' table is selected
// - For 'pixi.toml' manifests where it is the only choice
// - When explicitly requested
// - When a specific platform is requested, as markers are not supported (https://github.com/prefix-dev/pixi/issues/2149)
// - When an editable install is requested
if matches!(self, ManifestSource::PixiToml(_))
|| matches!(location, Some(PypiDependencyLocation::Pixi))
|| platform.is_some()
|| editable.is_some_and(|e| e)
{
let mut pypi_requirement =
PyPiRequirement::try_from(requirement.clone()).map_err(Box::new)?;
if let Some(editable) = editable {
pypi_requirement.set_editable(editable);
}
};

let dependency_table = TableName::new()
.with_prefix(self.table_prefix())
.with_platform(platform.as_ref())
.with_feature_name(Some(feature_name))
.with_table(Some(consts::PYPI_DEPENDENCIES));

self.manifest()
.get_or_insert_nested_table(dependency_table.to_string().as_str())?
.insert(
requirement.name.as_ref(),
Item::Value(pypi_requirement.into()),
);
return Ok(());
}

// Otherwise:
// - the [project.dependencies] array is selected for the default feature
// - the [dependency-groups.feature_name] array is selected unless
// - optional-dependencies is explicitly requested as location
let add_requirement =
|source: &mut ManifestSource, table, array| -> Result<(), TomlError> {
source
.manifest()
.get_or_insert_toml_array(table, array)?
.push(requirement.to_string());
Ok(())
};
if feature_name.is_default() {
add_requirement(self, "project", "dependencies")?
} else if matches!(location, Some(PypiDependencyLocation::OptionalDependencies)) {
add_requirement(
self,
"project.optional-dependencies",
&feature_name.to_string(),
)?
} else {
add_requirement(self, "dependency-groups", &feature_name.to_string())?
}
Ok(())
}

Expand Down
105 changes: 64 additions & 41 deletions crates/pixi_manifest/src/pyproject.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use miette::{Diagnostic, IntoDiagnostic, Report, WrapErr};
use pep440_rs::VersionSpecifiers;
use pep508_rs::Requirement;
use pixi_spec::PixiSpec;
use pyproject_toml::{self, Project};
use pyproject_toml::{self, pep735_resolve::Pep735Error, Contact, Project};
use rattler_conda_types::{PackageName, ParseStrictness::Lenient, VersionSpec};
use serde::Deserialize;
use thiserror::Error;
Expand Down Expand Up @@ -165,11 +165,10 @@ impl PyProjectManifest {
return Some(
pyproject_authors
.iter()
.filter_map(|contact| match (contact.name(), contact.email()) {
(Some(name), Some(email)) => Some(format!("{} <{}>", name, email)),
(Some(name), None) => Some(name.to_string()),
(None, Some(email)) => Some(email.to_string()),
(None, None) => None,
.map(|contact| match contact {
Contact::NameEmail { name, email } => format!("{} <{}>", name, email),
Contact::Name { name } => name.clone(),
Contact::Email { email } => email.clone(),
})
.collect(),
);
Expand All @@ -192,15 +191,19 @@ impl PyProjectManifest {
self.project().and_then(|p| p.optional_dependencies.clone())
}

/// Builds a list of pixi environments from pyproject groups of extra
/// dependencies:
/// - one environment is created per group of extra, with the same name as
/// the group of extra
/// - each environment includes the feature of the same name as the group
/// of extra
/// Returns dependency groups from the `[dependency-groups]` table
fn dependency_groups(&self) -> Option<Result<IndexMap<String, Vec<Requirement>>, Pep735Error>> {
self.dependency_groups.as_ref().map(|dg| dg.resolve())
}

/// Builds a list of pixi environments from pyproject groups of optional
/// dependencies and/or dependency groups:
/// - one environment is created per group with the same name
/// - each environment includes the feature of the same name
/// - it will also include other features inferred from any self references
/// to other groups of extras
pub fn environments_from_extras(&self) -> HashMap<String, Vec<String>> {
/// to other groups of optional dependencies (but won't for dependency groups,
/// as recursion between groups is resolved upstream)
pub fn environments_from_extras(&self) -> Result<HashMap<String, Vec<String>>, Pep735Error> {
let mut environments = HashMap::new();
if let Some(extras) = self.optional_dependencies() {
let pname = self.package_name();
Expand All @@ -218,14 +221,27 @@ impl PyProjectManifest {
environments.insert(extra.replace('_', "-").clone(), features);
}
}
environments

if let Some(groups) = self.dependency_groups().transpose()? {
for group in groups.into_keys() {
let normalised = group.replace('_', "-");
// Nothing to do if a group of optional dependencies has the same name as the dependency group
if !environments.contains_key(&normalised) {
environments.insert(normalised.clone(), vec![normalised]);
}
}
}

Ok(environments)
}
}

#[derive(Debug, Error, Diagnostic)]
pub enum PyProjectToManifestError {
#[error("Unsupported pep508 requirement: '{0}'")]
DependencyError(Requirement, #[source] DependencyError),
#[error(transparent)]
DependencyGroupError(#[from] Pep735Error),
}

impl TryFrom<PyProjectManifest> for ParsedManifest {
Expand Down Expand Up @@ -289,32 +305,37 @@ impl TryFrom<PyProjectManifest> for ParsedManifest {
}
}

// For each extra group, create a feature of the same name if it does not exist,
// and add pypi dependencies from project.optional-dependencies,
// filtering out self-references
if let Some(extras) = item.optional_dependencies() {
let project_name = item.package_name();
for (extra, reqs) in extras {
let feature_name = FeatureName::Named(extra.to_string());
let target = manifest
.features
.entry(feature_name.clone())
.or_insert_with(move || Feature::new(feature_name))
.targets
.default_mut();
for requirement in reqs.iter() {
// filter out any self references in groups of extra dependencies
if project_name.as_ref() != Some(&requirement.name) {
target
.try_add_pep508_dependency(
requirement,
None,
DependencyOverwriteBehavior::Error,
)
.map_err(|err| {
PyProjectToManifestError::DependencyError(requirement.clone(), err)
})?;
}
// Define an iterator over both optional dependencies and dependency groups
let groups = item
.optional_dependencies()
.into_iter()
.chain(item.dependency_groups().transpose()?)
.flat_map(|map| map.into_iter());

// For each group of optional dependency or dependency group,
// create a feature of the same name if it does not exist,
// and add pypi dependencies, filtering out self-references in optional dependencies
let project_name = item.package_name();
for (group, reqs) in groups {
let feature_name = FeatureName::Named(group.to_string());
let target = manifest
.features
.entry(feature_name.clone())
.or_insert_with(move || Feature::new(feature_name))
.targets
.default_mut();
for requirement in reqs.iter() {
// filter out any self references in groups of extra dependencies
if project_name.as_ref() != Some(&requirement.name) {
target
.try_add_pep508_dependency(
requirement,
None,
DependencyOverwriteBehavior::Error,
)
.map_err(|err| {
PyProjectToManifestError::DependencyError(requirement.clone(), err)
})?;
}
}
}
Expand Down Expand Up @@ -534,6 +555,7 @@ mod tests {
&FeatureName::Default,
None,
DependencyOverwriteBehavior::Overwrite,
&None,
)
.unwrap();

Expand All @@ -557,6 +579,7 @@ mod tests {
&FeatureName::Named("test".to_string()),
None,
DependencyOverwriteBehavior::Overwrite,
&None,
)
.unwrap();
assert!(manifest
Expand Down
Loading

0 comments on commit 3cda60d

Please sign in to comment.