Merge remote-tracking branch 'upstream/main' into skip_test_if_emulate

docs/authentication_and_upload.md

@@ -56,8 +56,48 @@ authenticate with the server.
 
 ### prefix.dev
 
-To upload to [prefix.dev](https://prefix.dev), you need to have an account and a
-token. You can create a token in the settings of your account. The token is used
+#### Trusted publishing via OIDC
+
+`rattler-build` supports authentication with https://prefix.dev through OIDC with GitHub Actions.
+An API key is no longer required, rattler-build can manage the complete authentication workflow for you.
+You only have to set up a specific repository and workflow under "Trusted Publishers" on prefix.dev.
+
+![Trusted Publisher](assets/trusted_publisher.png)
+
+
+Here you can find an example GitHub Actions workflow
+
+```yaml title=".github/workflows/build.yml"
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build conda package
+        uses: prefix-dev/[email protected]
+
+      - name: Upload all packages
+        shell: bash
+        run: |
+          shopt -s nullglob
+          EXIT_CODE=0
+          for pkg in $(find output -type f \( -name "*.conda" -o -name "*.tar.bz2" \) ); do
+            if ! rattler-build upload prefix -c my-channel "${pkg}"; then
+              EXIT_CODE=1
+            fi
+          done
+          exit $EXIT_CODE
+```
+
+
+#### Token
+
+To upload to [prefix.dev](https://prefix.dev), you need to have an account.
+You can then create a token in the settings of your account. The token is used
 to authenticate the upload.
 
 ```bash

docs/reference/cli.md

@@ -261,6 +261,11 @@ e.g. `tar-bz2:<number>` (from 1 to 9) or `conda:<number>` (from -7 to
 			Skip packages that already exist in any channel
 
 
+- `--noarch-build-platform <NOARCH_BUILD_PLATFORM>`
+
+	Define a "noarch platform" for which the noarch packages will be built for. The noarch builds will be skipped on the other platforms
+
+
 
 
 

docs/tutorials/javascript.md

@@ -45,4 +45,4 @@ requirements:
 tests:
   - script:
     - bibtex-tidy --version
-```
+```

scripts/activate.sh

@@ -9,4 +9,3 @@ export CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS="-C link-arg=-Wl,-rpath,$CONDA
 export CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS="-C link-arg=-Wl,-rpath,$CONDA_PREFIX/lib"
 
 export RATTLER_BUILD_PATH="$PIXI_PROJECT_ROOT/target-pixi/release/rattler-build"
-

src/lib.rs

@@ -134,6 +134,7 @@ pub fn get_tool_config(
         .with_zstd_repodata_enabled(args.common.use_zstd)
         .with_bz2_repodata_enabled(args.common.use_zstd)
         .with_skip_existing(args.skip_existing)
+        .with_noarch_build_platform(args.noarch_build_platform)
         .finish())
 }
 
@@ -151,15 +152,6 @@ pub async fn get_build_output(
     if output_dir.exists() {
         output_dir = canonicalize(&output_dir).into_diagnostic()?;
     }
-    if output_dir.starts_with(
-        recipe_path
-            .parent()
-            .expect("Could not get parent of recipe"),
-    ) {
-        return Err(miette::miette!(
-            "The output directory cannot be in the recipe directory.\nThe current output directory is: {}\nSelect a different output directory with the --output-dir option or set the CONDA_BLD_PATH environment variable"
-        , output_dir.to_string_lossy()));
-    }
 
     let recipe_text = fs::read_to_string(recipe_path).into_diagnostic()?;
 
@@ -419,6 +411,35 @@ pub async fn run_build_from_args(
     Ok(())
 }
 
+/// Check if the noarch builds should be skipped because the noarch platform has been set
+pub async fn skip_noarch(
+    mut outputs: Vec<Output>,
+    tool_configuration: &tool_configuration::Configuration,
+) -> miette::Result<Vec<Output>> {
+    if let Some(noarch_build_platform) = tool_configuration.noarch_build_platform {
+        outputs.retain(|output| {
+            // Skip the build if:
+            // - target_platform is "noarch"
+            // and
+            // - build_platform != noarch_build_platform
+            let should_skip = output.build_configuration.target_platform == Platform::NoArch
+                && output.build_configuration.build_platform.platform != noarch_build_platform;
+
+            if should_skip {
+                // The identifier should always be set at this point
+                tracing::info!(
+                    "Skipping build because noarch_build_platform is set to {} for {}",
+                    noarch_build_platform,
+                    output.identifier()
+                );
+            }
+            !should_skip
+        });
+    }
+
+    Ok(outputs)
+}
+
 /// Runs test.
 pub async fn run_test_from_args(
     args: TestOpts,

src/main.rs

@@ -11,8 +11,8 @@ use rattler_build::{
     console_utils::init_logging,
     get_build_output, get_recipe_path, get_tool_config,
     opt::{App, ShellCompletion, SubCommands},
-    rebuild_from_args, run_build_from_args, run_test_from_args, sort_build_outputs_topologically,
-    upload_from_args,
+    rebuild_from_args, run_build_from_args, run_test_from_args, skip_noarch,
+    sort_build_outputs_topologically, upload_from_args,
 };
 use tempfile::tempdir;
 
@@ -133,6 +133,9 @@ async fn main() -> miette::Result<()> {
                     return Ok(());
                 }
 
+                // Skip noarch builds before the topological sort
+                outputs = skip_noarch(outputs, &tool_config).await?;
+
                 sort_build_outputs_topologically(&mut outputs, build_args.up_to.as_deref())?;
                 run_build_from_args(outputs, tool_config).await?;
             }

src/opt.rs

@@ -365,6 +365,11 @@ pub struct BuildOpts {
     )]
     pub skip_existing: SkipExisting,
 
+    /// Define a "noarch platform" for which the noarch packages will be built
+    /// for. The noarch builds will be skipped on the other platforms.
+    #[arg(long, default_value = None, help_heading = "Modifying result")]
+    pub noarch_build_platform: Option<Platform>,
+
     /// Extra metadata to include in about.json
     #[arg(long, value_parser = parse_key_val)]
     pub extra_meta: Option<Vec<(String, Value)>>,

src/packaging.rs

@@ -22,7 +22,14 @@ mod metadata;
 pub use file_finder::{content_type, Files, TempFiles};
 pub use metadata::{contains_prefix_binary, contains_prefix_text, create_prefix_placeholder};
 
-use crate::{metadata::Output, package_test::write_test_files, post_process, tool_configuration};
+use crate::{
+    metadata::Output,
+    package_test::write_test_files,
+    post_process,
+    recipe::parser::GlobVec,
+    source::{self, copy_dir},
+    tool_configuration,
+};
 
 #[allow(missing_docs)]
 #[derive(Debug, thiserror::Error)]
@@ -58,7 +65,7 @@ pub enum PackagingError {
     RelinkError(#[from] crate::post_process::relink::RelinkError),
 
     #[error(transparent)]
-    SourceError(#[from] crate::source::SourceError),
+    SourceError(#[from] source::SourceError),
 
     #[error("could not create python entry point: {0}")]
     CannotCreateEntryPoint(String),
@@ -90,7 +97,7 @@ fn copy_license_files(
         let licenses_folder = tmp_dir_path.join("info/licenses/");
         fs::create_dir_all(&licenses_folder)?;
 
-        let copy_dir = crate::source::copy_dir::CopyDir::new(
+        let copy_dir = copy_dir::CopyDir::new(
             &output.build_configuration.directories.recipe_dir,
             &licenses_folder,
         )
@@ -101,7 +108,7 @@ fn copy_license_files(
         let copied_files_recipe_dir = copy_dir.copied_paths();
         let any_include_matched_recipe_dir = copy_dir.any_include_glob_matched();
 
-        let copy_dir = crate::source::copy_dir::CopyDir::new(
+        let copy_dir = copy_dir::CopyDir::new(
             &output.build_configuration.directories.work_dir,
             &licenses_folder,
         )
@@ -139,8 +146,25 @@ fn write_recipe_folder(
     let recipe_folder = tmp_dir_path.join("info/recipe/");
     let recipe_dir = &output.build_configuration.directories.recipe_dir;
     let recipe_path = &output.build_configuration.directories.recipe_path;
+    let output_dir = &output.build_configuration.directories.output_dir;
+
+    let mut copy_builder = copy_dir::CopyDir::new(recipe_dir, &recipe_folder)
+        .use_gitignore(true)
+        .ignore_hidden_files(true);
+
+    // if the output dir is inside the same directory as the recipe, then we
+    // need to ignore the output dir when copying
+    if let Ok(ignore_output) = output_dir.strip_prefix(recipe_dir) {
+        tracing::info!(
+            "Ignoring output dir in recipe folder: {}",
+            output_dir.to_string_lossy()
+        );
+        let output_dir_glob = format!("{}/**", ignore_output.to_string_lossy());
+        let glob_vec = GlobVec::from_vec(vec![], Some(vec![&output_dir_glob]));
+        copy_builder = copy_builder.with_globvec(&glob_vec);
+    }
 
-    let copy_result = crate::source::copy_dir::CopyDir::new(recipe_dir, &recipe_folder).run()?;
+    let copy_result = copy_builder.run()?;
 
     let mut files = Vec::from(copy_result.copied_paths());
 

src/recipe/parser/glob_vec.rs

@@ -179,7 +179,6 @@ impl GlobVec {
     }
 
     /// Only used for testing
-    #[cfg(test)]
     pub fn from_vec(include: Vec<&str>, exclude: Option<Vec<&str>>) -> Self {
         let include_vec: Vec<Glob> = include
             .into_iter()

src/source/copy_dir.rs

@@ -107,8 +107,6 @@ pub(crate) fn copy_file(
 /// It uses the `ignore` crate to read the `.gitignore` file in the source directory and uses the globs
 /// to filter the files and directories to copy.
 ///
-/// The copy process also ignores hidden files and directories by default.
-///
 /// # Return
 ///
 /// The returned `Vec<PathBuf>` contains the paths of the copied files.
@@ -130,8 +128,11 @@ impl<'a> CopyDir<'a> {
             from_path,
             to_path,
             globvec: GlobVec::default(),
+            // use the gitignore file by default
             use_gitignore: false,
+            // use the global git ignore file by default
             use_git_global: false,
+            // include hidden files by default
             hidden: false,
             copy_options: CopyOptions::default(),
         }
@@ -154,7 +155,7 @@ impl<'a> CopyDir<'a> {
     }
 
     #[allow(unused)]
-    pub fn hidden(mut self, b: bool) -> Self {
+    pub fn ignore_hidden_files(mut self, b: bool) -> Self {
         self.hidden = b;
         self
     }

src/tool_configuration.rs

@@ -6,7 +6,7 @@ use std::{path::PathBuf, sync::Arc};
 use clap::ValueEnum;
 use minijinja::machinery::ast::Test;
 use rattler::package_cache::PackageCache;
-use rattler_conda_types::ChannelConfig;
+use rattler_conda_types::{ChannelConfig, Platform};
 use rattler_networking::{
     authentication_storage::{self, backends::file::FileStorageError},
     AuthenticationMiddleware, AuthenticationStorage,
@@ -69,6 +69,9 @@ pub struct Configuration {
     /// Whether to skip existing packages
     pub skip_existing: SkipExisting,
 
+    /// The noarch platform to use (noarch builds are skipped on other platforms)
+    pub noarch_build_platform: Option<Platform>,
+
     /// The channel configuration to use when parsing channels.
     pub channel_config: ChannelConfig,
 
@@ -130,6 +133,7 @@ pub struct ConfigurationBuilder {
     use_zstd: bool,
     use_bz2: bool,
     skip_existing: SkipExisting,
+    noarch_build_platform: Option<Platform>,
     channel_config: Option<ChannelConfig>,
     compression_threads: Option<u32>,
 }
@@ -153,6 +157,7 @@ impl ConfigurationBuilder {
             use_zstd: true,
             use_bz2: false,
             skip_existing: SkipExisting::None,
+            noarch_build_platform: None,
             channel_config: None,
             compression_threads: None,
         }
@@ -247,6 +252,14 @@ impl ConfigurationBuilder {
         }
     }
 
+    /// Define the noarch platform
+    pub fn with_noarch_build_platform(self, noarch_build_platform: Option<Platform>) -> Self {
+        Self {
+            noarch_build_platform,
+            ..self
+        }
+    }
+
     /// Construct a [`Configuration`] from the builder.
     pub fn finish(self) -> Configuration {
         let cache_dir = self.cache_dir.unwrap_or_else(|| {
@@ -284,6 +297,7 @@ impl ConfigurationBuilder {
             use_zstd: self.use_zstd,
             use_bz2: self.use_bz2,
             skip_existing: self.skip_existing,
+            noarch_build_platform: self.noarch_build_platform,
             channel_config,
             compression_threads: self.compression_threads,
             package_cache,

test-data/recipes/pin_subpackage/recipe.yaml

@@ -20,4 +20,4 @@ outputs:
       noarch: generic
     requirements:
       run:
-        - ${{ pin_subpackage(name, exact=true) }}
+        - ${{ pin_subpackage(name, exact=true) }}