feat: add --test={skip|native|native-and-emulated} (#1190)

docs/reference/cli.md

@@ -219,12 +219,26 @@ e.g. `tar-bz2:<number>` (from 1 to 9) or `conda:<number>` (from -7 to
 
 - `--no-test`
 
-	Don't run the tests after building the package
+	Do not run tests after building (deprecated, use `--test=skip` instead)
 
 	- Default value: `false`
 	- Possible values: `true`, `false`
 
 
+- `--test <TEST>`
+
+	The strategy to use for running tests
+
+	- Default value: `native-and-emulated`
+	- Possible values:
+		- `skip`:
+			Skip the tests
+		- `native`:
+			Run the tests only if the build platform is the same as the host platform. Otherwise, skip the tests. If the target platform is noarch, the tests are always executed
+		- `native-and-emulated`:
+			Always run the tests
+
+
 - `--color-build-log`
 
 	Don't force colors in the output of the build script
@@ -335,7 +349,7 @@ These test files are written at "package creation time" and are part of the pack
 
 Rebuild a package from a package file instead of a recipe
 
-**Usage:** `rattler-build rebuild [OPTIONS] --package-file <PACKAGE_FILE>`
+**Usage:** `rattler-build rebuild [OPTIONS] --package-file <PACKAGE_FILE> --test <TEST>`
 
 ##### **Options:**
 
@@ -346,7 +360,7 @@ Rebuild a package from a package file instead of a recipe
 
 - `--no-test`
 
-	Do not run tests after building
+	Do not run tests after building (deprecated, use `--test=skip` instead)
 
 	- Default value: `false`
 	- Possible values: `true`, `false`
@@ -387,6 +401,19 @@ Rebuild a package from a package file instead of a recipe
 
 ###### **Modifying result**
 
+- `--test <TEST>`
+
+	The strategy to use for running tests
+
+	- Possible values:
+		- `skip`:
+			Skip the tests
+		- `native`:
+			Run the tests only if the build platform is the same as the host platform. Otherwise, skip the tests. If the target platform is noarch, the tests are always executed
+		- `native-and-emulated`:
+			Always run the tests
+
+
 - `--output-dir <OUTPUT_DIR>`
 
 	Output directory for build artifacts.

src/build.rs

@@ -3,16 +3,15 @@
 use std::{path::PathBuf, vec};
 
 use miette::{Context, IntoDiagnostic};
-use rattler_conda_types::{Channel, MatchSpec, ParseStrictness};
+use rattler_conda_types::{Channel, MatchSpec, ParseStrictness, Platform};
 use rattler_solve::{ChannelPriority, SolveStrategy};
 
 use crate::{
     metadata::{build_reindexed_channels, Output},
-    package_test,
-    package_test::TestConfiguration,
+    package_test::{self, TestConfiguration},
     recipe::parser::TestType,
     render::solver::load_repodatas,
-    tool_configuration,
+    tool_configuration::{self, TestStrategy},
 };
 
 /// Check if the build should be skipped because it already exists in any of the
@@ -160,8 +159,27 @@ pub async fn run_build(
         directories.clean().into_diagnostic()?;
     }
 
-    if tool_configuration.no_test {
-        tracing::info!("Skipping tests");
+    // Decide whether the tests should be skipped or not
+    let (skip_test, skip_test_reason) = match tool_configuration.test_strategy {
+        TestStrategy::Skip => (true, "the argument --test=skip was set".to_string()),
+        TestStrategy::Native => {
+            // Skip if `host_platform != build_platform` and `target_platform != noarch`
+            if output.build_configuration.target_platform != Platform::NoArch
+                && output.build_configuration.host_platform.platform
+                    != output.build_configuration.build_platform.platform
+            {
+                let reason = format!("the argument --test=native was set and the build is a cross-compilation (target_platform={}, build_platform={}, host_platform={})", output.build_configuration.target_platform, output.build_configuration.build_platform.platform, output.build_configuration.host_platform.platform);
+
+                (true, reason)
+            } else {
+                (false, "".to_string())
+            }
+        }
+        TestStrategy::NativeAndEmulated => (false, "".to_string()),
+    };
+
+    if skip_test {
+        tracing::info!("Skipping tests because {}", skip_test_reason);
     } else {
         package_test::run_test(
             &result,

src/lib.rs

@@ -131,6 +131,7 @@ pub fn get_tool_config(
         .with_compression_threads(args.compression_threads)
         .with_reqwest_client(client)
         .with_testing(!args.no_test)
+        .with_test_strategy(args.test)
         .with_zstd_repodata_enabled(args.common.use_zstd)
         .with_bz2_repodata_enabled(args.common.use_zstd)
         .with_skip_existing(args.skip_existing)
@@ -549,6 +550,7 @@ pub async fn rebuild_from_args(
                 .into_diagnostic()?,
         )
         .with_testing(!args.no_test)
+        .with_test_strategy(args.test)
         .with_zstd_repodata_enabled(args.common.use_zstd)
         .with_bz2_repodata_enabled(args.common.use_zstd)
         .finish();

src/opt.rs

@@ -15,7 +15,7 @@ use url::Url;
 use crate::recipe_generator::GenerateRecipeOpts;
 use crate::{
     console_utils::{Color, LogStyle},
-    tool_configuration::SkipExisting,
+    tool_configuration::{SkipExisting, TestStrategy},
 };
 
 /// Application subcommands.
@@ -339,10 +339,18 @@ pub struct BuildOpts {
     #[arg(long, help_heading = "Modifying result")]
     pub no_include_recipe: bool,
 
-    /// Don't run the tests after building the package
+    /// Do not run tests after building (deprecated, use `--test=skip` instead)
     #[arg(long, default_value = "false", help_heading = "Modifying result")]
     pub no_test: bool,
 
+    /// The strategy to use for running tests
+    #[arg(
+        long,
+        default_value = "native-and-emulated",
+        help_heading = "Modifying result"
+    )]
+    pub test: TestStrategy,
+
     /// Don't force colors in the output of the build script
     #[arg(long, default_value = "true", help_heading = "Modifying result")]
     pub color_build_log: bool,
@@ -420,10 +428,14 @@ pub struct RebuildOpts {
     #[arg(short, long)]
     pub package_file: PathBuf,
 
-    /// Do not run tests after building
+    /// Do not run tests after building (deprecated, use `--test=skip` instead)
     #[arg(long, default_value = "false")]
     pub no_test: bool,
 
+    /// The strategy to use for running tests
+    #[arg(long, help_heading = "Modifying result")]
+    pub test: TestStrategy,
+
     /// The number of threads to use for compression.
     #[clap(long, env = "RATTLER_COMPRESSION_THREADS")]
     pub compression_threads: Option<u32>,

src/tool_configuration.rs

@@ -29,6 +29,20 @@ pub enum SkipExisting {
     All,
 }
 
+/// Container for the CLI test strategy
+#[derive(Debug, Clone, Copy, ValueEnum, Default)]
+pub enum TestStrategy {
+    /// Skip the tests
+    Skip,
+    /// Run the tests only if the build platform is the same as the host platform.
+    /// Otherwise, skip the tests. If the target platform is noarch,
+    /// the tests are always executed.
+    Native,
+    /// Always run the tests
+    #[default]
+    NativeAndEmulated,
+}
+
 /// Global configuration for the build
 #[derive(Clone)]
 pub struct Configuration {
@@ -42,8 +56,8 @@ pub struct Configuration {
     /// is done
     pub no_clean: bool,
 
-    /// Whether to skip the test phase
-    pub no_test: bool,
+    /// The strategy to use for running tests
+    pub test_strategy: TestStrategy,
 
     /// Whether to use zstd
     pub use_zstd: bool,
@@ -115,6 +129,7 @@ pub struct ConfigurationBuilder {
     client: Option<ClientWithMiddleware>,
     no_clean: bool,
     no_test: bool,
+    test_strategy: TestStrategy,
     use_zstd: bool,
     use_bz2: bool,
     skip_existing: SkipExisting,
@@ -139,6 +154,7 @@ impl ConfigurationBuilder {
             client: None,
             no_clean: false,
             no_test: false,
+            test_strategy: TestStrategy::default(),
             use_zstd: true,
             use_bz2: false,
             skip_existing: SkipExisting::None,
@@ -221,6 +237,14 @@ impl ConfigurationBuilder {
         }
     }
 
+    /// Sets the test strategy to use for running tests.
+    pub fn with_test_strategy(self, test_strategy: TestStrategy) -> Self {
+        Self {
+            test_strategy,
+            ..self
+        }
+    }
+
     /// Whether downloading repodata as `.zst` files is enabled.
     pub fn with_zstd_repodata_enabled(self, zstd_repodata_enabled: bool) -> Self {
         Self {
@@ -274,11 +298,16 @@ impl ConfigurationBuilder {
             })
             .finish();
 
+        let test_strategy = match self.no_test {
+            true => TestStrategy::Skip,
+            false => self.test_strategy,
+        };
+
         Configuration {
             fancy_log_handler: self.fancy_log_handler.unwrap_or_default(),
             client,
             no_clean: self.no_clean,
-            no_test: self.no_test,
+            test_strategy,
             use_zstd: self.use_zstd,
             use_bz2: self.use_bz2,
             skip_existing: self.skip_existing,

test-data/recipes/test_strategy/recipe-noarch.yaml

@@ -0,0 +1,6 @@
+package:
+  name: testing-test-strategy
+  version: 0.0.1
+
+build:
+  noarch: generic

test-data/recipes/test_strategy/recipe.yaml

@@ -0,0 +1,3 @@
+package:
+  name: testing-test-strategy
+  version: 0.0.1

test/end-to-end/helpers.py

@@ -89,3 +89,19 @@ def get_extracted_package(folder: Path, glob="*.tar.bz2"):
     extract_path = folder / "extract" / package_without_extension
     extract(str(package_path), dest_dir=str(extract_path))
     return extract_path
+
+
+def check_build_output(
+    rattler_build: RattlerBuild,
+    capfd,
+    recipe_path,
+    output_path,
+    extra_args: list,
+    string_to_check: str,
+):
+    """Run a build and check the output for a specific string."""
+
+    rattler_build.build(recipe_path, output_path, extra_args=extra_args)
+    _, err = capfd.readouterr()
+    print(err)  # to debug in case it fails
+    assert string_to_check in err

test/end-to-end/test_simple.py

@@ -8,7 +8,8 @@
 import pytest
 import requests
 import yaml
-from helpers import RattlerBuild, get_extracted_package, get_package
+from helpers import (RattlerBuild, check_build_output, get_extracted_package,
+                     get_package)
 
 
 def test_functionality(rattler_build: RattlerBuild):
@@ -111,7 +112,7 @@ def variant_hash(variant):
 
 
 def test_pkg_hash(rattler_build: RattlerBuild, recipes: Path, tmp_path: Path):
-    rattler_build.build(recipes / "pkg_hash", tmp_path, extra_args=["--no-test"])
+    rattler_build.build(recipes / "pkg_hash", tmp_path, extra_args=["--test=skip"])
     pkg = get_package(tmp_path, "pkg_hash")
     expected_hash = variant_hash({"target_platform": host_subdir()})
     assert pkg.name.endswith(f"pkg_hash-1.0.0-{expected_hash}_my_pkg.tar.bz2")
@@ -1009,3 +1010,83 @@ def test_pin_subpackage(
     )
     pkg = get_extracted_package(tmp_path, "my.package-a")
     assert (pkg / "info/index.json").exists()
+
+
+def test_testing_strategy(
+    rattler_build: RattlerBuild,
+    recipes: Path,
+    tmp_path: Path,
+    capfd,
+):
+    # --test=skip
+    check_build_output(
+        rattler_build,
+        capfd,
+        recipe_path=recipes / "test_strategy" / "recipe.yaml",
+        output_path=tmp_path,
+        extra_args=["--test=skip"],
+        string_to_check="Skipping tests because the argument --test=skip was set",
+    )
+
+    # --test=native
+    check_build_output(
+        rattler_build,
+        capfd,
+        recipe_path=recipes / "test_strategy" / "recipe.yaml",
+        output_path=tmp_path,
+        extra_args=["--test=native"],
+        string_to_check="all tests passed!",
+    )
+
+    # --test=native and cross-compiling
+    check_build_output(
+        rattler_build,
+        capfd,
+        recipe_path=recipes / "test_strategy" / "recipe.yaml",
+        output_path=tmp_path,
+        extra_args=[
+            "--test=native",
+            "--target-platform=linux-64",
+            "--build-platform=osx-64",
+        ],
+        string_to_check="Skipping tests because the argument "
+        "--test=native was set and the build is a cross-compilation",
+    )
+
+    # --test=native-and-emulated
+    check_build_output(
+        rattler_build,
+        capfd,
+        recipe_path=recipes / "test_strategy" / "recipe.yaml",
+        output_path=tmp_path,
+        extra_args=["--test=native-and-emulated"],
+        string_to_check="all tests passed!",
+    )
+
+    #  --test=native-and-emulated and cross-compiling
+    check_build_output(
+        rattler_build,
+        capfd,
+        recipe_path=recipes / "test_strategy" / "recipe.yaml",
+        output_path=tmp_path,
+        extra_args=[
+            "--test=native-and-emulated",
+            "--target-platform=linux-64",
+            "--build-platform=osx-64",
+        ],
+        string_to_check="all tests passed!",
+    )
+
+    # --test=native and cross-compiling and noarch
+    check_build_output(
+        rattler_build,
+        capfd,
+        recipe_path=recipes / "test_strategy" / "recipe-noarch.yaml",
+        output_path=tmp_path,
+        extra_args=[
+            "--test=native",
+            "--target-platform=linux-64",
+            "--build-platform=osx-64",
+        ],
+        string_to_check="all tests passed!",
+    )