diff --git a/docs/build/dependency_types.md b/docs/build/dependency_types.md new file mode 100644 index 000000000..23fb1ee9e --- /dev/null +++ b/docs/build/dependency_types.md @@ -0,0 +1,104 @@ +# Run, Host and Build Dependencies + +If you add a package to the [dependency table](../reference/pixi_manifest.md#dependencies) of a feature that dependency will be available in all environments that include that feature. +The dependencies of a package that is being built are a bit more granular. +Here you can see the three types of dependencies for a simple C++ package. + +```toml +--8<-- "docs/source_files/pixi_tomls/dependency_types.toml:dependencies" +``` + +Each dependency is used at a different step of the package building process. +`gxx` is used to build the package, `catch` will be linked into the package and `git` will be available during runtime. + +Let's delve deeper into the various types of package dependencies and their specific roles in the build process. + +### [Build Dependencies](../reference/pixi_manifest.md#build-dependencies) +!!! note "pixi-build-cmake" + When using the `pixi-build-cmake` backend you do not need to specify `cmake` or the compiler as a dependency. + The backend will install `cmake`, `ninja` and the C++ compilers by default. + +This table contains dependencies that are needed to build the project. +Different from dependencies and host-dependencies these packages are installed for the architecture of the build machine. +This enables cross-compiling from one machine architecture to another. + +Typical examples of build dependencies are: + +- Compilers are invoked on the build machine, but they generate code for the target machine. + If the project is cross-compiled, the architecture of the build and target machine might differ. +- `cmake` is invoked on the build machine to generate additional code- or project-files which are then include in the compilation process. + +!!! info + The _build_ target refers to the machine that will execute the build. + Programs and libraries installed by these dependencies will be executed on the build machine. + + For example, if you compile on a MacBook with an Apple Silicon chip but target Linux x86_64 then your *build* platform is `osx-arm64` and your *host* platform is `linux-64`. + +### [Host Dependencies](../reference/pixi_manifest.md#host-dependencies) + +Host dependencies are the dependencies needed during build/link time that are specific to the host machine. +The difference to build dependencies becomes for example important during cross compilation. +The compiler is a build dependency since it is specific to your machine. +In contrast, the libraries you link to are host dependencies since they are specific to the host machine. +Typical examples of host dependencies are: + +- Base interpreters: a Python package would list `python` here and an R package would list `mro-base` or `r-base`. +- Libraries your project links against like `openssl`, `rapidjson`, or `xtensor`. + +#### Python code +Because of the way building currently works, dependencies like `hatchling`,`pip`,`uv` etc. are host dependencies. +Otherwise, it would use the wrong python prefix during the build process. + +This is more of a technical limitation, and we are looking into ways to make this less of a hassle. +But for now, you will need to add these dependencies to the `host-dependencies` section. + +So as an example, say we want to use `hatchling` and `uv` as to build a python package. +You need to use, something like this in your manifest file: + +```toml +[host-dependencies] +hatchling = "*" +uv = "*" +``` + +#### Native code +When cross-compiling, you might need to specify host dependencies that should have the *target* machine architecture, and are used during the build process. +When linking a library, for example. +Let's recap an explanation that can be found here [A Master Guide To Linux Cross-Compiling](https://ruvi-d.medium.com/a-master-guide-to-linux-cross-compiling-b894bf909386) + +- *Build machine*: where the code is built. +- *Host machine*: where the built code runs. +- *Target machine*: where the binaries spit out by the built code runs. + +Lets say we are using a Linux PC (linux-64) to cross compile a CMake application called `Awesome` to run on a Linux ARM target machine (linux-aarch64). +We would get the following table: + +| Component | Type | Build | Host | Target | +|-----------|-------------|--------|--------|--------| +| GCC | Compiler | x86_64 | x86_64 | aarch64| +| CMake | Build tool | x86_64 | x86_64 | N/A | +| Awesome | Application | x86_64 | aarch64 | N/A | + +So if I need to use a library like SDL2, I would need to add it to the `host-dependencies` table. +As the machine running `Awesome` will have a different host architecture than the build architecture. + +Giving you something like this in your manifest file: + +```toml + # in our example these dependencies will use the aarch64 binaries +[host-dependencies] +sdl2 = "*" +``` + +#### Run-exports + +Conda packages, can define `run-exports`, that are dependencies that when specified in the `host-dependencies` section, will be implicitly be added to the `run-dependencies` section. +This is useful to avoid having to specify the same dependencies in both sections. +As most packages on conda-forge will have these `run-exports` defined. +When using something like `zlib`, you would only need to specify it in the `host-dependencies` section, and it will be used as a run-dependency automatically. + + +### [Dependencies (Run Dependencies)](../reference/pixi_manifest.md#dependencies) + +These are the dependencies that are required to when running the package, they are the most common dependencies. +And are what you would usually use in a `workspace`. diff --git a/docs/build/getting_started.md b/docs/build/getting_started.md new file mode 100644 index 000000000..3a69bfc8d --- /dev/null +++ b/docs/build/getting_started.md @@ -0,0 +1,51 @@ + +## Introduction + +Next to managing workflows and environments, pixi can also build packages. +This is useful for the following reasons: + +- Building and uploading a package to a conda channel +- Allowing users to directly depend on the source and build it automatically +- Managing multiple packages in a workspace + +We've been working to support these use-cases with the `build` feature in pixi. +The vision is to enable building of packages from source, for any language, on any platform. + + +!!! note "Known limitations" + Currently, the `build` feature has a number of limitations: + + 1. Limited set of [build-backends](https://github.com/prefix-dev/pixi-build-backends). + 2. Build-backends are probably missing a lot of parameters/features. + 3. Recursive source dependencies are not supported. ( source dependencies that have source dependencies ) + 4. Workspace dependencies cannot be inherited. + +## Setting up the Manifest +In this example, we are using `pixi-build-python` in order to build a Python package. +If the package itself has dependencies they need to be mentioned here. +The different kinds of dependencies are explained at the [dependency types chapter](dependency_types.md). + +This is what the `pixi.toml` file looks like for a simple Python package: +```toml +--8<-- "docs/source_files/pixi_tomls/simple_pixi_build.toml:all" +``` + +1. Specifies workspace properties like the name, channels, and platforms. This is currently an alias for `project`. +2. Since the build feature is still in preview, you have to add "pixi-build" to `workspace.preview`. +3. We need to add our package as dependency to the workspace. +4. In `package` you specify properties specific to the package you want to build. +5. Packages are built by using build backends. + By specifying `build-system.build-backend` and `build-system.channels` you determine which backend is used and from which channel it will be downloaded. +6. There are different build backends. + Pixi backends can describe how to build a conda package, for a certain language or build tool. + For example, `pixi-build-python`, allows building a Python package into a conda package. +7. `simple_python` uses `hatchling` as Python build backend so this needs to be mentioned in `host-dependencies`. + Read up on host-dependencies in the [Dependency Types](./dependency_types.md#host-dependencies) +8. Python PEP517 backends like `hatchling` know how to build a Python package. + So `hatchling` creates a Python package, and `pixi-build-python` turns the Python package into a conda package. + +## CLI Commands +Using the preview feature you can now build packages from source. + +- `pixi build` *has been addeded* and will build your source package into a `.conda` file. +- Other commands like `pixi install`, `pixi run` etc. automatically make use of the build feature. diff --git a/docs/reference/pixi_manifest.md b/docs/reference/pixi_manifest.md index d6bb7065e..006d1b72f 100644 --- a/docs/reference/pixi_manifest.md +++ b/docs/reference/pixi_manifest.md @@ -94,7 +94,7 @@ This should be a valid version based on the conda Version Spec. See the [version documentation](https://docs.rs/rattler_conda_types/latest/rattler_conda_types/struct.Version.html), for an explanation of what is allowed in a Version Spec. ```toml --8<-- "docs/source_files/pixi_tomls/main_pixi.toml:project_version" +--8<-- "docs/source_files/pixi_tomls/main_pixi.toml:project_version" ``` ### `authors` (optional) @@ -354,6 +354,8 @@ By default, `uv` and thus `pixi`, will stop at the first index on which a given The `index-strategy` only changes PyPI package resolution and not conda package resolution. ## The `dependencies` table(s) +??? info "Details regarding the dependencies" + For more detail regarding the dependency types, make sure to check the [Run, Host, Build](../build/dependency_types.md) dependency documentation. This section defines what dependencies you would like to use for your project. @@ -363,6 +365,7 @@ The default is `[dependencies]`, which are dependencies that are shared across p Dependencies are defined using a [VersionSpec](https://docs.rs/rattler_conda_types/latest/rattler_conda_types/version_spec/enum.VersionSpec.html). A `VersionSpec` combines a [Version](https://docs.rs/rattler_conda_types/latest/rattler_conda_types/struct.Version.html) with an optional operator. + Some examples are: ```toml @@ -403,6 +406,29 @@ rust = "1.72" pytorch-cpu = { version = "~=1.1", channel = "pytorch" } ``` + +### `host-dependencies` + +```toml +[host-dependencies] +python = "~=3.10.3" +``` +Typical examples of host dependencies are: + +- Base interpreters: a Python package would list `python` here and an R package would list `mro-base` or `r-base`. +- Libraries your project links against during compilation like `openssl`, `rapidjson`, or `xtensor`. + +### `build-dependencies` + +This table contains dependencies that are needed to build the project. +Different from `dependencies` and `host-dependencies` these packages are installed for the architecture of the _build_ machine. +This enables cross-compiling from one machine architecture to another. + +```toml +[build-dependencies] +cmake = "~=3.24" +``` + ### `pypi-dependencies` ??? info "Details regarding the PyPI integration" @@ -551,46 +577,6 @@ Sdists usually depend on system packages to be built, especially when compiling Think for example of Python SDL2 bindings depending on the C library: SDL2. To help built these dependencies we activate the conda environment that includes these pypi dependencies before resolving. This way when a source distribution depends on `gcc` for example, it's used from the conda environment instead of the system. - -### `host-dependencies` - -This table contains dependencies that are needed to build your project but which should not be included when your project is installed as part of another project. -In other words, these dependencies are available during the build but are no longer available when your project is installed. -Dependencies listed in this table are installed for the architecture of the target machine. - -```toml -[host-dependencies] -python = "~=3.10.3" -``` - -Typical examples of host dependencies are: - -- Base interpreters: a Python package would list `python` here and an R package would list `mro-base` or `r-base`. -- Libraries your project links against during compilation like `openssl`, `rapidjson`, or `xtensor`. - -### `build-dependencies` - -This table contains dependencies that are needed to build the project. -Different from `dependencies` and `host-dependencies` these packages are installed for the architecture of the _build_ machine. -This enables cross-compiling from one machine architecture to another. - -```toml -[build-dependencies] -cmake = "~=3.24" -``` - -Typical examples of build dependencies are: - -- Compilers are invoked on the build machine, but they generate code for the target machine. - If the project is cross-compiled, the architecture of the build and target machine might differ. -- `cmake` is invoked on the build machine to generate additional code- or project-files which are then include in the compilation process. - -!!! info - The _build_ target refers to the machine that will execute the build. - Programs and libraries installed by these dependencies will be executed on the build machine. - - For example, if you compile on a MacBook with an Apple Silicon chip but target Linux x86_64 then your *build* platform is `osx-arm64` and your *host* platform is `linux-64`. - ## The `activation` table The activation table is used for specialized activation operations that need to be run when the environment is activated. diff --git a/docs/source_files/pixi_tomls/dependency_types.toml b/docs/source_files/pixi_tomls/dependency_types.toml new file mode 100644 index 000000000..525342b68 --- /dev/null +++ b/docs/source_files/pixi_tomls/dependency_types.toml @@ -0,0 +1,26 @@ +[workspace] +channels = ["https://prefix.dev/conda-forge"] +platforms = ["win-64", "linux-64", "osx-arm64", "osx-64"] +preview = ["pixi-build"] + +[package] +name = "simple_cpp" +version = "0.1.0" + +[build-system] +build-backend = { name = "pixi-build-cmake", version = "*" } +channels = [ + "https://prefix.dev/pixi-build-backends", + "https://prefix.dev/conda-forge", +] +# --8<-- [start:dependencies] +[build-dependencies] +gxx = "*" + +[host-dependencies] +catch = "*" + +[run-dependencies] +git = "*" +# --8<-- [end:dependencies] +# simple_cpp = { path = "." } diff --git a/docs/source_files/pixi_tomls/simple_pixi_build.toml b/docs/source_files/pixi_tomls/simple_pixi_build.toml new file mode 100644 index 000000000..12d38c7c5 --- /dev/null +++ b/docs/source_files/pixi_tomls/simple_pixi_build.toml @@ -0,0 +1,34 @@ +# --8<-- [start:all] +# --8<-- [start:preview] +[workspace] # (1)! +preview = ["pixi-build"] # (2)! +# --8<-- [end:preview] +channels = ["https://prefix.dev/conda-forge"] +platforms = ["win-64", "linux-64", "osx-arm64", "osx-64"] + +# --8<-- [start:dependencies] +[dependencies] # (3)! +simple_python = { path = "." } +# --8<-- [end:dependencies] + +# --8<-- [start:package] +[package] # (4)! +name = "simple_python" +version = "0.1.0" +# --8<-- [end:package] + +# --8<-- [start:build-system] +[build-system] # (5)! +build-backend = { name = "pixi-build-python", version = "*" } # (6)! +channels = [ + "https://prefix.dev/pixi-build-backends", + "https://prefix.dev/conda-forge", +] +# --8<-- [end:build-system] + +# --8<-- [start:host-dependencies] +[host-dependencies] # (7)! +hatchling = "*" # (8)! +# --8<-- [end:host-dependencies] + +# --8<-- [start:all] diff --git a/mkdocs.yml b/mkdocs.yml index a9b1748d8..77d5b7d0e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -123,11 +123,14 @@ nav: - Lockfile: features/lockfile.md - System Requirements: features/system_requirements.md - Global Tools: features/global_tools.md + - Building Packages: + - Getting started: build/getting_started.md + - Dependency Types: build/dependency_types.md - Advanced: - Authentication: advanced/authentication.md - - Info Command: advanced/explain_info_command.md - Channel Logic: advanced/channel_priority.md - GitHub Actions: advanced/github_actions.md + - Info Command: advanced/explain_info_command.md - Updates using GitHub Actions: advanced/updates_github_actions.md - Production Deployment: advanced/production_deployment.md - Pyproject.toml: advanced/pyproject_toml.md