From c594701d6d370fe038ad3a5fb0b7596b3328803d Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 3 Jan 2024 12:02:55 +0100 Subject: [PATCH 1/2] docs: document the channel logic as an advanced document --- docs/advanced/channel_priority.md | 96 +++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 97 insertions(+) create mode 100644 docs/advanced/channel_priority.md diff --git a/docs/advanced/channel_priority.md b/docs/advanced/channel_priority.md new file mode 100644 index 000000000..a6ad27caa --- /dev/null +++ b/docs/advanced/channel_priority.md @@ -0,0 +1,96 @@ +All logic regarding the decision which dependencies can be installed from which channel is done by the instruction we give the solver. + +The actual code regarding this is in the [`rattler_solve`](https://github.com/mamba-org/rattler/blob/02e68c9539c6009cc1370fbf46dc69ca5361d12d/crates/rattler_solve/src/resolvo/mod.rs) crate. +This might however be hard to read. +Therefore, this document will continue with pseudocode. + +# Channel specific dependencies +When a user defines a channel per dependency, the solver needs to know the other channels are unusable for this dependency. +```toml +[project] +channels = ["conda-forge", "my-channel"] + +[dependencies] +packgex = { version = "*", channel = "my-channel" } +``` +This will ensure you will only get that package from that channel. +The pseudocode of the logic that excludes all other channels looks like this: +```rust +// Given a set of requirements and channels +let requirements = vec![packgex]; +let channels = vec![conda_forge, my_channel]; + +// Check each channel +for channel in channels { + // Search for the required package in the channel + for requirement in requirements { + let package = channel.packages.find(requirement.name); + if package.is_some() && requirement.channel != channel { + // Exclude packages from other channels + candidates.excluded.push(package); + } + } +} +``` +This ensures that if a dependency's channel is specified, the solver only considers that channel for the dependency. + +# Channel priority +Channel priority is dictated by the order in the `project.channels` array, where the first channel is the highest priority. +For instance: +```toml +[project] +channels = ["conda-forge", "my-channel"] +``` +If the package is found in `conda-forge` the solver will not look for it in `my-channel`, because we tell the solver they are excluded. +Here is the pseudocode for that logic: +```rust +// Given a set of requirements and channels +let requirements = vec![packgex]; +let channels = vec![conda_forge, my_channel]; + +for requirement in requirements{ + let mut first_channel = None; + + for channel in channels{ + if channel.packages.find(requirement.name) != None { + if first_channel.is_none() || channel.name == first_channel { + first_channel = Some(channel.name); + candidates.push(package); + } else { + // If the package is found in a different channel, add it to the excluded candidates + candidates.excluded.push(package); + } + } + } +} +``` +This method ensures the solver only adds a package to the candidates if it's found in the highest priority channel available. +If you have 10 channels and the package is found in the 5th channel it will exclude the next 5 channels from the candidates if they also contain the package. + +# Use case: pytorch and nvidia with conda-forge +A common use case is to use `pytorch` with `nvidia` drivers, while also needing the `conda-forge` channel for the main dependencies. +```toml +[project] +channels = ["nvidia/label/cuda-11.8.0", "nvidia", "conda-forge", "pytorch"] +platforms = ["linux-64"] + +[dependencies] +cuda = {version = "*", channel="nvidia/label/cuda-11.8.0"} +pytorch = {version = "2.0.1.*", channel="pytorch"} +torchvision = {version = "0.15.2.*", channel="pytorch"} +pytorch-cuda = {version = "11.8.*", channel="pytorch"} +python = "3.10.*" +``` +What this will do is get as much as possible from the `nvidia/label/cuda-11.8.0` channel, which is actually only the `cuda` package. + +Then it will get all packages from the `nvidia` channel, which is a little more and some packages overlap the `nvidia` and `conda-forge` channel. +Like the `cuda-cudart` package, which will now only be retrieved from the `nvidia` channel because of the priority logic. + +Then it will get the packages from the `conda-forge` channel, which is the main channel for the dependencies. + +But the user only wants the pytorch packages from the `pytorch` channel, which is why `pytorch` is added last and the dependencies are added as channel specific dependencies. + +We don't define the `pytorch` channel before `conda-forge` because we want to get as much as possible from the `conda-forge` as the pytorch channel is not always shipping the best versions of all packages. + +For example, it also ships the `ffmpeg` package, but only an old version which doesn't work with the newer pytorch versions. +Thus breaking the installation if we would skip the `conda-forge` channel for `ffmpeg` with the priority logic. diff --git a/mkdocs.yml b/mkdocs.yml index 9a3c41c0f..543fdce1b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -77,6 +77,7 @@ nav: - Tasks: advanced/advanced_tasks.md - Multi Platform: advanced/multi_platform_configuration.md - Info command: advanced/explain_info_command.md + - Channel Logic: advanced/channel_priority.md - Examples: - C++/Cmake: examples/cpp-sdl.md - OpenCV: examples/opencv.md From cdc344228a0f5b942b5c429bb22ab6b5b5bddbce Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 4 Jan 2024 15:02:38 +0100 Subject: [PATCH 2/2] change pseudo code examples for mermaid flowcharts --- docs/advanced/channel_priority.md | 76 ++++++++++++++----------------- mkdocs.yml | 5 +- 2 files changed, 38 insertions(+), 43 deletions(-) diff --git a/docs/advanced/channel_priority.md b/docs/advanced/channel_priority.md index a6ad27caa..a89231e1b 100644 --- a/docs/advanced/channel_priority.md +++ b/docs/advanced/channel_priority.md @@ -2,7 +2,7 @@ All logic regarding the decision which dependencies can be installed from which The actual code regarding this is in the [`rattler_solve`](https://github.com/mamba-org/rattler/blob/02e68c9539c6009cc1370fbf46dc69ca5361d12d/crates/rattler_solve/src/resolvo/mod.rs) crate. This might however be hard to read. -Therefore, this document will continue with pseudocode. +Therefore, this document will continue with simplified flow charts. # Channel specific dependencies When a user defines a channel per dependency, the solver needs to know the other channels are unusable for this dependency. @@ -13,57 +13,49 @@ channels = ["conda-forge", "my-channel"] [dependencies] packgex = { version = "*", channel = "my-channel" } ``` -This will ensure you will only get that package from that channel. -The pseudocode of the logic that excludes all other channels looks like this: -```rust -// Given a set of requirements and channels -let requirements = vec![packgex]; -let channels = vec![conda_forge, my_channel]; +In the `packagex` example, the solver will understand that the package is only available in `my-channel` and will not look for it in `conda-forge`. -// Check each channel -for channel in channels { - // Search for the required package in the channel - for requirement in requirements { - let package = channel.packages.find(requirement.name); - if package.is_some() && requirement.channel != channel { - // Exclude packages from other channels - candidates.excluded.push(package); - } - } -} +The flowchart of the logic that excludes all other channels: + +``` mermaid +flowchart TD + A[Start] --> B[Given a Dependency] + B --> C{Channel Specific Dependency?} + C -->|Yes| D[Exclude All Other Channels for This Package] + C -->|No| E{Any Other Dependencies?} + E -->|Yes| B + E -->|No| F[End] + D --> E ``` -This ensures that if a dependency's channel is specified, the solver only considers that channel for the dependency. # Channel priority Channel priority is dictated by the order in the `project.channels` array, where the first channel is the highest priority. For instance: ```toml [project] -channels = ["conda-forge", "my-channel"] +channels = ["conda-forge", "my-channel", "your-channel"] ``` -If the package is found in `conda-forge` the solver will not look for it in `my-channel`, because we tell the solver they are excluded. -Here is the pseudocode for that logic: -```rust -// Given a set of requirements and channels -let requirements = vec![packgex]; -let channels = vec![conda_forge, my_channel]; - -for requirement in requirements{ - let mut first_channel = None; - - for channel in channels{ - if channel.packages.find(requirement.name) != None { - if first_channel.is_none() || channel.name == first_channel { - first_channel = Some(channel.name); - candidates.push(package); - } else { - // If the package is found in a different channel, add it to the excluded candidates - candidates.excluded.push(package); - } - } - } -} +If the package is found in `conda-forge` the solver will not look for it in `my-channel` and `your-channel`, because it tells the solver they are excluded. +If the package is not found in `conda-forge` the solver will look for it in `my-channel` and if it **is** found there it will tell the solver to exclude `your-channel` for this package. +This diagram explains the logic: +``` mermaid +flowchart TD + A[Start] --> B[Given a Dependency] + B --> C{Loop Over Channels} + C --> D{Package in This Channel?} + D -->|No| C + D -->|Yes| E{"This the first channel + for this package?"} + E -->|Yes| F[Include Package in Candidates] + E -->|No| G[Exclude Package from Candidates] + F --> H{Any Other Channels?} + G --> H + H -->|Yes| C + H -->|No| I{Any Other Dependencies?} + I -->|No| J[End] + I -->|Yes| B ``` + This method ensures the solver only adds a package to the candidates if it's found in the highest priority channel available. If you have 10 channels and the package is found in the 5th channel it will exclude the next 5 channels from the candidates if they also contain the package. diff --git a/mkdocs.yml b/mkdocs.yml index 543fdce1b..8a86e2a37 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -48,7 +48,10 @@ markdown_extensions: - pymdownx.inlinehilite - pymdownx.snippets - pymdownx.details - - pymdownx.superfences + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid - pymdownx.tabbed: alternate_style: true - toc: