feat: use python_site_packages_path field when available for installi…

…ng noarch: python packages, CEP-17 (#909)

### Description

Use the `python_site_packages_path` field from repodata to set the
directory when `noarch: python` packages get installed as specified in
[CEP-17](conda/ceps#90).

This the start of an implementation. Wanted to get feedback on the
approach before updating other crates and tests.

---------

Co-authored-by: Bas Zalmstra <[email protected]>

Cargo.toml

@@ -33,7 +33,7 @@ async-compression = { version = "0.4.17", features = [
     "zstd",
 ] }
 async-fd-lock = "0.2.0"
-fs4 = "0.10.0"
+fs4 = "0.11.0"
 async-trait = "0.1.83"
 axum = { version = "0.7.7", default-features = false, features = [
     "tokio",

crates/rattler/src/install/clobber_registry.rs

@@ -1154,9 +1154,12 @@ mod tests {
         // Create a transaction
         let operations = test_python_noarch_operations();
 
-        let python_info =
-            PythonInfo::from_version(&Version::from_str("3.11.0").unwrap(), Platform::current())
-                .unwrap();
+        let python_info = PythonInfo::from_version(
+            &Version::from_str("3.11.0").unwrap(),
+            None,
+            Platform::current(),
+        )
+        .unwrap();
         let transaction = transaction::Transaction::<PrefixRecord, RepoDataRecord> {
             operations,
             python_info: Some(python_info.clone()),

crates/rattler/src/install/entry_point.rs

@@ -205,17 +205,25 @@ mod test {
             "/prefix",
             false,
             &EntryPoint::from_str("jupyter-lab = jupyterlab.labapp:main").unwrap(),
-            &PythonInfo::from_version(&Version::from_str("3.11.0").unwrap(), Platform::Linux64)
-                .unwrap(),
+            &PythonInfo::from_version(
+                &Version::from_str("3.11.0").unwrap(),
+                None,
+                Platform::Linux64,
+            )
+            .unwrap(),
         );
         insta::assert_snapshot!(script);
 
         let script = super::python_entry_point_template(
             "/prefix",
             true,
             &EntryPoint::from_str("jupyter-lab = jupyterlab.labapp:main").unwrap(),
-            &PythonInfo::from_version(&Version::from_str("3.11.0").unwrap(), Platform::Linux64)
-                .unwrap(),
+            &PythonInfo::from_version(
+                &Version::from_str("3.11.0").unwrap(),
+                None,
+                Platform::Linux64,
+            )
+            .unwrap(),
         );
         insta::assert_snapshot!("windows", script);
     }

crates/rattler/src/install/mod.rs

@@ -784,7 +784,8 @@ mod test {
 
         // Specify python version
         let python_version =
-            PythonInfo::from_version(&Version::from_str("3.11.0").unwrap(), platform).unwrap();
+            PythonInfo::from_version(&Version::from_str("3.11.0").unwrap(), None, platform)
+                .unwrap();
 
         // Download and install each layer into an environment.
         let install_driver = InstallDriver::default();

crates/rattler/src/install/python.rs

@@ -1,9 +1,13 @@
-use rattler_conda_types::{Platform, Version};
-use std::borrow::Cow;
-use std::path::{Path, PathBuf};
+use std::{
+    borrow::Cow,
+    path::{Path, PathBuf},
+};
 
-/// Information required for linking no-arch python packages. The struct contains information about
-/// a specific Python version that is installed in an environment.
+use rattler_conda_types::{PackageRecord, Platform, Version};
+
+/// Information required for linking no-arch python packages. The struct
+/// contains information about a specific Python version that is installed in an
+/// environment.
 #[derive(Debug, Clone)]
 pub struct PythonInfo {
     /// The platform that the python package is installed for
@@ -29,9 +33,26 @@ pub enum PythonInfoError {
 }
 
 impl PythonInfo {
-    /// Build an instance based on the version of the python package and the platform it is
-    /// installed for.
-    pub fn from_version(version: &Version, platform: Platform) -> Result<Self, PythonInfoError> {
+    /// Build an instance based on metadata of the package that represents the
+    /// python interpreter.
+    pub fn from_python_record(
+        record: &PackageRecord,
+        platform: Platform,
+    ) -> Result<Self, PythonInfoError> {
+        Self::from_version(
+            record.version.version(),
+            record.python_site_packages_path.as_deref(),
+            platform,
+        )
+    }
+
+    /// Build an instance based on the version of the python package and the
+    /// platform it is installed for.
+    pub fn from_version(
+        version: &Version,
+        site_packages_path: Option<&str>,
+        platform: Platform,
+    ) -> Result<Self, PythonInfoError> {
         // Determine the major, and minor versions of the version
         let (major, minor) = version
             .as_major_minor()
@@ -45,11 +66,16 @@ impl PythonInfo {
         };
 
         // Find the location of the site packages
-        let site_packages_path = if platform.is_windows() {
-            PathBuf::from("Lib/site-packages")
-        } else {
-            PathBuf::from(format!("lib/python{major}.{minor}/site-packages"))
-        };
+        let site_packages_path = site_packages_path.map_or_else(
+            || {
+                if platform.is_windows() {
+                    PathBuf::from("Lib/site-packages")
+                } else {
+                    PathBuf::from(format!("lib/python{major}.{minor}/site-packages"))
+                }
+            },
+            PathBuf::from,
+        );
 
         // Binary directory
         let bin_dir = if platform.is_windows() {
@@ -89,8 +115,8 @@ impl PythonInfo {
         }
     }
 
-    /// Returns the target location of a file in a noarch python package given its location in its
-    /// package archive.
+    /// Returns the target location of a file in a noarch python package given
+    /// its location in its package archive.
     pub fn get_python_noarch_target_path<'a>(&self, relative_path: &'a Path) -> Cow<'a, Path> {
         if let Ok(rest) = relative_path.strip_prefix("site-packages/") {
             self.site_packages_path.join(rest).into()
@@ -101,8 +127,8 @@ impl PythonInfo {
         }
     }
 
-    /// Returns true if this version of python differs so much that a relink is required for all
-    /// noarch python packages.
+    /// Returns true if this version of python differs so much that a relink is
+    /// required for all noarch python packages.
     pub fn is_relink_required(&self, previous: &PythonInfo) -> bool {
         self.short_version.0 != previous.short_version.0
             || self.short_version.1 != previous.short_version.1

crates/rattler/src/install/transaction.rs

@@ -1,9 +1,9 @@
 use std::collections::HashSet;
 
-use crate::install::python::PythonInfoError;
-use crate::install::PythonInfo;
 use rattler_conda_types::{PackageRecord, Platform};
 
+use crate::install::{python::PythonInfoError, PythonInfo};
+
 /// Error that occurred during creation of a Transaction
 #[derive(Debug, thiserror::Error)]
 pub enum TransactionError {
@@ -31,17 +31,19 @@ pub enum TransactionOperation<Old, New> {
         new: New,
     },
 
-    /// Reinstall a package. This can happen if the Python version changed in the environment, we
-    /// need to relink all noarch python packages in that case.
+    /// Reinstall a package. This can happen if the Python version changed in
+    /// the environment, we need to relink all noarch python packages in
+    /// that case.
     Reinstall(Old),
 
     /// Completely remove a package
     Remove(Old),
 }
 
 impl<Old: AsRef<New>, New> TransactionOperation<Old, New> {
-    /// Returns the record of the package to install for this operation. If this operation does not
-    /// refer to an installable package, `None` is returned.
+    /// Returns the record of the package to install for this operation. If this
+    /// operation does not refer to an installable package, `None` is
+    /// returned.
     pub fn record_to_install(&self) -> Option<&New> {
         match self {
             TransactionOperation::Install(record) => Some(record),
@@ -53,8 +55,9 @@ impl<Old: AsRef<New>, New> TransactionOperation<Old, New> {
 }
 
 impl<Old, New> TransactionOperation<Old, New> {
-    /// Returns the record of the package to remove for this operation. If this operation does not
-    /// refer to an removable package, `None` is returned.
+    /// Returns the record of the package to remove for this operation. If this
+    /// operation does not refer to an removable package, `None` is
+    /// returned.
     pub fn record_to_remove(&self) -> Option<&Old> {
         match self {
             TransactionOperation::Install(_) => None,
@@ -65,25 +68,28 @@ impl<Old, New> TransactionOperation<Old, New> {
     }
 }
 
-/// Describes the operations to perform to bring an environment from one state into another.
+/// Describes the operations to perform to bring an environment from one state
+/// into another.
 #[derive(Debug)]
 pub struct Transaction<Old, New> {
     /// A list of operations to update an environment
     pub operations: Vec<TransactionOperation<Old, New>>,
 
-    /// The python version of the target state, or None if python doesnt exist in the environment.
+    /// The python version of the target state, or None if python doesnt exist
+    /// in the environment.
     pub python_info: Option<PythonInfo>,
 
-    /// The python version of the current state, or None if python didnt exist in the previous
-    /// environment.
+    /// The python version of the current state, or None if python didnt exist
+    /// in the previous environment.
     pub current_python_info: Option<PythonInfo>,
 
     /// The target platform of the transaction
     pub platform: Platform,
 }
 
 impl<Old, New> Transaction<Old, New> {
-    /// Return an iterator over the prefix records of all packages that are going to be removed.
+    /// Return an iterator over the prefix records of all packages that are
+    /// going to be removed.
     pub fn removed_packages(&self) -> impl Iterator<Item = &Old> + '_ {
         self.operations
             .iter()
@@ -97,7 +103,8 @@ impl<Old, New> Transaction<Old, New> {
 }
 
 impl<Old: AsRef<New>, New> Transaction<Old, New> {
-    /// Return an iterator over the prefix records of all packages that are going to be installed.
+    /// Return an iterator over the prefix records of all packages that are
+    /// going to be installed.
     pub fn installed_packages(&self) -> impl Iterator<Item = &New> + '_ {
         self.operations
             .iter()
@@ -111,8 +118,8 @@ impl<Old: AsRef<New>, New> Transaction<Old, New> {
 }
 
 impl<Old: AsRef<PackageRecord>, New: AsRef<PackageRecord>> Transaction<Old, New> {
-    /// Constructs a [`Transaction`] by taking the current situation and diffing that against the
-    /// desired situation.
+    /// Constructs a [`Transaction`] by taking the current situation and diffing
+    /// that against the desired situation.
     pub fn from_current_and_desired<
         CurIter: IntoIterator<Item = Old>,
         NewIter: IntoIterator<Item = New>,
@@ -148,7 +155,8 @@ impl<Old: AsRef<PackageRecord>, New: AsRef<PackageRecord>> Transaction<Old, New>
             .map(|r| r.as_ref().name.clone())
             .collect::<HashSet<_>>();
 
-        // Remove all current packages that are not in desired (but keep order of current)
+        // Remove all current packages that are not in desired (but keep order of
+        // current)
         for record in current_iter {
             if !desired_names.contains(&record.as_ref().name) {
                 operations.push(TransactionOperation::Remove(record));
@@ -158,7 +166,8 @@ impl<Old: AsRef<PackageRecord>, New: AsRef<PackageRecord>> Transaction<Old, New>
         // reverse all removals, last in first out
         operations.reverse();
 
-        // Figure out the operations to perform, but keep the order of the original "desired" iterator
+        // Figure out the operations to perform, but keep the order of the original
+        // "desired" iterator
         for record in desired_iter {
             let name = &record.as_ref().name;
             let old_record = current_map.remove(name);
@@ -190,16 +199,16 @@ impl<Old: AsRef<PackageRecord>, New: AsRef<PackageRecord>> Transaction<Old, New>
     }
 }
 
-/// Determine the version of Python used by a set of packages. Returns `None` if none of the
-/// packages refers to a Python installation.
+/// Determine the version of Python used by a set of packages. Returns `None` if
+/// none of the packages refers to a Python installation.
 fn find_python_info(
     records: impl IntoIterator<Item = impl AsRef<PackageRecord>>,
     platform: Platform,
 ) -> Result<Option<PythonInfo>, PythonInfoError> {
     records
         .into_iter()
         .find(|r| is_python_record(r.as_ref()))
-        .map(|record| PythonInfo::from_version(&record.as_ref().version, platform))
+        .map(|record| PythonInfo::from_python_record(record.as_ref(), platform))
         .map_or(Ok(None), |info| info.map(Some))
 }
 

crates/rattler/src/install/unlink.rs

@@ -163,7 +163,7 @@ mod tests {
         rattler_package_streaming::fs::extract(&package_path, package_dir.path()).unwrap();
 
         let py_info =
-            PythonInfo::from_version(&Version::from_str("3.10").unwrap(), Platform::Linux64)
+            PythonInfo::from_version(&Version::from_str("3.10").unwrap(), None, Platform::Linux64)
                 .unwrap();
         let install_options = InstallOptions {
             python_info: Some(py_info),

crates/rattler_conda_types/src/match_spec/parse.rs

@@ -1360,7 +1360,7 @@ mod tests {
         });
 
         // insta check all the strings
-        let vec_strings = specs.iter().map(|s| s.to_string()).collect::<Vec<_>>();
+        let vec_strings = specs.iter().map(ToString::to_string).collect::<Vec<_>>();
         insta::assert_debug_snapshot!(vec_strings);
 
         // parse back the strings and check if they are the same

crates/rattler_conda_types/src/package/index.rs

@@ -1,16 +1,17 @@
 use std::path::Path;
 
-use super::PackageFile;
-use crate::{NoArchType, PackageName, VersionWithSource};
+use rattler_macros::sorted;
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, skip_serializing_none, OneOrMany};
 
-use rattler_macros::sorted;
+use super::PackageFile;
+use crate::{NoArchType, PackageName, VersionWithSource};
 
 /// A representation of the `index.json` file found in package archives.
 ///
-/// The `index.json` file contains information about the package build and dependencies of the package.
-/// This data makes up the repodata.json file in the repository.
+/// The `index.json` file contains information about the package build and
+/// dependencies of the package. This data makes up the repodata.json file in
+/// the repository.
 #[serde_as]
 #[sorted]
 #[skip_serializing_none]
@@ -22,7 +23,8 @@ pub struct IndexJson {
     /// The build string of the package.
     pub build: String,
 
-    /// The build number of the package. This is also included in the build string.
+    /// The build number of the package. This is also included in the build
+    /// string.
     pub build_number: u64,
 
     /// The package constraints of the package
@@ -33,8 +35,9 @@ pub struct IndexJson {
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub depends: Vec<String>,
 
-    /// Features are a deprecated way to specify different feature sets for the conda solver. This is not
-    /// supported anymore and should not be used. Instead, `mutex` packages should be used to specify
+    /// Features are a deprecated way to specify different feature sets for the
+    /// conda solver. This is not supported anymore and should not be used.
+    /// Instead, `mutex` packages should be used to specify
     /// mutually exclusive features.
     pub features: Option<String>,
 
@@ -47,23 +50,29 @@ pub struct IndexJson {
     /// The lowercase name of the package
     pub name: PackageName,
 
-    /// If this package is independent of architecture this field specifies in what way. See
-    /// [`NoArchType`] for more information.
+    /// If this package is independent of architecture this field specifies in
+    /// what way. See [`NoArchType`] for more information.
     #[serde(skip_serializing_if = "NoArchType::is_none")]
     pub noarch: NoArchType,
 
     /// Optionally, the OS the package is build for.
     pub platform: Option<String>,
 
+    /// Optionally a path within the environment of the site-packages directory.
+    /// This field is only present for python interpreter packages.
+    /// This field was introduced with <https://github.com/conda/ceps/blob/main/cep-17.md>.
+    pub python_site_packages_path: Option<String>,
+
     /// The subdirectory that contains this package
     pub subdir: Option<String>,
 
     /// The timestamp when this package was created
     #[serde_as(as = "Option<crate::utils::serde::Timestamp>")]
     pub timestamp: Option<chrono::DateTime<chrono::Utc>>,
 
-    /// Track features are nowadays only used to downweight packages (ie. give them less priority). To
-    /// that effect, the number of track features is counted (number of commas) and the package is downweighted
+    /// Track features are nowadays only used to downweight packages (ie. give
+    /// them less priority). To that effect, the number of track features is
+    /// counted (number of commas) and the package is downweighted
     /// by the number of track_features.
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     #[serde_as(as = "OneOrMany<_>")]