use std::fs;
use std::path::{Path, PathBuf};
use std::collections::HashMap;
use std::io::{Read, Seek};
use std::process::Command;
use tar::Archive;
use log;
use lazy_static::lazy_static;
use color_eyre::Result;
use color_eyre::eyre::{self, WrapErr};
use bzip2::read::BzDecoder;
use zstd::stream::read::Decoder as ZstdDecoder;
use zip::ZipArchive;
use regex;
use crate::lfs;
use crate::tar_extract::{create_package_dirs, ExtractConfig, extract_archive};
pub const VERSION_BUILD_SEPARATOR: &str = "-";
lazy_static! {
pub static ref PACKAGE_KEY_MAPPING: HashMap<&'static str, &'static str> = {
let mut m = HashMap::new();
m.insert("name", "pkgname");
m.insert("version", "version");
m.insert("summary", "summary");
m.insert("description", "description");
m.insert("url", "homepage");
m.insert("license", "license");
m.insert("license_family", "licenseFamily");
m.insert("build", "buildString");
m.insert("build_number", "buildNumber");
m.insert("timestamp", "buildTime");
m.insert("size", "size");
m.insert("arch", "arch");
m.insert("platform", "platform");
m.insert("subdir", "subdir");
m.insert("depends", "requires");
m.insert("constrains", "constrains");
m.insert("track_features", "trackFeatures");
m.insert("features", "features");
m.insert("md5", "md5sum");
m.insert("sha256", "sha256");
m.insert("noarch", "noarch");
m.insert("preferred_env", "preferredEnv");
m.insert("python_site_packages_path", "pythonSitePackagesPath");
m
};
pub static ref SCRIPT_MAPPING: HashMap<&'static str, &'static str> = {
let mut m = HashMap::new();
m.insert("pre-link.sh", "pre_install.sh");
m.insert("post-link.sh", "post_install.sh");
m.insert("pre-unlink.sh", "pre_remove.sh");
m.insert("post-unlink.sh", "post_remove.sh");
m.insert("activate.sh", "activate.sh");
m.insert("deactivate.sh", "deactivate.sh");
m.insert("pre-link.bat", "pre_install.bat");
m.insert("post-link.bat", "post_install.bat");
m.insert("pre-unlink.bat", "pre_remove.bat");
m.insert("post-unlink.bat", "post_remove.bat");
m.insert("activate.bat", "activate.bat");
m.insert("deactivate.bat", "deactivate.bat");
m
};
}
pub fn unpack_package<P: AsRef<Path>>(conda_file: P, store_tmp_dir: P, pkgkey: Option<&str>) -> Result<()> {
let conda_file = conda_file.as_ref();
let store_tmp_dir = store_tmp_dir.as_ref();
create_package_dirs(store_tmp_dir, "conda")?;
log::debug!("Unpacking Conda package: {}", conda_file.display());
let file_name = conda_file.file_name().and_then(|n| n.to_str()).unwrap_or("");
if file_name.ends_with(".conda") {
unpack_conda_format(conda_file, store_tmp_dir)
.wrap_err_with(|| format!("Failed to unpack .conda format: {}", conda_file.display()))?;
} else if file_name.ends_with(".tar.bz2") {
unpack_tar_bz2_format(conda_file, store_tmp_dir)
.wrap_err_with(|| format!("Failed to unpack .tar.bz2 format: {}", conda_file.display()))?;
} else {
return Err(eyre::eyre!("Unsupported Conda package format: {}", file_name));
}
crate::store::create_filelist_txt(store_tmp_dir)
.wrap_err_with(|| format!("Failed to create filelist.txt for {}", store_tmp_dir.display()))?;
create_package_txt(store_tmp_dir, pkgkey)
.wrap_err_with(|| format!("Failed to create package.txt for {}", store_tmp_dir.display()))?;
create_scriptlets(store_tmp_dir)
.wrap_err_with(|| format!("Failed to create scriptlets for {}", store_tmp_dir.display()))?;
Ok(())
}
fn unpack_conda_format<P: AsRef<Path>>(conda_file: P, store_tmp_dir: &Path) -> Result<()> {
let conda_file = conda_file.as_ref();
let metadata = lfs::metadata_on_host(conda_file)
.wrap_err_with(|| format!("Failed to read file metadata: {}", conda_file.display()))?;
let file_size = metadata.len();
if file_size == 0 {
return Err(eyre::eyre!(
"File is empty (0 bytes): {}. The download may be incomplete or the file may be corrupted.",
conda_file.display()
));
}
let mut file = fs::File::open(conda_file)
.wrap_err_with(|| format!("Failed to open file: {}", conda_file.display()))?;
let mut magic_bytes = [0u8; 2];
file.read_exact(&mut magic_bytes)
.wrap_err_with(|| format!("Failed to read file header: {}", conda_file.display()))?;
file.seek(std::io::SeekFrom::Start(0))
.wrap_err_with(|| format!("Failed to seek to start: {}", conda_file.display()))?;
if magic_bytes != [0x50, 0x4B] {
return Err(eyre::eyre!(
"File does not appear to be a valid ZIP archive (missing PK header): {}. File size: {} bytes. The file may be corrupted or incomplete.",
conda_file.display(),
file_size
));
}
let mut archive = match ZipArchive::new(file) {
Ok(archive) => archive,
Err(e) => {
let error_msg = e.to_string();
if error_msg.contains("central directory end") || error_msg.contains("Could not find") {
return Err(eyre::eyre!(
"Incomplete .conda package file: {}\n\
File size: {} bytes\n\
The ZIP archive is missing the central directory end, which indicates the download did not complete.\n\
\n\
Solution: Delete the incomplete file and re-download the package:\n\
1. Delete: {}\n\
2. Re-run the install command to re-download the package",
conda_file.display(),
file_size,
conda_file.display()
));
}
return Err(eyre::eyre!(
"Failed to open .conda archive: {}\n\
File size: {} bytes\n\
Error: {}\n\
The ZIP archive may be corrupted or incomplete.",
conda_file.display(),
file_size,
e
));
}
};
let package_stem = conda_file.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("package");
let mut info_component = None;
let mut pkg_component = None;
for i in 0..archive.len() {
let entry = archive.by_index(i)?;
let name = entry.name();
if name.starts_with(&format!("info-{}", package_stem)) && name.ends_with(".tar.zst") {
info_component = Some(name.to_string());
} else if name.starts_with(&format!("pkg-{}", package_stem)) && name.ends_with(".tar.zst") {
pkg_component = Some(name.to_string());
}
}
if let Some(info_name) = info_component {
let info_reader = archive.by_name(&info_name)?;
extract_zstd_tar_stream(info_reader, &crate::dirs::path_join(store_tmp_dir, &["info", "conda"]), Some("info/".to_string()))
.wrap_err_with(|| format!("Failed to extract info component: {} for {}", info_name, conda_file.display()))?;
} else {
return Err(eyre::eyre!("No info component found in .conda package"));
}
if let Some(pkg_name) = pkg_component {
let pkg_reader = archive.by_name(&pkg_name)?;
extract_zstd_tar_stream(pkg_reader, &store_tmp_dir.join("fs"), None::<String>)
.wrap_err_with(|| format!("Failed to extract pkg component: {} for {}", pkg_name, conda_file.display()))?;
} else {
return Err(eyre::eyre!("No pkg component found in .conda package"));
}
Ok(())
}
fn unpack_tar_bz2_format<P: AsRef<Path>>(conda_file: P, store_tmp_dir: &Path) -> Result<()> {
let conda_file = conda_file.as_ref();
let file = fs::File::open(conda_file)?;
let decoder = BzDecoder::new(file);
let mut archive = Archive::new(decoder);
let config = ExtractConfig::new(store_tmp_dir.join("fs"))
.meta_dir(crate::dirs::path_join(store_tmp_dir, &["info", "conda"]));
let entries_processed = extract_archive(&mut archive, &config)?;
if !crate::dirs::path_join(store_tmp_dir, &["info", "conda", "index.json"]).exists() {
return Err(eyre::eyre!("No index.json found in Conda package"));
}
log::debug!("Successfully unpacked .tar.bz2 Conda package with {} entries", entries_processed);
Ok(())
}
fn conda_zstd_path_policy(
path: &Path,
_is_hard_link: bool,
target_dir: &Path,
strip_prefix: &Option<String>,
) -> Option<PathBuf> {
let mut final_path = path.to_path_buf();
if let Some(ref prefix) = strip_prefix {
let prefix_path = Path::new(prefix);
if let Ok(stripped) = final_path.strip_prefix(prefix_path) {
final_path = stripped.to_path_buf();
}
}
Some(target_dir.join(final_path))
}
fn extract_zstd_tar_stream<R: Read>(
reader: R,
target_dir: &Path,
strip_prefix: Option<String>,
) -> Result<()> {
lfs::create_dir_all_with_case_sensitivity(target_dir)?;
let decoder = ZstdDecoder::new(reader).wrap_err("Failed to create zstd decoder")?;
let archive = Archive::new(decoder);
let policy: crate::tar_extract::PathPolicy =
Box::new(move |path: &Path, is_hard_link: bool, target_dir: &Path| {
conda_zstd_path_policy(path, is_hard_link, target_dir, &strip_prefix)
});
let config = crate::tar_extract::ExtractConfig::new(target_dir)
.handle_hard_links(false);
let mut archive = archive;
crate::tar_extract::extract_archive_with_policy(&mut archive, &config, policy)?;
Ok(())
}
fn create_package_txt<P: AsRef<Path>>(store_tmp_dir: P, pkgkey: Option<&str>) -> Result<()> {
let store_tmp_dir = store_tmp_dir.as_ref();
let conda_info_dir = crate::dirs::path_join(store_tmp_dir, &["info", "conda"]);
let index_json_path = conda_info_dir.join("index.json");
let index_data: serde_json::Value = crate::io::read_json_file(&index_json_path)?;
let mut package_fields: std::collections::HashMap<String, String> = std::collections::HashMap::new();
if let Some(object) = index_data.as_object() {
for (key, value) in object {
let mapped_key = PACKAGE_KEY_MAPPING
.get(key.as_str())
.unwrap_or(&key.as_str())
.to_string();
let string_value = match value {
serde_json::Value::String(s) => s.clone(),
serde_json::Value::Number(n) => n.to_string(),
serde_json::Value::Bool(b) => b.to_string(),
serde_json::Value::Array(arr) => {
arr.iter()
.filter_map(|v| v.as_str())
.collect::<Vec<_>>()
.join(", ")
}
_ => continue,
};
if !string_value.is_empty() {
package_fields.insert(mapped_key, string_value);
}
}
}
let files_path = conda_info_dir.join("files");
if lfs::exists_on_host(&files_path) {
log::debug!("Found files metadata");
}
let recipe_path = conda_info_dir.join("recipe");
if lfs::exists_on_host(&recipe_path) {
log::debug!("Found recipe metadata");
}
if let Some(build_string) = package_fields.get("buildString").cloned() {
if let Some(version) = package_fields.get_mut("version") {
version.push_str(VERSION_BUILD_SEPARATOR);
version.push_str(&build_string);
}
}
package_fields.insert("format".to_string(), "conda".to_string());
crate::store::save_package_txt(package_fields, store_tmp_dir, pkgkey)
.wrap_err("Failed to save package.txt")?;
Ok(())
}
fn create_scriptlets<P: AsRef<Path>>(store_tmp_dir: P) -> Result<()> {
let store_tmp_dir = store_tmp_dir.as_ref();
let conda_info_dir = crate::dirs::path_join(store_tmp_dir, &["info", "conda"]);
let install_dir = crate::dirs::path_join(store_tmp_dir, &["info", "install"]);
crate::utils::copy_scriptlets_by_mapping(&SCRIPT_MAPPING, &conda_info_dir, &install_dir, false)?;
Ok(())
}
#[cfg(target_os = "linux")]
pub fn detect_glibc_version() -> Result<Option<(String, String)>> {
{
let output = match Command::new("ldd").arg("--version").output() {
Err(_) => {
log::debug!("Failed to execute `ldd --version`. Assuming glibc is not available.");
return Ok(None);
}
Ok(output) => output,
};
let stdout = String::from_utf8_lossy(&output.stdout);
if let Some(version) = parse_glibc_ldd_version(&stdout)? {
return Ok(Some(("glibc".to_string(), version)));
}
Ok(None)
}
}
#[cfg(target_os = "linux")]
fn parse_glibc_ldd_version(input: &str) -> Result<Option<String>> {
let re = regex::Regex::new(r"\)\s*([0-9]+\.[0-9a-b.]+)").unwrap();
if let Some(captures) = re.captures(input) {
if let Some(version_match) = captures.get(1) {
return Ok(Some(version_match.as_str().to_string()));
}
}
Ok(None)
}
pub fn detect_linux_version() -> Result<Option<String>> {
#[cfg(target_os = "linux")]
{
if let Ok(proc_version) = fs::read_to_string("/proc/version") {
if let Some(version) = extract_linux_version_from_proc(&proc_version) {
return Ok(Some(version));
}
}
if let Ok(output) = Command::new("uname").arg("-r").output() {
let version_str = String::from_utf8_lossy(&output.stdout).trim().to_string();
if let Some(version) = extract_linux_version_part(&version_str) {
return Ok(Some(version));
}
}
Ok(None)
}
#[cfg(not(target_os = "linux"))]
{
Ok(None)
}
}
#[cfg(target_os = "linux")]
fn extract_linux_version_from_proc(proc_version: &str) -> Option<String> {
let re = regex::Regex::new(r"Linux version ([0-9]+\.[0-9]+(?:\.[0-9]+)*(?:\.[0-9]+)?)").ok()?;
if let Some(captures) = re.captures(proc_version) {
if let Some(version_match) = captures.get(1) {
return extract_linux_version_part(version_match.as_str());
}
}
None
}
#[cfg(target_os = "linux")]
fn extract_linux_version_part(version_str: &str) -> Option<String> {
let re = regex::Regex::new(r"^([0-9]+\.[0-9]+(?:\.[0-9]+)?(?:\.[0-9]+)?)").ok()?;
if let Some(captures) = re.captures(version_str) {
if let Some(version_match) = captures.get(1) {
return Some(version_match.as_str().to_string());
}
}
None
}
#[cfg(target_os = "macos")]
pub fn detect_osx_version() -> Result<Option<String>> {
if let Ok(output) = Command::new("sw_vers").arg("-productVersion").output() {
let version_str = String::from_utf8_lossy(&output.stdout).trim().to_string();
if !version_str.is_empty() {
return Ok(Some(normalize_osx_version(&version_str)));
}
}
if let Ok(output) = Command::new("uname").arg("-r").output() {
let version_str = String::from_utf8_lossy(&output.stdout).trim().to_string();
if let Some(version) = extract_osx_version_from_darwin(&version_str) {
return Ok(Some(version));
}
}
Ok(None)
}
#[cfg(target_os = "macos")]
fn normalize_osx_version(version: &str) -> String {
let parts: Vec<&str> = version.split('.').collect();
match parts.len() {
1 => format!("{}.{}", parts[0], "0"),
2 => version.to_string(),
_ => parts[..3].join("."),
}
}
#[cfg(target_os = "macos")]
fn extract_osx_version_from_darwin(darwin_version: &str) -> Option<String> {
let re = regex::Regex::new(r"^([0-9]+)").ok()?;
if let Some(captures) = re.captures(darwin_version) {
if let Some(major_match) = captures.get(1) {
let darwin_major: u32 = major_match.as_str().parse().ok()?;
if darwin_major >= 20 {
let macos_major = darwin_major - 9;
return Some(format!("{}.{}", macos_major, "0"));
}
}
}
None
}
pub fn detect_cuda_version() -> Result<Option<String>> {
if let Ok(output) = Command::new("nvidia-smi")
.arg("--query")
.arg("-u")
.arg("-x")
.env_remove("CUDA_VISIBLE_DEVICES")
.output()
{
let stdout = String::from_utf8_lossy(&output.stdout);
let re = regex::Regex::new(r"<cuda_version>(.*?)</cuda_version>").unwrap();
if let Some(captures) = re.captures(&stdout) {
if let Some(version_match) = captures.get(1) {
return Ok(Some(version_match.as_str().to_string()));
}
}
}
Ok(None)
}
pub fn is_unix() -> bool {
cfg!(unix)
}
pub fn create_virtual_package(
pkgname: &str,
version: &str,
build_string: Option<&str>,
) -> crate::models::Package {
use crate::models::PackageFormat;
let version_for_pkgkey = if let Some(build) = build_string {
format!("{}{}{}", version, VERSION_BUILD_SEPARATOR, build)
} else {
version.to_string()
};
crate::package_cache::create_virtual_package(
pkgname,
version,
Some(&version_for_pkgkey),
PackageFormat::Conda,
)
}
pub fn detect_archspec() -> Option<String> {
let arch = std::env::consts::ARCH;
match arch {
"x86_64" => {
#[cfg(target_os = "linux")]
{
if let Ok(cpuinfo) = fs::read_to_string("/proc/cpuinfo") {
for line in cpuinfo.lines() {
if line.starts_with("model name") {
let model = line.split(':').nth(1)?.trim().to_lowercase();
if model.contains("skylake") || model.contains("skx") {
return Some("skylake_avx512".to_string());
}
if model.contains("cascade") {
return Some("cascadelake".to_string());
}
if model.contains("sapphire") {
return Some("sapphirerapids".to_string());
}
if model.contains("ice lake") || model.contains("icelake") {
return Some("icelake".to_string());
}
if model.contains("zen") {
if model.contains("zen 4") || model.contains("zen4") {
return Some("zen4".to_string());
}
if model.contains("zen 3") || model.contains("zen3") {
return Some("zen3".to_string());
}
if model.contains("zen 2") || model.contains("zen2") {
return Some("zen2".to_string());
}
}
}
if line.starts_with("flags") && line.contains("avx512") {
return Some("x86_64_v4".to_string());
}
}
}
}
Some("x86_64".to_string())
}
"aarch64" | "arm64" => Some("aarch64".to_string()),
"powerpc64le" => Some("power10le".to_string()),
_ => Some(arch.to_string()),
}
}
pub fn detect_conda_virtual_packages() -> Result<Vec<crate::models::Package>> {
let mut virtual_packages = Vec::new();
if is_unix() {
virtual_packages.push(create_virtual_package("__unix", "0", None));
}
if let Ok(Some(linux_version)) = detect_linux_version() {
virtual_packages.push(create_virtual_package("__linux", &linux_version, None));
log::debug!("Detected __linux version: {}", linux_version);
}
#[cfg(target_os = "macos")]
{
if let Ok(Some(osx_version)) = detect_osx_version() {
virtual_packages.push(create_virtual_package("__osx", &osx_version, None));
log::debug!("Detected __osx version: {}", osx_version);
}
}
#[cfg(target_os = "linux")]
{
if let Ok(Some((_family, version))) = detect_glibc_version() {
virtual_packages.push(create_virtual_package("__glibc", &version, None));
log::debug!("Detected __glibc version: {}", version);
}
}
if let Ok(Some(cuda_version)) = detect_cuda_version() {
virtual_packages.push(create_virtual_package("__cuda", &cuda_version, None));
log::debug!("Detected __cuda version: {}", cuda_version);
}
if let Some(archspec_name) = detect_archspec() {
virtual_packages.push(create_virtual_package("__archspec", "1", Some(&archspec_name)));
log::debug!("Detected __archspec: {}", archspec_name);
}
log::info!("Detected {} Conda virtual packages", virtual_packages.len());
Ok(virtual_packages)
}