use std::path::PathBuf;
use std::sync::mpsc::Receiver;
use std::collections::HashMap;
use color_eyre::eyre::{Result, WrapErr};
use flate2::read::GzDecoder;
use crate::models::*;
use crate::dirs;
use crate::repo::*;
use crate::lfs;
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
#[serde(untagged)]
pub enum UsesFromMacosEntry {
Simple(String),
WithType(HashMap<String, UsesFromMacosType>),
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
#[serde(untagged)]
pub enum UsesFromMacosType {
Single(String),
Multiple(Vec<String>),
}
impl UsesFromMacosEntry {
pub fn dep_name(&self) -> String {
match self {
UsesFromMacosEntry::Simple(name) => name.clone(),
UsesFromMacosEntry::WithType(hash) => {
hash.keys().next().cloned().unwrap_or_default()
}
}
}
pub fn dep_types(&self) -> Vec<String> {
match self {
UsesFromMacosEntry::Simple(_) => vec![],
UsesFromMacosEntry::WithType(hash) => {
match hash.values().next() {
Some(UsesFromMacosType::Single(t)) => vec![t.clone()],
Some(UsesFromMacosType::Multiple(ts)) => ts.clone(),
None => vec![],
}
}
}
}
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct BrewFormula {
pub name: String,
#[serde(rename = "full_name")]
pub full_name: String,
#[serde(default)]
pub oldnames: Vec<String>,
#[serde(default)]
pub aliases: Vec<String>,
pub desc: Option<String>,
pub license: Option<String>,
pub homepage: Option<String>,
pub versions: BrewVersions,
pub revision: u64,
#[serde(default)]
pub caveats: Option<String>,
#[serde(default)]
pub dependencies: Vec<String>,
#[serde(default)]
#[serde(rename = "build_dependencies")]
pub build_dependencies: Vec<String>,
#[serde(default)]
#[serde(rename = "test_dependencies")]
pub test_dependencies: Vec<String>,
#[serde(default)]
#[serde(rename = "runtime_dependencies")]
pub runtime_dependencies: Vec<String>,
#[serde(default)]
#[serde(rename = "recommended_dependencies")]
pub recommended_dependencies: Vec<String>,
#[serde(default)]
#[serde(rename = "optional_dependencies")]
pub optional_dependencies: Vec<String>,
#[serde(default)]
pub uses_from_macos: Vec<UsesFromMacosEntry>,
#[serde(default)]
#[serde(rename = "uses_from_macos_bounds")]
pub uses_from_macos_bounds: Vec<HashMap<String, String>>,
#[serde(default)]
pub conflicts_with: Vec<String>,
#[serde(default)]
pub bottle: Option<BrewBottle>,
#[serde(default)]
pub variations: HashMap<String, BrewVariation>,
#[serde(default)]
pub post_install_defined: bool,
#[serde(default)]
pub service: Option<BrewService>,
#[serde(default, rename = "keg_only")]
pub keg_only: bool,
#[serde(default, rename = "keg_only_reason")]
pub keg_only_reason: Option<BrewKegOnlyReason>,
#[serde(default)]
pub requirements: Vec<BrewRequirement>,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct BrewVariation {
#[serde(default)]
pub dependencies: Vec<String>,
#[serde(default)]
#[serde(rename = "build_dependencies")]
pub build_dependencies: Vec<String>,
#[serde(default)]
#[serde(rename = "test_dependencies")]
pub test_dependencies: Vec<String>,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct BrewVersions {
pub stable: Option<String>,
pub head: Option<String>,
pub bottle: bool,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct BrewBottle {
pub stable: Option<BrewBottleStable>,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct BrewBottleStable {
pub rebuild: u64,
#[serde(rename = "root_url")]
pub root_url: String,
pub files: HashMap<String, BrewBottleFile>,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct BrewBottleFile {
pub cellar: String,
pub url: String,
pub sha256: String,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct BrewService {
#[serde(default)]
pub name: Option<BrewServiceName>,
#[serde(default)]
pub run: Option<serde_json::Value>,
#[serde(default, rename = "run_type")]
pub run_type: Option<String>,
#[serde(default)]
pub interval: Option<u64>,
#[serde(default)]
pub cron: Option<String>,
#[serde(default, rename = "keep_alive")]
pub keep_alive: Option<serde_json::Value>,
#[serde(default, rename = "launch_only_once")]
pub launch_only_once: Option<bool>,
#[serde(default, rename = "require_root")]
pub require_root: Option<bool>,
#[serde(default, rename = "environment_variables")]
pub environment_variables: Option<HashMap<String, String>>,
#[serde(default, rename = "working_dir")]
pub working_dir: Option<String>,
#[serde(default, rename = "root_dir")]
pub root_dir: Option<String>,
#[serde(default, rename = "log_path")]
pub log_path: Option<String>,
#[serde(default, rename = "error_log_path")]
pub error_log_path: Option<String>,
#[serde(default, rename = "restart_delay")]
pub restart_delay: Option<u64>,
#[serde(default)]
pub sockets: Option<serde_json::Value>,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct BrewServiceName {
#[serde(default)]
pub macos: Option<String>,
#[serde(default)]
pub linux: Option<String>,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct BrewKegOnlyReason {
#[serde(default)]
pub reason: String,
#[serde(default)]
pub explanation: String,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct BrewRequirement {
#[serde(default)]
pub name: String,
#[serde(default)]
pub cask: Option<String>,
#[serde(default)]
pub download: Option<String>,
#[serde(default)]
pub version: Option<String>,
#[serde(default)]
pub contexts: Vec<String>,
#[serde(default)]
pub specs: Vec<String>,
}
fn get_bottle_tag_fallbacks(bottle_tag: &str) -> Vec<String> {
let (arm_prefix, tags): (&str, &[&str]) = if bottle_tag.starts_with("arm64_") {
("arm64_", &["tahoe", "sequoia", "sonoma", "ventura", "monterey"])
} else if bottle_tag.starts_with("x86_64_") {
return vec![bottle_tag.to_string()];
} else {
("", &["tahoe", "sequoia", "sonoma", "ventura", "monterey"])
};
let current_base = bottle_tag.strip_prefix(arm_prefix).unwrap_or(bottle_tag);
let current_idx = tags.iter().position(|&t| t == current_base).unwrap_or(0);
tags[current_idx..]
.iter()
.map(|&t| format!("{}{}", arm_prefix, t))
.collect()
}
impl BrewFormula {
fn find_best_bottle<'a>(&'a self, bottle: &'a BrewBottleStable, bottle_tag: &str) -> Option<(&'a BrewBottleFile, String)> {
if let Some(file) = bottle.files.get(bottle_tag) {
return Some((file, bottle_tag.to_string()));
}
if let Some(file) = bottle.files.get("all") {
return Some((file, "all".to_string()));
}
for fallback_tag in get_bottle_tag_fallbacks(bottle_tag) {
if fallback_tag != bottle_tag {
if let Some(file) = bottle.files.get(&fallback_tag) {
log::debug!("Using fallback bottle tag {} instead of {} for {}",
fallback_tag, bottle_tag, self.name);
return Some((file, fallback_tag));
}
}
}
None
}
pub fn to_package(&self, bottle_tag: &str) -> Option<Package> {
let bottle = self.bottle.as_ref()?.stable.as_ref()?;
let (bottle_file, actual_tag) = self.find_best_bottle(bottle, bottle_tag)?;
let is_noarch = bottle.files.contains_key("all") && !bottle.files.contains_key(bottle_tag);
let version = self.versions.stable.clone()?;
let mut deps = self.runtime_dependencies.clone();
deps.extend(self.dependencies.clone());
let mut build_deps = self.build_dependencies.clone();
let mut test_deps = self.test_dependencies.clone();
let mut recommended = self.recommended_dependencies.clone();
let optional = self.optional_dependencies.clone();
if let Some(variation) = self.variations.get(bottle_tag) {
if !variation.dependencies.is_empty() {
deps = variation.dependencies.clone();
}
if !variation.build_dependencies.is_empty() {
build_deps = variation.build_dependencies.clone();
}
if !variation.test_dependencies.is_empty() {
test_deps = variation.test_dependencies.clone();
}
}
if bottle_tag.ends_with("_linux") {
for entry in &self.uses_from_macos {
let dep_name = entry.dep_name();
let types = entry.dep_types();
if types.is_empty() {
recommended.push(dep_name);
} else {
for t in &types {
if t == "build" {
recommended.push(dep_name.clone());
} else if t == "test" {
recommended.push(dep_name.clone());
} else {
recommended.push(dep_name.clone());
}
}
}
}
}
let pkg_version = if self.revision > 0 {
format!("{}_{}", version, self.revision)
} else {
version.clone()
};
let bottle_filename = if bottle.rebuild > 0 {
format!("{}-{}.{}.bottle.{}.tar.gz", self.name, pkg_version, actual_tag, bottle.rebuild)
} else {
format!("{}-{}.{}.bottle.tar.gz", self.name, pkg_version, actual_tag)
};
let location = bottle_filename;
let arch = if is_noarch {
"all"
} else if bottle_tag.starts_with("arm64_") {
"arm64"
} else {
"x86_64"
};
let mut provides = vec![self.name.clone()];
provides.extend(self.aliases.clone());
provides.extend(self.oldnames.clone());
let service_json = self.service.as_ref()
.and_then(|s| serde_json::to_string(s).ok());
Some(Package {
pkgname: self.name.clone(),
version: format!("{}_{}", version, self.revision),
arch: arch.to_string(),
location,
sha256sum: Some(bottle_file.sha256.clone()),
requires: deps,
build_requires: build_deps,
check_requires: test_deps,
recommends: recommended,
suggests: optional,
conflicts: self.conflicts_with.clone(),
provides,
summary: self.desc.clone().unwrap_or_default(),
description: self.desc.clone(),
homepage: self.homepage.clone().unwrap_or_default(),
caveats: self.caveats.clone(),
license: self.license.clone(),
tag: Some(actual_tag.to_string()),
service_json,
format: PackageFormat::Brew,
..Default::default()
})
}
}
pub fn get_bottle_tag() -> String {
#[cfg(target_os = "macos")]
{
match std::process::Command::new("uname").arg("-r").output() {
Ok(output) => {
let version = String::from_utf8_lossy(&output.stdout);
let parts: Vec<&str> = version.trim().split('.').collect();
if let Ok(major) = parts[0].parse::<u32>() {
match major {
25 => {
#[cfg(target_arch = "aarch64")]
return "arm64_tahoe".to_string();
#[cfg(target_arch = "x86_64")]
return "tahoe".to_string();
}
24 => {
#[cfg(target_arch = "aarch64")]
return "arm64_sequoia".to_string();
#[cfg(target_arch = "x86_64")]
return "sequoia".to_string();
}
23 => {
#[cfg(target_arch = "aarch64")]
return "arm64_sonoma".to_string();
#[cfg(target_arch = "x86_64")]
return "sonoma".to_string();
}
22 => {
#[cfg(target_arch = "aarch64")]
return "arm64_ventura".to_string();
#[cfg(target_arch = "x86_64")]
return "ventura".to_string();
}
21 => {
#[cfg(target_arch = "aarch64")]
return "arm64_monterey".to_string();
#[cfg(target_arch = "x86_64")]
return "monterey".to_string();
}
_ => {
if major > 25 {
#[cfg(target_arch = "aarch64")]
return "arm64_tahoe".to_string();
#[cfg(target_arch = "x86_64")]
return "tahoe".to_string();
} else {
#[cfg(target_arch = "aarch64")]
return "arm64_ventura".to_string();
#[cfg(target_arch = "x86_64")]
return "ventura".to_string();
}
}
}
}
}
Err(_) => {}
}
#[cfg(target_arch = "aarch64")]
return "arm64_ventura".to_string();
#[cfg(target_arch = "x86_64")]
return "ventura".to_string();
}
#[cfg(all(target_os = "linux", target_arch = "aarch64"))]
return "arm64_linux".to_string();
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
return "x86_64_linux".to_string();
#[cfg(not(any(
target_os = "macos",
all(target_os = "linux", target_arch = "aarch64"),
all(target_os = "linux", target_arch = "x86_64"),
)))]
{
"x86_64_linux".to_string()
}
}
pub fn parse_formula_json(repo: &RepoRevise, _release_dir: &PathBuf) -> Result<Vec<RepoReleaseItem>> {
let mut release_items = Vec::new();
let bottle_tag = get_bottle_tag();
let repo_dir = dirs::get_repo_dir(&repo);
let output_path = repo_dir.join("packages.txt");
let url = repo.index_url.clone();
let location = url.split('/').last().unwrap_or("formula.jws.json.gz").to_string();
let package_baseurl = "https://mirrors.tuna.tsinghua.edu.cn/homebrew-bottles/bottles".to_string();
let download_path = crate::mirror::Mirrors::url_to_cache_path(&url, &repo.repodata_name)
.with_context(|| format!("Failed to convert URL to cache path: {}", url))?;
let release_status = should_refresh_release_file(&download_path, repo)?;
let need_download = matches!(release_status, ReleaseStatus::NeedDownload | ReleaseStatus::NeedUpdate);
let need_convert = !lfs::exists_on_host(&output_path) || {
let repoindex_path = repo_dir.join("RepoIndex.json");
!lfs::exists_on_host(&repoindex_path)
};
release_items.push(RepoReleaseItem {
repo_revise: repo.clone(),
need_download,
need_convert,
arch: bottle_tag,
url,
package_baseurl,
hash_type: "SHA256".to_string(),
location,
is_packages: true,
output_path,
download_path,
..Default::default()
});
Ok(release_items)
}
pub fn process_packages_content(data_rx: Receiver<Vec<u8>>, repo_dir: &PathBuf, revise: &RepoReleaseItem) -> Result<PackagesFileInfo> {
log::debug!("Starting to process Brew formula content for {}", revise.location);
log::debug!(" repo_dir: {:?}", repo_dir);
log::debug!(" output_path: {:?}", revise.output_path);
log::debug!(" arch: {}", revise.arch);
let mut all_data = Vec::new();
for chunk in data_rx {
all_data.extend_from_slice(&chunk);
}
log::debug!("Received {} bytes of data", all_data.len());
let json_data = extract_json_array_from_jws(&all_data)?;
let formulas: Vec<BrewFormula> = if revise.location.ends_with(".gz") {
let decoder = GzDecoder::new(&json_data[..]);
serde_json::from_reader(decoder)
.with_context(|| format!("Failed to parse compressed JSON for {}", revise.location))?
} else {
serde_json::from_slice(&json_data)
.with_context(|| format!("Failed to parse JSON for {}", revise.location))?
};
log::debug!("Successfully parsed JSON, found {} formulas", formulas.len());
process_brew_formulas(repo_dir, revise, formulas)
.with_context(|| format!("Failed to process brew formulas for {}", revise.location))
}
fn extract_json_array_from_jws(data: &[u8]) -> Result<Vec<u8>> {
if serde_json::from_slice::<Vec<BrewFormula>>(data).is_ok() {
return Ok(data.to_vec());
}
let data_str = String::from_utf8_lossy(data);
if data_str.starts_with("{\"payload\":\"") {
#[derive(serde::Deserialize)]
struct JwsFormat {
payload: String,
}
if let Ok(jws) = serde_json::from_str::<JwsFormat>(&data_str) {
log::debug!("Extracted JSON array from JWS payload string ({} bytes)", jws.payload.len());
return Ok(jws.payload.into_bytes());
}
}
log::debug!("Could not detect JWS format, trying to parse as plain JSON");
Ok(data.to_vec())
}
fn brew_package_entry_lines(package: &Package) -> Vec<String> {
let mut lines = Vec::new();
lines.push(format!("pkgname: {}", package.pkgname));
lines.push(format!("version: {}", package.version));
lines.push(format!("arch: {}", package.arch));
if let Some(ref tag) = package.tag {
lines.push(format!("tag: {}", tag));
}
lines.push(format!("location: {}", package.location));
if let Some(ref sha256) = package.sha256sum {
lines.push(format!("sha256: {}", sha256));
}
if !package.requires.is_empty() {
lines.push(format!("requires: {}", package.requires.join(", ")));
}
if !package.build_requires.is_empty() {
lines.push(format!("buildRequires: {}", package.build_requires.join(", ")));
}
if !package.check_requires.is_empty() {
lines.push(format!("checkRequires: {}", package.check_requires.join(", ")));
}
if !package.recommends.is_empty() {
lines.push(format!("recommends: {}", package.recommends.join(", ")));
}
if !package.suggests.is_empty() {
lines.push(format!("suggests: {}", package.suggests.join(", ")));
}
if !package.conflicts.is_empty() {
lines.push(format!("conflicts: {}", package.conflicts.join(", ")));
}
if !package.provides.is_empty() {
lines.push(format!("provides: {}", package.provides.join(", ")));
}
if !package.summary.is_empty() {
lines.push(format!("summary: {}", package.summary));
}
if !package.homepage.is_empty() {
lines.push(format!("homepage: {}", package.homepage));
}
if let Some(ref license) = package.license {
lines.push(format!("license: {}", license));
}
if let Some(ref caveats) = package.caveats {
let caveats_lines: Vec<String> = caveats.lines().enumerate().map(|(i, line)| {
if i == 0 {
format!("caveats: {}", line)
} else {
format!(" {}", line)
}
}).collect();
lines.push(caveats_lines.join("\n"));
}
if let Some(ref service_json) = package.service_json {
lines.push(format!("serviceJson: {}", service_json));
}
lines
}
fn process_brew_formulas(repo_dir: &PathBuf, revise: &RepoReleaseItem, formulas: Vec<BrewFormula>) -> Result<PackagesFileInfo> {
use std::io::Write;
use std::collections::{HashMap, HashSet, BTreeMap};
use crate::models::PackageRange;
let bottle_tag = &revise.arch;
let output_path = &revise.output_path;
if let Some(parent) = output_path.parent() {
lfs::create_dir_all(parent)?;
}
let mut file = std::fs::File::create(output_path)
.with_context(|| format!("Failed to create output file: {:?}", output_path))?;
let mut package_count = 0;
let mut pkgname2ranges: BTreeMap<String, Vec<PackageRange>> = BTreeMap::new();
let mut provide2pkgnames: HashMap<String, Vec<String>> = HashMap::new();
let mut current_offset: usize = 0;
for formula in formulas {
if let Some(package) = formula.to_package(bottle_tag) {
let package_begin = current_offset;
let pkgname = &package.pkgname;
for alias in &formula.aliases {
provide2pkgnames.entry(alias.clone())
.or_insert_with(Vec::new)
.push(pkgname.clone());
}
for oldname in &formula.oldnames {
provide2pkgnames.entry(oldname.clone())
.or_insert_with(Vec::new)
.push(pkgname.clone());
}
let lines = brew_package_entry_lines(&package);
let pkg_block = lines.join("\n") + "\n\n";
file.write_all(pkg_block.as_bytes())
.with_context(|| format!("Failed to write package: {}", package.pkgname))?;
let package_len = pkg_block.len();
pkgname2ranges.insert(package.pkgname.clone(), vec![PackageRange { begin: package_begin, len: package_len }]);
current_offset += package_len;
package_count += 1;
}
}
file.flush()?;
log::info!("Converted {} brew formulas for {}", package_count, bottle_tag);
let filename = output_path.file_name()
.and_then(|n| n.to_str())
.unwrap_or("packages.txt")
.to_string();
let nr_provides = provide2pkgnames.len();
let provide2pkgnames_path = repo_dir.join(filename.replace("packages", "provide2pkgnames")).with_extension("rkyv");
crate::mmio::serialize_provide2pkgnames(&provide2pkgnames_path, &provide2pkgnames)
.with_context(|| "Failed to serialize provide2pkgnames")?;
let essential_pkgnames_path = repo_dir.join(filename.replace("packages", "essential_pkgnames"));
#[allow(unused_mut)]
let mut essential_pkgnames: HashSet<String> = HashSet::new();
#[cfg(target_os = "linux")]
{
essential_pkgnames.insert("glibc".to_string());
essential_pkgnames.insert("portable-ruby".to_string());
}
#[cfg(target_os = "macos")]
{
essential_pkgnames.insert("portable-ruby".to_string());
}
crate::mmio::serialize_essential_pkgnames(&essential_pkgnames_path, &essential_pkgnames)
.with_context(|| "Failed to serialize essential_pkgnames")?;
let pkgname2ranges_path = output_path.with_extension("idx");
crate::mmio::serialize_pkgname2ranges(&pkgname2ranges_path, &pkgname2ranges)
.with_context(|| "Failed to serialize pkgname2ranges")?;
let repoindex_path = repo_dir.join("RepoIndex.json");
let repoindex = RepoIndex {
url: revise.url.clone(),
package_count,
timestamp: std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs(),
};
let repoindex_json = serde_json::to_string(&repoindex)
.with_context(|| "Failed to serialize RepoIndex")?;
lfs::write(&repoindex_path, repoindex_json.as_bytes())?;
let nr_essentials = essential_pkgnames.len();
let json_path = repo_dir.join(".packages.json");
let packages_file_info = PackagesFileInfo {
filename,
sha256sum: String::new(),
datetime: String::new(),
size: 0,
nr_packages: package_count,
nr_provides,
nr_essentials,
};
let packages_json = serde_json::to_string(&packages_file_info)
.with_context(|| "Failed to serialize PackagesFileInfo")?;
lfs::write(&json_path, packages_json.as_bytes())?;
Ok(packages_file_info)
}
#[derive(Debug, serde::Serialize)]
struct RepoIndex {
url: String,
package_count: usize,
timestamp: u64,
}