354da4f3创建于 2025年7月24日历史提交
use ek_base::error::{EKError, EKResult};

use crate::safetensor::transformer::VitalMeta;

pub struct WeightSrvClient {
    pub client: reqwest::Client,
    pub addr: String,
    token: tokio::sync::Semaphore,
}

impl WeightSrvClient {
    pub fn new(addr: String) -> Self {
        let client = reqwest::Client::builder()
            .timeout(std::time::Duration::from_secs(60))
            .build()
            .unwrap();
        Self {
            client,
            addr,
            token: tokio::sync::Semaphore::new(50),
        }
    }

    pub async fn load_expert(&self, model: &str, layer: usize, expert: usize) -> EKResult<Vec<u8>> {
        let _g = self.token.acquire().await.unwrap();

        let url = format!("{}/expert/{}/{}/{}", self.addr, model, layer, expert);
        let res = self.client.get(&url).send().await?;
        if res.status().is_success() {
            Ok(res.bytes().await?.to_vec())
        } else {
            Err(EKError::NotFound(format!(
                "failed to load expert from {url}"
            )))
        }
    }

    pub async fn load_layer(&self, model: &str, key: &str) -> EKResult<Vec<u8>> {
        let _g = self.token.acquire().await.unwrap();
        let url = format!("{}/weight/{}/{}", self.addr, model, key);
        let res = self.client.get(&url).send().await?;
        if res.status().is_success() {
            Ok(res.bytes().await?.to_vec())
        } else {
            Err(EKError::NotFound(format!(
                "failed to load layer from {url}"
            )))
        }
    }
    pub async fn load_meta_vital(&self, model: &str) -> EKResult<VitalMeta> {
        let _g = self.token.acquire().await.unwrap();
        let url = format!("{}/meta/vital/{}", self.addr, model);
        let res = self.client.get(&url).send().await?;
        if res.status().is_success() {
            let res = serde_json::from_str::<VitalMeta>(&res.text().await?)?;
            Ok(res)
        } else {
            Err(EKError::NotFound(format!(
                "failed to load meta vital from {url}"
            )))
        }
    }
}