"""
GitCode Hub SDK - 类似huggingface_hub的SDK接口
这个模块提供了类似huggingface_hub的编程接口,
让用户可以通过Python代码直接使用GitCode平台的功能。
"""
import os
from typing import Optional, Union, List, Dict, Any
from pathlib import Path
os.environ["OPENMIND_HUB_ENDPOINT"] = "https://hub.gitcode.com"
cache_dir = os.path.expanduser("~/.cache/gitcode")
os.makedirs(cache_dir, exist_ok=True)
os.environ["HF_HOME"] = cache_dir
from openmind_hub import snapshot_download as om_snapshot_download
from openmind_hub import om_hub_download, upload_folder as om_upload_folder, create_repo
try:
from openmind import OmDataset
from openmind.utils.hub import OM_DATASETS_CACHE
DATASET_SUPPORT = True
except ImportError:
DATASET_SUPPORT = False
OmDataset = None
OM_DATASETS_CACHE = None
try:
from .config import config
except ImportError:
from config import config
def _normalize_repo_id(repo_id: str) -> str:
"""标准化仓库ID,处理三层格式转换"""
parts = repo_id.split('/')
if len(parts) >= 3:
first_part = parts[0]
second_part = parts[1]
remaining_parts = parts[2:]
normalized = first_part + '%2F' + second_part
if remaining_parts:
normalized += '/' + '/'.join(remaining_parts)
return normalized
return repo_id
def _get_token() -> Optional[str]:
"""获取保存的认证token"""
try:
credentials = config.get_credentials()
if credentials and 'token' in credentials:
return credentials['token']
except Exception:
pass
return None
def snapshot_download(
repo_id: str,
revision: Optional[str] = None,
cache_dir: Optional[Union[str, Path]] = None,
local_dir: Optional[Union[str, Path]] = None,
local_dir_use_symlinks: Union[bool, str] = "auto",
library_name: Optional[str] = None,
library_version: Optional[str] = None,
user_agent: Optional[Union[str, Dict[str, str]]] = None,
proxies: Optional[Dict[str, str]] = None,
etag_timeout: float = 10,
resume_download: bool = False,
force_download: bool = False,
token: Optional[Union[str, bool]] = None,
local_files_only: bool = False,
allow_patterns: Optional[Union[List[str], str]] = None,
ignore_patterns: Optional[Union[List[str], str]] = None,
max_workers: int = 8,
tqdm_class: Optional[Any] = None,
disable_tqdm: bool = False
) -> str:
"""
从GitCode Hub下载整个仓库的快照到本地目录
参数:
repo_id (str): 仓库ID,格式为 "username/repo-name" 或 "username/namespace/repo-name"
revision (str, 可选): 指定版本/分支/标签,默认为 "main"
cache_dir (str 或 Path, 可选): 缓存目录路径
local_dir (str 或 Path, 可选): 下载到的本地目录路径
local_dir_use_symlinks (bool 或 str, 可选): 是否使用符号链接,默认为 "auto"
library_name (str, 可选): 调用库的名称
library_version (str, 可选): 调用库的版本
user_agent (str 或 dict, 可选): 用户代理
proxies (dict, 可选): 代理配置
etag_timeout (float, 可选): ETag超时时间,默认10秒
resume_download (bool, 可选): 是否断点续传,默认False
force_download (bool, 可选): 是否强制重新下载,默认False
token (str 或 bool, 可选): 认证token,如果为None则尝试使用保存的token
local_files_only (bool, 可选): 仅使用本地文件,默认False
allow_patterns (list 或 str, 可选): 允许下载的文件模式
ignore_patterns (list 或 str, 可选): 忽略的文件模式
max_workers (int, 可选): 最大并发数,默认8
tqdm_class (可选): 进度条类
disable_tqdm (bool, 可选): 是否禁用进度条,默认False
返回:
str: 下载完成的本地目录路径
示例:
>>> from gitcode_hub import snapshot_download
>>> local_path = snapshot_download("wuyw/Qwen3-Reranker-0.6B-test", local_dir="./models")
"""
normalized_repo_id = _normalize_repo_id(repo_id)
if token is None:
token = _get_token()
kwargs = {
'repo_id': normalized_repo_id,
}
if revision is not None:
kwargs['revision'] = revision
if cache_dir is not None:
kwargs['cache_dir'] = str(cache_dir)
if local_dir is not None:
kwargs['local_dir'] = str(local_dir)
if token is not None:
kwargs['token'] = token
kwargs.update({
'local_dir_use_symlinks': local_dir_use_symlinks,
'library_name': library_name or "gitcode_hub",
'library_version': library_version,
'user_agent': user_agent,
'proxies': proxies,
'etag_timeout': etag_timeout,
'resume_download': resume_download,
'force_download': force_download,
'local_files_only': local_files_only,
'allow_patterns': allow_patterns,
'ignore_patterns': ignore_patterns,
'max_workers': max_workers,
'tqdm_class': tqdm_class,
'disable_tqdm': disable_tqdm,
})
try:
if not local_files_only and token is None:
try:
result = om_snapshot_download(**{k: v for k, v in kwargs.items() if v is not None})
return result
except Exception as e:
error_msg = str(e)
if ("403" in error_msg or "FORBIDDEN" in error_msg or
"no scopes" in error_msg or "unauthorized" in error_msg.lower()):
saved_token = _get_token()
if saved_token:
kwargs['token'] = saved_token
else:
raise Exception(f"仓库访问需要认证,但未找到保存的token。请先使用 'gitcode login' 登录,或在调用时提供token参数。")
else:
raise e
result = om_snapshot_download(**{k: v for k, v in kwargs.items() if v is not None})
return result
except Exception as e:
error_msg = str(e)
if "401" in error_msg or "403" in error_msg:
raise Exception(f"认证失败:{error_msg}。请检查token是否正确,或使用 'gitcode login' 重新登录。")
elif "404" in error_msg:
raise Exception(f"仓库不存在:{repo_id}。请检查仓库名称是否正确。")
else:
raise Exception(f"下载失败:{error_msg}")
def hub_download_url(
repo_id: str,
filename: str,
revision: Optional[str] = None,
repo_type: Optional[str] = None,
) -> str:
"""
获取GitCode Hub上文件的下载URL
参数:
repo_id (str): 仓库ID
filename (str): 文件名
revision (str, 可选): 版本/分支/标签
repo_type (str, 可选): 仓库类型
返回:
str: 文件的下载URL
"""
normalized_repo_id = _normalize_repo_id(repo_id)
base_url = "https://hub.gitcode.com"
if revision is None:
revision = "main"
if repo_type is None:
repo_type = "model"
url = f"{base_url}/{normalized_repo_id}/resolve/{revision}/{filename}"
return url
def download_file(
repo_id: str,
filename: str,
local_dir: Optional[Union[str, Path]] = None,
revision: Optional[str] = None,
token: Optional[str] = None,
force_download: bool = False,
) -> str:
"""
从GitCode Hub下载单个文件
参数:
repo_id (str): 仓库ID
filename (str): 文件名
local_dir (str 或 Path, 可选): 本地目录
revision (str, 可选): 版本/分支/标签
token (str, 可选): 认证token
force_download (bool, 可选): 是否强制重新下载
返回:
str: 下载的文件路径
"""
normalized_repo_id = _normalize_repo_id(repo_id)
if token is None:
token = _get_token()
kwargs = {
'repo_id': normalized_repo_id,
'filename': filename,
}
if local_dir is not None:
kwargs['local_dir'] = str(local_dir)
if revision is not None:
kwargs['revision'] = revision
if token is not None:
kwargs['token'] = token
if force_download:
kwargs['force_download'] = force_download
try:
if token is None:
try:
result = om_hub_download(**{k: v for k, v in kwargs.items() if v is not None})
return result
except Exception as e:
error_msg = str(e)
if ("403" in error_msg or "FORBIDDEN" in error_msg or
"no scopes" in error_msg or "unauthorized" in error_msg.lower()):
saved_token = _get_token()
if saved_token:
kwargs['token'] = saved_token
else:
raise Exception(f"文件访问需要认证,但未找到保存的token。请先使用 'gitcode login' 登录。")
else:
raise e
result = om_hub_download(**{k: v for k, v in kwargs.items() if v is not None})
return result
except Exception as e:
error_msg = str(e)
if "401" in error_msg or "403" in error_msg:
raise Exception(f"认证失败:{error_msg}")
elif "404" in error_msg:
raise Exception(f"文件不存在:{repo_id}/{filename}")
else:
raise Exception(f"下载失败:{error_msg}")
def upload_folder(
folder_path: Union[str, Path],
repo_id: str,
token: Optional[str] = None,
repo_type: Optional[str] = None,
revision: Optional[str] = None,
commit_message: Optional[str] = None,
commit_description: Optional[str] = None,
path_in_repo: str = "./",
ignore_patterns: Optional[List[str]] = None,
) -> str:
"""
上传文件夹到GitCode Hub
参数:
folder_path (str 或 Path): 本地文件夹路径
repo_id (str): 仓库ID
token (str, 可选): 认证token
repo_type (str, 可选): 仓库类型
revision (str, 可选): 分支名
commit_message (str, 可选): 提交消息
commit_description (str, 可选): 提交描述
path_in_repo (str, 可选): 在仓库中的路径,默认为根目录
ignore_patterns (List[str], 可选): 要忽略的文件模式
返回:
str: 提交的URL或ID
示例:
>>> upload_folder("./my-model/", "username/repo-name")
>>> upload_folder("./data/", "username/repo", path_in_repo="datasets/")
"""
normalized_repo_id = _normalize_repo_id(repo_id)
folder_path = Path(folder_path)
if not folder_path.exists():
raise FileNotFoundError(f"文件夹不存在: {folder_path}")
if not folder_path.is_dir():
raise NotADirectoryError(f"路径不是目录: {folder_path}")
if token is None:
token = _get_token()
if token is None:
raise Exception("上传需要认证token,请先使用 'gitcode login' 登录,或提供token参数。")
import tempfile
import shutil
if path_in_repo == "./" or path_in_repo == "." or path_in_repo == "":
upload_path = str(folder_path)
else:
with tempfile.TemporaryDirectory() as temp_dir:
temp_path = Path(temp_dir)
target_path = temp_path / path_in_repo.strip('./')
target_path.parent.mkdir(parents=True, exist_ok=True)
shutil.copytree(folder_path, target_path, dirs_exist_ok=True)
upload_path = str(temp_path)
try:
commit_msg = commit_message or f"Upload folder {folder_path.name}"
result = om_upload_folder(
repo_id=normalized_repo_id,
folder_path=upload_path,
token=token,
commit_message=commit_msg
)
return result
except Exception as e:
error_msg = str(e)
if "401" in error_msg or "403" in error_msg:
raise Exception(f"认证失败:{error_msg}")
elif "404" in error_msg:
raise Exception(f"仓库不存在:{repo_id}")
else:
raise Exception(f"上传失败:{error_msg}")
def create_repository(
repo_id: str,
token: Optional[str] = None,
private: bool = False,
repo_type: str = "model",
exist_ok: bool = False,
space_sdk: Optional[str] = None,
space_hardware: Optional[str] = None,
space_storage: Optional[str] = None,
space_sleep_time: Optional[int] = None,
space_secrets: Optional[List[Dict]] = None,
space_variables: Optional[List[Dict]] = None,
) -> str:
"""
在GitCode Hub上创建仓库
参数:
repo_id (str): 仓库ID
token (str, 可选): 认证token
private (bool, 可选): 是否为私有仓库,默认False
repo_type (str, 可选): 仓库类型,默认"model"
exist_ok (bool, 可选): 如果仓库已存在是否报错,默认False
其他参数: 主要用于Space类型仓库
返回:
str: 仓库的URL
"""
normalized_repo_id = _normalize_repo_id(repo_id)
if token is None:
token = _get_token()
if token is None:
raise Exception("创建仓库需要认证token,请先使用 'gitcode login' 登录,或提供token参数。")
try:
result = create_repo(
repo_id=normalized_repo_id,
token=token,
private=private,
repo_type=repo_type,
exist_ok=exist_ok,
)
return result
except Exception as e:
error_msg = str(e)
if "401" in error_msg or "403" in error_msg:
raise Exception(f"认证失败:{error_msg}")
elif "409" in error_msg or "exists" in error_msg.lower():
if exist_ok:
return f"https://hub.gitcode.com/{repo_id}"
else:
raise Exception(f"仓库已存在:{repo_id}")
else:
raise Exception(f"创建仓库失败:{error_msg}")
def load_dataset(
path: str,
name: Optional[str] = None,
data_dir: Optional[str] = None,
data_files: Optional[Union[str, List[str], Dict]] = None,
split: Optional[str] = None,
cache_dir: Optional[Union[str, Path]] = None,
token: Optional[str] = None,
streaming: bool = False,
revision: Optional[str] = None,
**kwargs
) -> Any:
"""
从GitCode Hub加载数据集
参数:
path (str): 数据集路径,格式为 "username/dataset-name"
name (str, 可选): 数据集名称
data_dir (str, 可选): 数据目录路径
data_files (str, List[str], Dict, 可选): 数据文件
split (str, 可选): 数据集分割,如 "train", "test", "validation"
cache_dir (str 或 Path, 可选): 缓存目录路径,默认使用OM_DATASETS_CACHE
token (str, 可选): 认证token,如果为None则尝试使用保存的token
streaming (bool, 可选): 是否以流式方式加载数据集,默认False
revision (str, 可选): 指定版本/分支/标签,默认为 "main"
**kwargs: 其他传递给OmDataset.load_dataset的参数
返回:
Dataset或DatasetDict: 加载的数据集对象
示例:
>>> from gitcode_hub import load_dataset
>>> dataset = load_dataset("wuyw/my-dataset", split="train")
>>> dataset = load_dataset("wuyw/my-dataset", streaming=True)
异常:
ImportError: 如果openmind库未安装
Exception: 各种下载和认证错误
"""
if not DATASET_SUPPORT:
raise ImportError("数据集功能需要安装openmind库。请运行: pip install openmind")
normalized_path = _normalize_repo_id(path)
if token is None:
token = _get_token()
if cache_dir is None:
cache_dir = OM_DATASETS_CACHE
load_kwargs = {
'path': normalized_path,
'cache_dir': str(cache_dir),
'streaming': streaming,
}
if name is not None:
load_kwargs['name'] = name
if data_dir is not None:
load_kwargs['data_dir'] = data_dir
if data_files is not None:
load_kwargs['data_files'] = data_files
if split is not None:
load_kwargs['split'] = split
if token is not None:
load_kwargs['token'] = token
if revision is not None:
load_kwargs['revision'] = revision
load_kwargs.update(kwargs)
try:
if token is None:
try:
dataset = OmDataset.load_dataset(**load_kwargs)
return dataset
except Exception as e:
error_msg = str(e)
if ("403" in error_msg or "FORBIDDEN" in error_msg or
"no scopes" in error_msg or "unauthorized" in error_msg.lower() or
"401" in error_msg):
saved_token = _get_token()
if saved_token:
load_kwargs['token'] = saved_token
else:
raise Exception(f"数据集访问需要认证,但未找到保存的token。请先使用 'gitcode login' 登录,或在调用时提供token参数。原始错误:{error_msg}")
else:
raise e
dataset = OmDataset.load_dataset(**load_kwargs)
return dataset
except Exception as e:
error_msg = str(e)
if "401" in error_msg or "403" in error_msg:
raise Exception(f"认证失败:{error_msg}。请检查token是否正确,或使用 'gitcode login' 重新登录。")
elif "404" in error_msg:
raise Exception(f"数据集不存在:{path}。请检查数据集路径是否正确。")
elif "ImportError" in error_msg or "No module named" in error_msg:
raise ImportError("数据集功能需要安装openmind库。请运行: pip install openmind")
else:
raise Exception(f"加载数据集失败:{error_msg}")
__all__ = [
'snapshot_download',
'hub_download_url',
'download_file',
'upload_folder',
'create_repository',
'load_dataset',
]