import urllib.request
import subprocess
import os
import re
from pathlib import Path
git_repo_urls = []
current_dir = os.path.dirname(os.path.abspath(__file__))
download_dir = os.path.join(current_dir, "cann_3rd_lib_path_download")
os.makedirs(download_dir, exist_ok=True)
def execute_process(cmd_list: list, cwd=None):
result = subprocess.run(cmd_list, capture_output=True, text=True, check=False, cwd=cwd)
if result.returncode != 0:
raise Exception(f"Failed to execute command: {cmd_list}, error: {result.stderr}")
def git_download():
for url in list(set(git_repo_urls)):
file_name = url.split('/')[-1]
file_name = file_name.rsplit(".git", 1)[0]
if not file_name:
file_name = "downloaded_file"
file_path = os.path.join(download_dir, file_name)
if not Path(file_path).exists():
execute_process(["git", "clone", url, file_path])
else:
execute_process(["git", "fetch", "origin"], cwd=file_path)
execute_process(["git", "reset", "--hard", "origin/HEAD"], cwd=file_path)
def down_files_native(url_list):
for url in url_list:
file_name = url.split('/')[-1]
if not file_name:
file_name = "downloaded_file"
file_path = os.path.join(download_dir, file_name)
print(f"正在下载 {url} 到 {file_path}")
urllib.request.urlretrieve(url, file_path)
def extract_urls_from_cmake(cmake_file):
"""
从单个 .cmake 文件中提取所有 HTTPS URL
支持格式:
- set(REQ_URL "https://...")
- set(DOWNLOAD_URL "https://...")
- URL https://...
- "https://..."
"""
urls = []
repo_urls = []
content = cmake_file.read_text(encoding='utf-8')
patterns = [
r'set\s*\(\s*\w+\s+["\']?(https://[^"\'\s)]+)["\']?\s*\)',
r'URL\s+["\']?(https://[^"\'\s)]+)["\']?',
r'["\'](https://[^"\']+)["\']',
r'(^|\s)(https://\S+)',
]
for pattern in patterns:
matches = re.findall(pattern, content, re.MULTILINE)
for match in matches:
if isinstance(match, tuple):
url = match[1] if len(match) > 1 and match[1] else match[0]
else:
url = match
url = url.strip().rstrip(')')
if url.endswith(".git"):
repo_urls.append(url)
continue
if url not in urls:
urls.append(url)
return (urls, repo_urls)
def scan_cmake_files(directory: Path) -> dict[str, list[str]]:
"""
扫描目录下所有 .cmake 文件,提取 URL
Returns:
{文件名: [url1, url2, ...]}
"""
results = {}
if not directory.exists():
raise FileNotFoundError(f"Directory not found: {directory}")
cmake_files = sorted(directory.glob("*.cmake"))
if not cmake_files:
print(f"Warning: No .cmake files found in {directory}")
return results
for cmake_file in cmake_files:
(urls, repo_urls) = extract_urls_from_cmake(cmake_file)
if urls:
results[cmake_file.name] = urls
if repo_urls:
git_repo_urls.extend(repo_urls)
if not urls and not repo_urls:
print(f"[{cmake_file.name}] No URLs found")
return results
def get_all_urls(directory: Path) -> list[str]:
"""
获取所有 .cmake 文件中的 URL合并为单一列表
"""
results = scan_cmake_files(directory)
all_urls = []
for urls in results.values():
all_urls.extend(urls)
return list(dict.fromkeys(all_urls))
if __name__ == "__main__":
script_path = Path(__file__).resolve()
cmake_dir = script_path.parent.parent.parent / "cmake" / "third_party"
all_urls = get_all_urls(cmake_dir)
git_download()
execute_process(['rm', '-fr', 'cann-cmake'], cwd=download_dir)
execute_process(['mv', 'cmake', 'cann-cmake'], cwd=download_dir)
third_partys = ['json', 'abseil-cpp', 'eigen', 'gtest', 'protobuf', 'makeself-fetch']
for third_lib in third_partys:
path = script_path.parent.parent.parent / "third_party" / "cann-cmake" / "third_party" / f"{third_lib}.cmake"
(urls, repo_urls) = extract_urls_from_cmake(path)
down_files_native(urls)