import urllib.request
import urllib.error
import subprocess
import os
import re
import sys
from pathlib import Path
git_repo_urls = []
current_dir = os.path.dirname(os.path.abspath(__file__))
download_dir = os.path.join(current_dir, "cann_3rd_lib_path_download")
os.makedirs(download_dir, exist_ok=True)
def execute_process(cmd_list: list, cwd=None):
result = subprocess.run(
cmd_list, capture_output=True, text=True, check=False, cwd=cwd
)
if result.returncode != 0:
raise Exception(
f"Failed to execute command: {cmd_list}, error: {result.stderr}"
)
def git_download():
failed_urls = []
for url in list(set(git_repo_urls)):
file_name = url.split("/")[-1]
file_name = file_name.rsplit(".git", 1)[0]
if not file_name:
file_name = "downloaded_file"
file_path = os.path.join(download_dir, file_name)
try:
if not Path(file_path).exists():
execute_process(["git", "clone", url, file_path])
else:
execute_process(["git", "fetch", "origin"], cwd=file_path)
execute_process(
["git", "reset", "--hard", "origin/HEAD"], cwd=file_path
)
except Exception as ex:
print(f"Failed to clone {url}, error: {ex}")
failed_urls.append(url)
return failed_urls
def down_files_native(url_list):
failed_urls = []
for url in url_list:
file_name = url.split("/")[-1]
if not file_name:
file_name = "downloaded_file"
file_path = os.path.join(download_dir, file_name)
print(f"正在下载 {url} 到 {file_path}")
try:
urllib.request.urlretrieve(url, file_path)
except (urllib.error.URLError, urllib.error.HTTPError, OSError) as ex:
print(f"Failed to download {url}, error: {ex}")
failed_urls.append(url)
if os.path.exists(file_path):
os.remove(file_path)
return failed_urls
def extract_urls_from_cmake(cmake_file):
"""
从单个 .cmake 文件中提取所有 HTTPS URL
支持格式:
- set(REQ_URL "https://...")
- set(DOWNLOAD_URL "https://...")
- URL https://...
- "https://..."
"""
urls = []
repo_urls = []
content = cmake_file.read_text(encoding="utf-8")
patterns = [
r'set\s*\(\s*\w+\s+["\']?(https://[^"\'\s)]+)["\']?\s*\)',
r'URL\s+["\']?(https://[^"\'\s)]+)["\']?',
r'["\'](https://[^"\']+)["\']',
r"(^|\s)(https://\S+)",
]
for pattern in patterns:
matches = re.findall(pattern, content, re.MULTILINE)
for match in matches:
if isinstance(match, tuple):
url = match[1] if len(match) > 1 and match[1] else match[0]
else:
url = match
url = url.strip().rstrip(")")
if url.endswith(".git"):
repo_urls.append(url)
continue
if url not in urls:
urls.append(url)
return (urls, repo_urls)
def scan_cmake_files(directory: Path) -> dict[str, list[str]]:
"""
扫描目录下所有 .cmake 文件,提取 URL
Returns:
{文件名: [url1, url2, ...]}
"""
results = {}
if not directory.exists():
raise FileNotFoundError(f"Directory not found: {directory}")
cmake_files = sorted(directory.glob("*.cmake"))
if not cmake_files:
print(f"Warning: No .cmake files found in {directory}")
return results
for cmake_file in cmake_files:
(urls, repo_urls) = extract_urls_from_cmake(cmake_file)
if urls:
results[cmake_file.name] = urls
if repo_urls:
git_repo_urls.extend(repo_urls)
if not urls and not repo_urls:
print(f"[{cmake_file.name}] No URLs found")
return results
def get_all_urls(directory: Path) -> list[str]:
"""
获取所有 .cmake 文件中的 URL合并为单一列表
"""
results = scan_cmake_files(directory)
all_urls = []
for urls in results.values():
all_urls.extend(urls)
return list(dict.fromkeys(all_urls))
if __name__ == "__main__":
script_path = Path(__file__).resolve()
cmake_dir = script_path.parent.parent.parent / "cmake" / "third_party"
all_urls = get_all_urls(cmake_dir)
failed_downloads = down_files_native(all_urls)
failed_clones = git_download()
execute_process(["rm", "-fr", "cann-cmake"], cwd=download_dir)
execute_process(["mv", "cmake", "cann-cmake"], cwd=download_dir)
third_partys = [
"json",
"abseil-cpp",
"eigen",
"gtest",
"protobuf",
"makeself-fetch",
]
for third_lib in third_partys:
path = (
script_path.parent.parent.parent
/ "third_party"
/ "cann-cmake"
/ "third_party"
/ f"{third_lib}.cmake"
)
(urls, repo_urls) = extract_urls_from_cmake(path)
failed_downloads.extend(down_files_native(urls))
if failed_downloads or failed_clones:
print(
"Third-party library download failed, failed urls: "
f"{failed_downloads + failed_clones}"
)
sys.exit(1)