已合并
update package script paths to cann-cmake #849
liu-wei创建于 5月14日
update package script paths to cann-cmake #849
已合并
共 2 个文件变更+2-801
| @@ -97,8 +97,8 @@ HOST_RUN_NAME="host.run" | |||
| 97 | HOST_EXTRACT_DIR="host" | 97 | HOST_EXTRACT_DIR="host" |
| 98 | MAKESELF_TARGET_DIR="build/makeself" | 98 | MAKESELF_TARGET_DIR="build/makeself" |
| 99 | RUNFILE_TARGET_DIR="build/_CPack_Packages/makeself_staging" | 99 | RUNFILE_TARGET_DIR="build/_CPack_Packages/makeself_staging" |
| 100 | -PACKAGE_SCRIPT="${WORKDIR}/scripts/package/package.py" | 100 | +PACKAGE_SCRIPT="${TOP_DIR}/open_source/cann-cmake/scripts/package/package.py" |
| 101 | -MERGE_SCRIPT="${WORKDIR}/scripts/package/common/py/merge_binary_info_config.py" | 101 | +MERGE_SCRIPT="${TOP_DIR}/open_source/cann-cmake/scripts/package/merge_binary_info_config.py" |
| 102 | PKG_OUTPUT_DIR="build/_CPack_Packages/makeself_staging" | 102 | PKG_OUTPUT_DIR="build/_CPack_Packages/makeself_staging" |
| 103 | RUN_PACKAGE_SAVE_AB_PATH=${TOP_DIR}/${RUN_PKG_SAVE_PATH} | 103 | RUN_PACKAGE_SAVE_AB_PATH=${TOP_DIR}/${RUN_PKG_SAVE_PATH} |
| 104 | ARCHIVE_RUN_DIR="${TOP_DIR}/vendor/hisi/build/delivery/${SOC}/" | 104 | ARCHIVE_RUN_DIR="${TOP_DIR}/vendor/hisi/build/delivery/${SOC}/" |
Dscripts/package/package.py+0-799
| @@ -1,799 +0,0 @@ | |||
| 1 | -#!/usr/bin/env python3 | ||
| 2 | -# -*- coding: UTF-8 -*- | ||
| 3 | -# ----------------------------------------------------------------------------------------------------------- | ||
| 4 | -# Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 5 | -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 6 | -# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 7 | -# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 8 | -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 9 | -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 10 | -# See LICENSE in the root of the software repository for the full text of the License. | ||
| 11 | -# ----------------------------------------------------------------------------------------------------------- | ||
| 12 | - | ||
| 13 | -import os | ||
| 14 | -import sys | ||
| 15 | -import subprocess | ||
| 16 | -import argparse | ||
| 17 | -import traceback | ||
| 18 | -import csv | ||
| 19 | -from argparse import Namespace | ||
| 20 | -from collections import namedtuple | ||
| 21 | -from datetime import datetime, timezone | ||
| 22 | -from functools import partial | ||
| 23 | -from itertools import chain | ||
| 24 | -from typing import Dict, Iterator, List, Set, Tuple, TextIO | ||
| 25 | -import shutil | ||
| 26 | - | ||
| 27 | -from common.py.utils import pkg_utils | ||
| 28 | -from common.py.filelist import ( | ||
| 29 | - FileItem, FileList, check_filelist, create_file_item, generate_filelist, | ||
| 30 | - get_transform_nested_path_func, | ||
| 31 | -) | ||
| 32 | -from common.py.packer import ( | ||
| 33 | - PackageName, create_makeself_pkg_params_factory, create_run_package_command | ||
| 34 | -) | ||
| 35 | -from common.py.pkg_parser import ( | ||
| 36 | - ParseOption, XmlConfig, parse_xml_config, get_cann_version_info, get_target_name | ||
| 37 | -) | ||
| 38 | -from common.py.utils.pkg_utils import ( | ||
| 39 | - CONFIG_SCRIPT_PATH, CompressError, ContainAsteriskError, DELIVERY_PATH, FAIL, | ||
| 40 | - FilelistError, GenerateFilelistError, PackageNameEmptyError, SUCC, TOP_DIR, | ||
| 41 | - UnknownOperateTypeError, path_join | ||
| 42 | -) | ||
| 43 | -from common.py.utils.funcbase import invoke, pipe | ||
| 44 | -from common.py.utils.comm_log import CommLog | ||
| 45 | - | ||
| 46 | - | ||
| 47 | -def get_comments(package_name: PackageName) -> str: | ||
| 48 | - """获取run包注释。""" | ||
| 49 | - comments = '_'.join( | ||
| 50 | - [package_name.product_name.upper(), package_name.func_name.upper(), 'RUN_PACKAGE'] | ||
| 51 | - ) | ||
| 52 | - return f'"{comments}"' | ||
| 53 | - | ||
| 54 | - | ||
| 55 | -def get_compress_cmd(pkg_args: Namespace, | ||
| 56 | - xml_config: XmlConfig) -> str: | ||
| 57 | - """获取makeself压缩命令""" | ||
| 58 | - suffix = xml_config.package_attr.get('suffix') | ||
| 59 | - if suffix == "run": | ||
| 60 | - package_name = PackageName(xml_config.package_attr, pkg_args, xml_config.version) | ||
| 61 | - factory = create_makeself_pkg_params_factory( | ||
| 62 | - package_name.getvalue(), get_comments(package_name) | ||
| 63 | - ) | ||
| 64 | - params = factory(xml_config.package_attr) | ||
| 65 | - pack_cmd, err_msg = create_run_package_command(params) | ||
| 66 | - if err_msg: | ||
| 67 | - CommLog.cilog_error(err_msg) | ||
| 68 | - CommLog.cilog_error("create_run_command failed!") | ||
| 69 | - else: | ||
| 70 | - CommLog.cilog_error("the repack type '%s' is not support!", suffix) | ||
| 71 | - sys.exit(FAIL) | ||
| 72 | - try: | ||
| 73 | - makeself_dir = os.path.join(TOP_DIR, "build/makeself.txt") | ||
| 74 | - with open(makeself_dir, 'w') as f: | ||
| 75 | - f.write(pack_cmd) | ||
| 76 | - except Exception as exception: | ||
| 77 | - CommLog.cilog_error(f"save makeself.txt failed!{str(exception)}") | ||
| 78 | - sys.exit(FAIL) | ||
| 79 | - return package_name.getvalue() | ||
| 80 | - | ||
| 81 | - | ||
| 82 | -def make_parse_option(args_: argparse.Namespace) -> ParseOption: | ||
| 83 | - """创建解析参数。""" | ||
| 84 | - | ||
| 85 | - return ParseOption( | ||
| 86 | - args_.os_arch, args_.pkg_version, | ||
| 87 | - args_.build_type, | ||
| 88 | - args_.package_check, | ||
| 89 | - args_.ext_name | ||
| 90 | - ) | ||
| 91 | - | ||
| 92 | - | ||
| 93 | -PrivatePackageOption = namedtuple( | ||
| 94 | - 'PrivatePackageOption', | ||
| 95 | - [ | ||
| 96 | - 'os_arch', 'package_suffix', 'not_in_name', 'pkg_version', 'ext_name', | ||
| 97 | - 'chip_name', 'func_name', 'version_dir', 'disable_multi_version', 'suffix' | ||
| 98 | - ] | ||
| 99 | -) | ||
| 100 | - | ||
| 101 | - | ||
| 102 | -class PackageOption(PrivatePackageOption): | ||
| 103 | - """打包配置参数。""" | ||
| 104 | - __slots__ = () # 优化内存，避免创建 __dict__ | ||
| 105 | - | ||
| 106 | - def __new__(cls, *package_option_args, **kwargs): | ||
| 107 | - return super().__new__(cls, *package_option_args, **kwargs) | ||
| 108 | - | ||
| 109 | - | ||
| 110 | -def do_copy(target_conf=None, | ||
| 111 | - delivery_dir='', | ||
| 112 | - release_dir='', | ||
| 113 | - package_name=None): | ||
| 114 | - ''' | ||
| 115 | - 功能描述：根据拷贝类型来执行文件或目录拷贝 | ||
| 116 | - 返回值：SUCC/FAIL | ||
| 117 | - ''' | ||
| 118 | - if target_conf is None: | ||
| 119 | - target_conf = {} | ||
| 120 | - target_name = get_target_name(target_conf) | ||
| 121 | - dst_path = os.path.join(release_dir, target_conf.get('dst_path', '')) | ||
| 122 | - dst_fullpath = os.path.join(dst_path, target_name) | ||
| 123 | - | ||
| 124 | - pkg_softlink = target_conf.get('pkg_softlink') | ||
| 125 | - if pkg_softlink: | ||
| 126 | - rets = [ | ||
| 127 | - create_softlink(dst_fullpath, os.path.join(release_dir, link)) | ||
| 128 | - for link in pkg_softlink | ||
| 129 | - ] | ||
| 130 | - if not all(rets): | ||
| 131 | - return FAIL | ||
| 132 | - return SUCC | ||
| 133 | - | ||
| 134 | - | ||
| 135 | -def do_chmod(target_conf=None, release_dir=''): | ||
| 136 | - """打包时设置权限 。""" | ||
| 137 | - if target_conf is None: | ||
| 138 | - target_conf = {} | ||
| 139 | - target_name = get_target_name(target_conf) | ||
| 140 | - dst_path = os.path.join(release_dir, target_conf.get('dst_path', '')) | ||
| 141 | - dst_fullpath = os.path.join(dst_path, target_name) | ||
| 142 | - pkg_mod = target_conf.get('pkg_mod', '') | ||
| 143 | - if pkg_mod: | ||
| 144 | - # 将命令拆分为列表，不使用 shell=True | ||
| 145 | - cmd_list = ['chmod', '-R', pkg_mod, dst_fullpath] | ||
| 146 | - | ||
| 147 | - try: | ||
| 148 | - result = subprocess.run(cmd_list, capture_output=True, text=True, check=False) | ||
| 149 | - status = result.returncode | ||
| 150 | - output = result.stdout + result.stderr | ||
| 151 | - | ||
| 152 | - if status != 0: # 这里的 SUCC 通常对应 0 | ||
| 153 | - CommLog.cilog_error("chmod failed! Command: %s", " ".join(cmd_list)) | ||
| 154 | - CommLog.cilog_info("%s", output) | ||
| 155 | - return FAIL | ||
| 156 | - except Exception as e: | ||
| 157 | - CommLog.cilog_error("Execute chmod exception: %s", str(e)) | ||
| 158 | - return FAIL | ||
| 159 | - | ||
| 160 | - return SUCC | ||
| 161 | - | ||
| 162 | - | ||
| 163 | -def create_softlink(source, target) -> bool: | ||
| 164 | - ''' | ||
| 165 | - 功能描述：创建软连接 | ||
| 166 | - 参数：source, target | ||
| 167 | - 返回值：成功或失败 | ||
| 168 | - ''' | ||
| 169 | - source = os.path.abspath(source.strip()) | ||
| 170 | - target = os.path.abspath(target.strip()) | ||
| 171 | - | ||
| 172 | - link_target_path = os.path.dirname(target) | ||
| 173 | - link_target_name = os.path.basename(target) | ||
| 174 | - relative_path = os.path.relpath(source, link_target_path) | ||
| 175 | - if os.path.isfile(target): | ||
| 176 | - cmd_list = ['rm', '-f', target] | ||
| 177 | - try: | ||
| 178 | - result = subprocess.run(cmd_list, capture_output=True, text=True, check=False) | ||
| 179 | - status = result.returncode | ||
| 180 | - output = result.stdout + result.stderr | ||
| 181 | - if status != 0: | ||
| 182 | - CommLog.cilog_error("Error: rm -f %s failed, %s", target, output) | ||
| 183 | - return False | ||
| 184 | - except Exception as e: | ||
| 185 | - CommLog.cilog_error("Execute rm exception: %s", str(e)) | ||
| 186 | - return False | ||
| 187 | - if os.path.isdir(target): | ||
| 188 | - CommLog.cilog_warning("Warning: %s is already a directory, skipping soft link creation.", target) | ||
| 189 | - return True | ||
| 190 | - if not os.path.exists(link_target_path): | ||
| 191 | - os.makedirs(link_target_path) | ||
| 192 | - tmp_dir = os.getcwd() | ||
| 193 | - os.chdir(link_target_path) | ||
| 194 | - os.symlink(relative_path, link_target_name) | ||
| 195 | - os.chdir(tmp_dir) | ||
| 196 | - return True | ||
| 197 | - | ||
| 198 | - | ||
| 199 | -def generate_info_content(target_conf, ext_name) -> List[str]: | ||
| 200 | - """生成info内容。""" | ||
| 201 | - | ||
| 202 | - def toolchain_llvm_config() -> Iterator[Tuple[str, str]]: | ||
| 203 | - if 'llvm' in ext_name: | ||
| 204 | - yield 'toolchain', 'llvm' | ||
| 205 | - | ||
| 206 | - content_list = [ | ||
| 207 | - f'{key}={value}' | ||
| 208 | - for key, value in chain( | ||
| 209 | - target_conf['content'].items(), toolchain_llvm_config() | ||
| 210 | - ) | ||
| 211 | - ] | ||
| 212 | - return content_list | ||
| 213 | - | ||
| 214 | - | ||
| 215 | -def generate_version_header_content(target_conf) -> Iterator[str]: | ||
| 216 | - """生成version_header内容。""" | ||
| 217 | - guard_name = target_conf['value'].replace('.', '_').upper() | ||
| 218 | - yield f'#ifndef {guard_name}' | ||
| 219 | - yield f'#define {guard_name}' | ||
| 220 | - yield '' | ||
| 221 | - for name, value in target_conf['content'].items(): | ||
| 222 | - if name.endswith('_VERSION'): | ||
| 223 | - version_infos = get_cann_version_info(name, value) | ||
| 224 | - for version_name, version_value in version_infos: | ||
| 225 | - yield f'#define {version_name} {version_value}' | ||
| 226 | - else: | ||
| 227 | - yield f'#define {name} {value}' | ||
| 228 | - yield '' | ||
| 229 | - yield f'#endif /* {guard_name} */' | ||
| 230 | - yield '' | ||
| 231 | - | ||
| 232 | - | ||
| 233 | -def generate_customized_file(target_conf, ext_name): | ||
| 234 | - filepath = os.path.join(TOP_DIR, "build", target_conf.get('value')) | ||
| 235 | - | ||
| 236 | - generator = target_conf.get('generator', 'info') | ||
| 237 | - if generator == 'version_header': | ||
| 238 | - content_list = generate_version_header_content(target_conf) | ||
| 239 | - else: | ||
| 240 | - content_list = generate_info_content(target_conf, ext_name) | ||
| 241 | - | ||
| 242 | - file_content = '\n'.join(content_list) | ||
| 243 | - try: | ||
| 244 | - with open(filepath, 'w') as file: | ||
| 245 | - file.write(file_content) | ||
| 246 | - except Exception as ex: | ||
| 247 | - CommLog.cilog_error(f"generate customized file {filepath} failed: {ex}!") | ||
| 248 | - return FAIL | ||
| 249 | - | ||
| 250 | - return SUCC | ||
| 251 | - | ||
| 252 | - | ||
| 253 | -def get_module(target_config) -> str: | ||
| 254 | - """获取配置模块。""" | ||
| 255 | - module = target_config.get('module', 'NA') | ||
| 256 | - return module if module else 'NA' | ||
| 257 | - | ||
| 258 | - | ||
| 259 | -def get_operation(operation, target_config) -> str: | ||
| 260 | - """获取操作类型。""" | ||
| 261 | - if operation in ('copy', 'move') and target_config.get('entity') == 'true': | ||
| 262 | - return 'copy_entity' | ||
| 263 | - return operation | ||
| 264 | - | ||
| 265 | - | ||
| 266 | -def get_permission(target_config) -> str: | ||
| 267 | - """获取配置权限。""" | ||
| 268 | - return target_config.get('install_mod', 'NA') | ||
| 269 | - | ||
| 270 | - | ||
| 271 | -def get_owner_group(target_config) -> str: | ||
| 272 | - """获取配置属主。""" | ||
| 273 | - # install_own的可能值为$username:$usergroup | ||
| 274 | - # 防止变量在install_common_parser.sh中，被eval展开，添加\转义$ | ||
| 275 | - # 由于awk会消耗1个\，所以需要2个转义符 | ||
| 276 | - return target_config.get('install_own', 'NA').replace('$', '\\\\$') | ||
| 277 | - | ||
| 278 | - | ||
| 279 | -def get_install_type(target_config) -> str: | ||
| 280 | - """获取安装类型。""" | ||
| 281 | - return target_config.get('install_type', 'NA') | ||
| 282 | - | ||
| 283 | - | ||
| 284 | -def get_softlink(target_config) -> List[str]: | ||
| 285 | - """获取配置软链。""" | ||
| 286 | - softlink_str = target_config.get('install_softlink') | ||
| 287 | - if not softlink_str: | ||
| 288 | - return [] | ||
| 289 | - return softlink_str.split(';') | ||
| 290 | - | ||
| 291 | - | ||
| 292 | -def get_feature(target_config) -> Set[str]: | ||
| 293 | - """获取配置特性。""" | ||
| 294 | - return target_config['feature'] | ||
| 295 | - | ||
| 296 | - | ||
| 297 | -def get_chip(target_config) -> Set[str]: | ||
| 298 | - """获取配置芯片。""" | ||
| 299 | - return target_config['chip'] | ||
| 300 | - | ||
| 301 | - | ||
| 302 | -def get_configurable(target_config) -> str: | ||
| 303 | - """获取配置是否为配置文件。""" | ||
| 304 | - return target_config.get('configurable', 'FALSE') | ||
| 305 | - | ||
| 306 | - | ||
| 307 | -def get_hash_value(target_config) -> str: | ||
| 308 | - """获取配置哈希值。""" | ||
| 309 | - return target_config.get('hash', 'NA') | ||
| 310 | - | ||
| 311 | - | ||
| 312 | -def get_block(target_config) -> str: | ||
| 313 | - """获取配置块信息。""" | ||
| 314 | - return target_config.get('name', 'NA') | ||
| 315 | - | ||
| 316 | - | ||
| 317 | -def get_pkg_inner_softlink(target_config) -> List[str]: | ||
| 318 | - """获取配置包内软链。""" | ||
| 319 | - softlink_str = target_config.get('pkg_inner_softlink') | ||
| 320 | - if not softlink_str: | ||
| 321 | - return [] | ||
| 322 | - return softlink_str.split(';') | ||
| 323 | - | ||
| 324 | - | ||
| 325 | -def validate_path_consistency(target_config, target_name): | ||
| 326 | - """验证 dst_path 和 install_path 一致性，不一致则抛出异常。""" | ||
| 327 | - dst_path_val = target_config.get('dst_path', '') | ||
| 328 | - install_path_val = target_config.get('install_path', '') | ||
| 329 | - | ||
| 330 | - if dst_path_val != install_path_val: | ||
| 331 | - value_path = target_config.get('value', 'unknown') | ||
| 332 | - CommLog.cilog_error( | ||
| 333 | - f"Configuration Error: 'dst_path' MUST be equal to 'install_path'.\n" | ||
| 334 | - f" - Current dst_path: {dst_path_val}\n" | ||
| 335 | - f" - Current install_path: {install_path_val}\n" | ||
| 336 | - ) | ||
| 337 | - raise GenerateFilelistError( | ||
| 338 | - f"dst_path ({dst_path_val}) does not match install_path ({install_path_val})" | ||
| 339 | - ) | ||
| 340 | - | ||
| 341 | - | ||
| 342 | -def parse_install_info(infos: List, operate_type, filter_key) -> Iterator[FileItem]: | ||
| 343 | - """根据配置解析生成安装信息。""" | ||
| 344 | - for target_config in infos: | ||
| 345 | - target_name = get_target_name(target_config) | ||
| 346 | - if target_config.get("optional") == 'true' and operate_type in ('copy', 'move'): | ||
| 347 | - path = os.path.join(TOP_DIR, DELIVERY_PATH, target_config.get('dst_path')) | ||
| 348 | - vaule = os.path.join(TOP_DIR, DELIVERY_PATH, target_config.get('dst_path'), target_name) | ||
| 349 | - if not os.path.exists(path): | ||
| 350 | - continue | ||
| 351 | - if not os.path.exists(vaule): | ||
| 352 | - continue | ||
| 353 | - if operate_type in ('copy', 'move'): | ||
| 354 | - relative_path_in_pkg = os.path.join(target_config.get('dst_path'), target_name) | ||
| 355 | - relative_install_path = path_join(target_config.get('install_path'), target_name) | ||
| 356 | - is_dir = target_config.get('is_dir', False) | ||
| 357 | - # 验证dst_path和install_path的一致性 | ||
| 358 | - validate_path_consistency(target_config, target_name) | ||
| 359 | - elif operate_type == 'mkdir': | ||
| 360 | - relative_path_in_pkg = 'NA' | ||
| 361 | - relative_install_path = target_config.get('value') | ||
| 362 | - is_dir = False | ||
| 363 | - elif operate_type == 'del': | ||
| 364 | - relative_path_in_pkg = 'NA' | ||
| 365 | - relative_install_path = path_join(target_config.get('install_path'), target_name) | ||
| 366 | - is_dir = False | ||
| 367 | - else: | ||
| 368 | - raise UnknownOperateTypeError(f"unknown operate type {operate_type}") | ||
| 369 | - | ||
| 370 | - if relative_install_path is None: | ||
| 371 | - continue | ||
| 372 | - | ||
| 373 | - install_type = get_install_type(target_config) | ||
| 374 | - if any(key in install_type for key in filter_key): | ||
| 375 | - is_in_docker = 'TRUE' | ||
| 376 | - else: | ||
| 377 | - is_in_docker = 'FALSE' | ||
| 378 | - | ||
| 379 | - file_item = create_file_item( | ||
| 380 | - get_module(target_config), | ||
| 381 | - get_operation(operate_type, target_config), | ||
| 382 | - relative_path_in_pkg, | ||
| 383 | - relative_install_path, | ||
| 384 | - is_in_docker, | ||
| 385 | - get_permission(target_config), | ||
| 386 | - get_owner_group(target_config), | ||
| 387 | - install_type, | ||
| 388 | - get_softlink(target_config), | ||
| 389 | - get_feature(target_config), 'N', | ||
| 390 | - get_configurable(target_config), | ||
| 391 | - get_hash_value(target_config), | ||
| 392 | - get_block(target_config), | ||
| 393 | - get_pkg_inner_softlink(target_config), | ||
| 394 | - get_chip(target_config), | ||
| 395 | - is_dir, | ||
| 396 | - ) | ||
| 397 | - | ||
| 398 | - yield file_item | ||
| 399 | - | ||
| 400 | - | ||
| 401 | -def execute_repack_process(xmlconfig: XmlConfig, | ||
| 402 | - delivery_dir: str, | ||
| 403 | - pkg_args: Namespace, | ||
| 404 | - package_name: PackageName = None, | ||
| 405 | - package_option: PackageOption = None): | ||
| 406 | - """ | ||
| 407 | - 功能描述: 执行打包流程(拷贝--->签名--->打包) | ||
| 408 | - 返回值: SUCC/FAIL | ||
| 409 | - """ | ||
| 410 | - status = SUCC | ||
| 411 | - release_dir = delivery_dir | ||
| 412 | - # 生成自定义文件 | ||
| 413 | - for item in xmlconfig.generate_infos: | ||
| 414 | - if generate_customized_file(item, package_option.ext_name): | ||
| 415 | - return FAIL | ||
| 416 | - | ||
| 417 | - for item in chain(xmlconfig.package_content_list, xmlconfig.move_content_list): | ||
| 418 | - #拷贝文件 | ||
| 419 | - if do_copy(item, | ||
| 420 | - delivery_dir, | ||
| 421 | - release_dir, | ||
| 422 | - package_name): | ||
| 423 | - status = FAIL | ||
| 424 | - continue | ||
| 425 | - if status != SUCC: | ||
| 426 | - return FAIL | ||
| 427 | - | ||
| 428 | - # 校验包中文件或目录大小 | ||
| 429 | - if pkg_args.check_size == "True": | ||
| 430 | - limit_list, tag = processing_csv_file( | ||
| 431 | - release_dir, package_name.func_name, package_name.chip_name, pkg_args.build_type | ||
| 432 | - ) | ||
| 433 | - if not tag: | ||
| 434 | - return FAIL | ||
| 435 | - if limit_list: | ||
| 436 | - abspath = os.path.abspath(release_dir) | ||
| 437 | - replace_path = abspath + "/" | ||
| 438 | - result = check_add_dir(replace_path, abspath, limit_list) | ||
| 439 | - if not result: | ||
| 440 | - return FAIL | ||
| 441 | - try: | ||
| 442 | - package_name = get_compress_cmd(pkg_args, xmlconfig) | ||
| 443 | - except CompressError: | ||
| 444 | - return FAIL | ||
| 445 | - | ||
| 446 | - CommLog.cilog_info("package %s generate filelist.csv and makeself cmd successfully!", | ||
| 447 | - package_name) | ||
| 448 | - return SUCC | ||
| 449 | - | ||
| 450 | - | ||
| 451 | -def check_path_is_conflict(xml_config): | ||
| 452 | - """ | ||
| 453 | - 功能描述: 检查打包时安装路径与软连接路径是否冲突 | ||
| 454 | - 参数: xml_config | ||
| 455 | - 返回值: SUCC/FAIL | ||
| 456 | - """ | ||
| 457 | - install_path_list = set() | ||
| 458 | - pkg_softlink_list = set() | ||
| 459 | - for item in xml_config.package_content_list: | ||
| 460 | - value_list = item.get('value').split('/') | ||
| 461 | - target_name = value_list[-1] if value_list[-1] else value_list[-2] | ||
| 462 | - if item.get('install_path'): | ||
| 463 | - install_path_list.add( | ||
| 464 | - os.path.join(item['install_path'], target_name) | ||
| 465 | - ) | ||
| 466 | - if item.get('pkg_inner_softlink'): | ||
| 467 | - pkg_softlink = item.get('pkg_inner_softlink') | ||
| 468 | - pkg_softlink_list.add(pkg_softlink) | ||
| 469 | - if install_path_list & pkg_softlink_list: | ||
| 470 | - CommLog.cilog_info('intersection:{}'.format(install_path_list & pkg_softlink_list)) | ||
| 471 | - CommLog.cilog_info('path conflicting: pkg_inner_softlink dir equals install_path!!') | ||
| 472 | - return FAIL | ||
| 473 | - return SUCC | ||
| 474 | - | ||
| 475 | - | ||
| 476 | -def checksum_value(limit_value, release_dir): | ||
| 477 | - """ | ||
| 478 | - 功能描叙: 校验传入的文件或目录大小是否合格 | ||
| 479 | - 参数: | ||
| 480 | - limit_value: limit.csv中的一行数据如[compiler/bin, 3976, 110%] | ||
| 481 | - 返回值: True/False | ||
| 482 | - """ | ||
| 483 | - path = os.path.join(release_dir, limit_value[1]) | ||
| 484 | - if len(limit_value) >= 7: | ||
| 485 | - try: | ||
| 486 | - max_value = int(limit_value[4]) | ||
| 487 | - except ValueError: | ||
| 488 | - CommLog.cilog_error("{0} configuration is not standard., Please check limit.csv.".format(path)) | ||
| 489 | - return True | ||
| 490 | - else: | ||
| 491 | - CommLog.cilog_error("{0} configuration is less than four, Please check limit.csv.".format(path)) | ||
| 492 | - return True | ||
| 493 | - if not os.path.exists(path): | ||
| 494 | - CommLog.cilog_warning("{0} doesn't exist, Please check limit.csv.".format(path)) | ||
| 495 | - return True | ||
| 496 | - size = 0 | ||
| 497 | - for root, dirs, files in os.walk(path): | ||
| 498 | - size += os.path.getsize(root) | ||
| 499 | - for f in files: | ||
| 500 | - filepath = os.path.join(root, f) | ||
| 501 | - if os.path.islink(filepath): | ||
| 502 | - continue | ||
| 503 | - if not os.path.exists(filepath): | ||
| 504 | - continue | ||
| 505 | - size += os.path.getsize(os.path.join(root, f)) | ||
| 506 | - if size == 0: | ||
| 507 | - size = os.path.getsize(path) | ||
| 508 | - if size > max_value * 1024: | ||
| 509 | - CommLog.cilog_error(f"\n{path} size {size} bytes exceeds maximum {max_value * 1024} bytes") | ||
| 510 | - return False | ||
| 511 | - return True | ||
| 512 | - | ||
| 513 | - | ||
| 514 | -def processing_csv_file(release_dir, package_name, chip_name, build_type): | ||
| 515 | - """ | ||
| 516 | - 功能描叙: 处理limit.csv文件数据 | ||
| 517 | - 返回值: [],True/[],False | ||
| 518 | - """ | ||
| 519 | - ret = True | ||
| 520 | - limit_list = [] | ||
| 521 | - product = os.path.basename(os.path.dirname(release_dir)) | ||
| 522 | - limit_path = os.path.join(pkg_utils.TOP_SOURCE_DIR, CONFIG_SCRIPT_PATH, "common/limit.csv") | ||
| 523 | - if not os.path.exists(limit_path): | ||
| 524 | - CommLog.cilog_warning("{0} doesn't exist.".format(limit_path)) | ||
| 525 | - return limit_list, ret | ||
| 526 | - with open(limit_path, "r") as file: | ||
| 527 | - reader = csv.reader(file) | ||
| 528 | - next(reader) | ||
| 529 | - for data in reader: | ||
| 530 | - if not data: | ||
| 531 | - CommLog.cilog_warning("The limit.csv file contains empty lines.") | ||
| 532 | - continue | ||
| 533 | - if is_match_line(package_name, chip_name, product, build_type, data): | ||
| 534 | - if data[1][-1] == "/": | ||
| 535 | - limit_list.append(data[1][:-1]) | ||
| 536 | - else: | ||
| 537 | - limit_list.append(data[1]) | ||
| 538 | - res = checksum_value(data, release_dir) | ||
| 539 | - if not res: | ||
| 540 | - ret = False | ||
| 541 | - return limit_list, ret | ||
| 542 | - | ||
| 543 | - | ||
| 544 | -def is_match_line(package_name, chip_name, product, build_type, data): | ||
| 545 | - return package_name == data[0] and chip_name == data[5] and product == data[6] and build_type == data[7].lower() | ||
| 546 | - | ||
| 547 | - | ||
| 548 | -def check_add_dir(package_path, dirs, limit_list, ret=True): | ||
| 549 | - """ | ||
| 550 | - 功能描述: 校验新增目录 | ||
| 551 | - 参数: path, limit_list | ||
| 552 | - 返回值: False/True | ||
| 553 | - """ | ||
| 554 | - for limit_path in limit_list: | ||
| 555 | - if dirs == os.path.join(os.path.split(dirs)[0], limit_path): | ||
| 556 | - return ret | ||
| 557 | - for dir_file in os.listdir(dirs): | ||
| 558 | - path = os.path.join(dirs, dir_file) | ||
| 559 | - relative_path = path.replace(package_path, "") | ||
| 560 | - if os.path.isfile(path) and relative_path not in limit_list: | ||
| 561 | - CommLog.cilog_error("{0} is not in limit.csv file and is newly added.".format(path)) | ||
| 562 | - ret = False | ||
| 563 | - elif os.path.isdir(path) and relative_path not in limit_list: | ||
| 564 | - ret = check_add_dir(package_path, path, limit_list, ret) | ||
| 565 | - return ret | ||
| 566 | - | ||
| 567 | - | ||
| 568 | -def gen_file_install_list(xml_config: XmlConfig, | ||
| 569 | - filter_key) -> Tuple[FileList, FileList]: | ||
| 570 | - """生成filelist列表。""" | ||
| 571 | - file_install_list = [] | ||
| 572 | - | ||
| 573 | - dir_filelist = parse_install_info( | ||
| 574 | - xml_config.dir_install_list, 'mkdir', filter_key | ||
| 575 | - ) | ||
| 576 | - move_filelist = parse_install_info( | ||
| 577 | - xml_config.move_content_list, 'move', filter_key | ||
| 578 | - ) | ||
| 579 | - pkg_filelist = parse_install_info( | ||
| 580 | - xml_config.package_content_list, 'copy', filter_key | ||
| 581 | - ) | ||
| 582 | - gen_filelist = parse_install_info( | ||
| 583 | - xml_config.generate_infos, 'copy', filter_key | ||
| 584 | - ) | ||
| 585 | - # file_info中配置为文件夹，这里是被展开的文件,则需要单独删除 | ||
| 586 | - del_filelist = parse_install_info( | ||
| 587 | - xml_config.expand_content_list, 'del', filter_key | ||
| 588 | - ) | ||
| 589 | - collect_filelist = list(chain(dir_filelist, move_filelist, pkg_filelist, gen_filelist)) | ||
| 590 | - collect_filelist = list(xml_config.packer_config.fill_is_common_path(collect_filelist)) | ||
| 591 | - all_filelist = list(chain(collect_filelist, del_filelist)) | ||
| 592 | - for file_item in all_filelist: | ||
| 593 | - file_install_list.append(file_item) | ||
| 594 | - | ||
| 595 | - return file_install_list, [] | ||
| 596 | - | ||
| 597 | - | ||
| 598 | -def generate_filelist_file_by_xml_config(xml_config: XmlConfig, | ||
| 599 | - filter_key: List[str], | ||
| 600 | - package_check: bool): | ||
| 601 | - """生成文件列表文件。""" | ||
| 602 | - check_move = xml_config.package_attr.get('use_move', False) | ||
| 603 | - transform_nested_path_func = get_transform_nested_path_func( | ||
| 604 | - xml_config.package_attr.get('parallel') or check_move | ||
| 605 | - ) | ||
| 606 | - check_features = xml_config.package_attr.get('check_features', False) | ||
| 607 | - | ||
| 608 | - file_install_list, [] = invoke( | ||
| 609 | - pipe( | ||
| 610 | - gen_file_install_list, | ||
| 611 | - partial(map, transform_nested_path_func), | ||
| 612 | - tuple, | ||
| 613 | - ), | ||
| 614 | - xml_config, filter_key | ||
| 615 | - ) | ||
| 616 | - generate_filelist(file_install_list, 'filelist.csv') | ||
| 617 | - # 先生成再检查，有利于问题定位 | ||
| 618 | - check_filelist(file_install_list, check_features, check_move) | ||
| 619 | - | ||
| 620 | - | ||
| 621 | -def get_pkg_xml_relative_path(pkg_args: Namespace) -> str: | ||
| 622 | - """获取包配置文件相对路径。""" | ||
| 623 | - | ||
| 624 | - def parts(): | ||
| 625 | - yield CONFIG_SCRIPT_PATH | ||
| 626 | - yield pkg_args.pkg_name | ||
| 627 | - if pkg_args.chip_scenes: | ||
| 628 | - yield pkg_args.chip_scenes | ||
| 629 | - # 可以通過build_rule指定xml_file，而且优先级高于默认值 | ||
| 630 | - if pkg_args.xml_file: | ||
| 631 | - yield pkg_args.xml_file | ||
| 632 | - else: | ||
| 633 | - yield f'{pkg_args.pkg_name}.xml' | ||
| 634 | - | ||
| 635 | - return os.path.join(*parts()) | ||
| 636 | - | ||
| 637 | - | ||
| 638 | -def write_config_inc_var(name: str, package_attr: Dict, file: TextIO): | ||
| 639 | - """向config.inc文件写入变量。""" | ||
| 640 | - if name in package_attr: | ||
| 641 | - value = str(package_attr[name]).lower() | ||
| 642 | - file.write(f"{name.upper()}={value}\n") | ||
| 643 | - | ||
| 644 | - | ||
| 645 | -def generate_config_inc(package_attr: Dict): | ||
| 646 | - """生成config.inc文件。""" | ||
| 647 | - if 'parallel' not in package_attr and 'parallel_limit' not in package_attr and 'use_move' not in package_attr: | ||
| 648 | - return | ||
| 649 | - year = datetime.now(timezone.utc).year | ||
| 650 | - config_inc = os.path.join(TOP_DIR, "build", 'config.inc') | ||
| 651 | - header = [ | ||
| 652 | - '#!/bin/sh\n', | ||
| 653 | - '#----------------------------------------------------------------------------\n', | ||
| 654 | - f'# Copyright Huawei Technologies Co., Ltd. 2023-{year}. All rights reserved.\n', | ||
| 655 | - '#----------------------------------------------------------------------------\n', | ||
| 656 | - '\n', | ||
| 657 | - ] | ||
| 658 | - if os.path.isfile(config_inc): | ||
| 659 | - os.chmod(config_inc, 0o700) | ||
| 660 | - with open(config_inc, 'w', encoding='utf-8') as file: | ||
| 661 | - file.writelines(header) | ||
| 662 | - write_config_inc_var('parallel', package_attr, file) | ||
| 663 | - write_config_inc_var('parallel_limit', package_attr, file) | ||
| 664 | - write_config_inc_var('use_move', package_attr, file) | ||
| 665 | - | ||
| 666 | - os.chmod(config_inc, 0o500) | ||
| 667 | - | ||
| 668 | - | ||
| 669 | -def main(pkg_name='', xml_file='', main_args=None): | ||
| 670 | - """ | ||
| 671 | - 功能描述: 执行打包流程(解析配置--->生成文件列表--->执行拷贝/打包动作) | ||
| 672 | - 参数: pkg_name, os_arch, type | ||
| 673 | - 返回值: SUCCESS/FAIL | ||
| 674 | - """ | ||
| 675 | - delivery_dir = os.path.join(TOP_DIR, DELIVERY_PATH) | ||
| 676 | - if not os.path.exists(delivery_dir): | ||
| 677 | - return FAIL | ||
| 678 | - | ||
| 679 | - config_relative_path = get_pkg_xml_relative_path(main_args) | ||
| 680 | - pkg_xml_file = os.path.join(pkg_utils.TOP_SOURCE_DIR, config_relative_path) | ||
| 681 | - parse_option = make_parse_option(main_args) | ||
| 682 | - | ||
| 683 | - try: | ||
| 684 | - xml_config = parse_xml_config( | ||
| 685 | - pkg_xml_file, delivery_dir, parse_option, main_args | ||
| 686 | - ) | ||
| 687 | - except ContainAsteriskError as ex: | ||
| 688 | - CommLog.cilog_error(f"Value contain '*' in {config_relative_path}. value is '{ex.value}'.") | ||
| 689 | - return FAIL | ||
| 690 | - | ||
| 691 | - if pkg_name in ['driver', 'firmware']: | ||
| 692 | - filter_key = ['all', 'docker'] | ||
| 693 | - elif pkg_name in ['aicpu_kernels_device', 'aicpu_kernels_host']: | ||
| 694 | - filter_key = [] | ||
| 695 | - else: | ||
| 696 | - filter_key = ['all', 'run'] | ||
| 697 | - | ||
| 698 | - # 生成filelist.csv安装列表文件 | ||
| 699 | - try: | ||
| 700 | - generate_filelist_file_by_xml_config( | ||
| 701 | - xml_config, filter_key, | ||
| 702 | - main_args.package_check or xml_config.package_attr.get('package_check') | ||
| 703 | - ) | ||
| 704 | - except PackageNameEmptyError: | ||
| 705 | - CommLog.cilog_error(f'package name is empty in {xml_file}, please check it') | ||
| 706 | - return FAIL | ||
| 707 | - except GenerateFilelistError as ex: | ||
| 708 | - CommLog.cilog_error(f'generate filelist {ex.filename} failed!', ) | ||
| 709 | - return FAIL | ||
| 710 | - except FilelistError as ex: | ||
| 711 | - CommLog.cilog_error('check filelist error! %s', str(ex)) | ||
| 712 | - return FAIL | ||
| 713 | - | ||
| 714 | - generate_config_inc(xml_config.package_attr) | ||
| 715 | - | ||
| 716 | - package_option = PackageOption( | ||
| 717 | - main_args.os_arch, main_args.package_suffix, main_args.not_in_name, main_args.pkg_version, main_args.ext_name, | ||
| 718 | - chip_name=main_args.chip_name, func_name=main_args.func_name, version_dir=main_args.version_dir, | ||
| 719 | - disable_multi_version=main_args.disable_multi_version, suffix=main_args.suffix) | ||
| 720 | - | ||
| 721 | - package_name = PackageName(xml_config.package_attr, main_args, xml_config.version) | ||
| 722 | - | ||
| 723 | - # 检查install_path与pkg_inner_softlink路径是否冲突，若冲突则报错 | ||
| 724 | - if check_path_is_conflict(xml_config) == FAIL: | ||
| 725 | - return FAIL | ||
| 726 | - | ||
| 727 | - # 生成打包命令 | ||
| 728 | - return execute_repack_process(xml_config, delivery_dir, main_args, | ||
| 729 | - package_name=package_name, package_option=package_option) | ||
| 730 | - | ||
| 731 | - | ||
| 732 | -def args_parse(): | ||
| 733 | - """ | ||
| 734 | - 功能描述 : 脚本入参解析 | ||
| 735 | - 参数 : 调用脚本的传参 | ||
| 736 | - 返回值 : 解析后的参数值 | ||
| 737 | - """ | ||
| 738 | - parser = argparse.ArgumentParser( | ||
| 739 | - description='This script is for package repack processing.') | ||
| 740 | - parser.add_argument('-c', '--chip_scenes', metavar='chip_scenes', required=False, dest='chip_scenes', nargs='?', | ||
| 741 | - const='', | ||
| 742 | - default='', help='This parameter define chip id for package.') | ||
| 743 | - parser.add_argument('-n', '--pkg_name', metavar='pkg_name', required=False, | ||
| 744 | - help='This parameter define pkg_name for config_xml.') | ||
| 745 | - parser.add_argument('-o', '--os_arch', metavar='os_arch', required=False, dest='os_arch', nargs='?', const='', | ||
| 746 | - default=None, help="This parameter define the package's os_arch") | ||
| 747 | - parser.add_argument('-t', '--type', metavar='type', required=False, dest='type', nargs='?', const='', | ||
| 748 | - default='repack', help="This parameter define this script's function") | ||
| 749 | - parser.add_argument('-i', '--not_in_name', metavar='not_in_name', required=False, dest='not_in_name', nargs='?', | ||
| 750 | - const='', | ||
| 751 | - default='', help="This parameter define the package's name not contain the element") | ||
| 752 | - parser.add_argument('-v', '--pkg_version', metavar='pkg_version', required=False, dest='pkg_version', nargs='?', | ||
| 753 | - const='', | ||
| 754 | - default='', help="This parameter define the version for package.") | ||
| 755 | - parser.add_argument('-e', '--ext_name', metavar='ext_name', required=False, dest='ext_name', nargs='?', const='', | ||
| 756 | - default='', help="This parameter define the package's ext_name") | ||
| 757 | - parser.add_argument('--package_suffix', nargs='?', const='none', | ||
| 758 | - default='none', help="This parameter define the package suffix, debug or none") | ||
| 759 | - parser.add_argument('--suffix', metavar='suffix', required=False, dest='suffix', nargs='?', const='', | ||
| 760 | - default=None, help="This parameter define the package suffix, for example such as tar.gz") | ||
| 761 | - parser.add_argument('-b', '--build_type', metavar='build_type', required=False, dest='build_type', nargs='?', | ||
| 762 | - const='', | ||
| 763 | - default='debug', help="This parameter define release type of package") | ||
| 764 | - parser.add_argument('-x', '--xml', metavar='xml_file', required=False, dest='xml_file', nargs='?', const='', | ||
| 765 | - default='', help="This parameter define xml file") | ||
| 766 | - parser.add_argument('--chip_name', metavar='chip_name', required=False, dest='chip_name', nargs='?', const=None, | ||
| 767 | - default=None, | ||
| 768 | - help="This parameter define package chip name, has higher priority than chip name in xml") | ||
| 769 | - parser.add_argument('--func_name', metavar='func_name', required=False, dest='func_name', nargs='?', const=None, | ||
| 770 | - default=None, | ||
| 771 | - help="This parameter define package func name, has higher priority than func name in xml") | ||
| 772 | - parser.add_argument('--source_root', metavar='source_root', required=False, dest='source_root', nargs='?', const='', | ||
| 773 | - help='source root dir.') | ||
| 774 | - parser.add_argument('--version_dir', nargs='?', const='', default='', help='Set version dir.') | ||
| 775 | - parser.add_argument('--tag', metavar='tag', nargs='?', const='', default='') | ||
| 776 | - parser.add_argument('--disable-multi-version', action='store_true', help='Disable multi version.') | ||
| 777 | - # 检查打包配置 | ||
| 778 | - parser.add_argument('--package-check', action='store_true', help='check package config.') | ||
| 779 | - parser.add_argument('--check_size', nargs='?', const='', default='', help="Check the size of a file or directory.") | ||
| 780 | - parser.add_argument('--pkg-name-style', metavar='pkg_name_style', default='common', help='Package name style.') | ||
| 781 | - return parser.parse_args() | ||
| 782 | - | ||
| 783 | - | ||
| 784 | -if __name__ == "__main__": | ||
| 785 | - CommLog.cilog_info("%s", " ".join(sys.argv)) | ||
| 786 | - args = args_parse() | ||
| 787 | - try: | ||
| 788 | - if args.source_root: | ||
| 789 | - pkg_utils.TOP_SOURCE_DIR = args.source_root | ||
| 790 | - if args.build_type == '': | ||
| 791 | - args.build_type = 'debug' | ||
| 792 | - else: | ||
| 793 | - args.build_type = args.build_type.lower() | ||
| 794 | - status = main(args.pkg_name, args.xml_file, main_args=args) | ||
| 795 | - except Exception as e: | ||
| 796 | - CommLog.cilog_error("exception occurred (%s)!", e) | ||
| 797 | - CommLog.cilog_info("%s", traceback.format_exc()) | ||
| 798 | - status = FAIL | ||
| 799 | - sys.exit(status) | ||