已合并
local exec CI build #2247
丛吉钰创建于 3月2日
local exec CI build #2247
已合并
共 4 个文件变更+220-7
| @@ -51,7 +51,6 @@ cmake-build-release | |||
| 51 | *.tmp | 51 | *.tmp |
| 52 | *.temp | 52 | *.temp |
| 53 | 53 | ||
| 54 | - | ||
| 55 | # UT report file | 54 | # UT report file |
| 56 | ut_report.txt | 55 | ut_report.txt |
| 57 | 56 | ||
| @@ -64,3 +63,8 @@ makeself | |||
| 64 | # tbetoolkits log files | 63 | # tbetoolkits log files |
| 65 | tbetoolkits-*.log | 64 | tbetoolkits-*.log |
| 66 | *_result.csv | 65 | *_result.csv |
| 66 | + | ||
| 67 | +# local build package | ||
| 68 | +single/ | ||
| 69 | +scripts/ci/tmp/ | ||
| 70 | +pr_filelist.txt | ||
| @@ -39,6 +39,18 @@ do | |||
| 39 | fi | 39 | fi |
| 40 | done | 40 | done |
| 41 | 41 | ||
| 42 | +cov="--cov" | ||
| 43 | +while [[ $# -gt 0 ]]; do | ||
| 44 | + case "$1" in | ||
| 45 | + --no_cov) | ||
| 46 | + cov="" | ||
| 47 | + shift | ||
| 48 | + ;; | ||
| 49 | + *) | ||
| 50 | + shift | ||
| 51 | + ;; | ||
| 52 | + esac | ||
| 53 | +done | ||
| 42 | 54 | ||
| 43 | #搜索变更文件清单，将存在于valid_dirs清单中的算子保存到ops_name中，路径中需要出现'op_kernel' | 55 | #搜索变更文件清单，将存在于valid_dirs清单中的算子保存到ops_name中，路径中需要出现'op_kernel' |
| 44 | 56 | ||
| @@ -83,8 +95,8 @@ for name in "${ops_name[@]}" | |||
| 83 | do | 95 | do |
| 84 | for soc_version in "${supportedSocVersion[@]}" | 96 | for soc_version in "${supportedSocVersion[@]}" |
| 85 | do | 97 | do |
| 86 | - echo "[EXECUTE_COMMAND] bash build.sh -u --opkernel --ops=$name --cov --soc=$soc_version" | 98 | + echo "[EXECUTE_COMMAND] bash build.sh -u --opkernel --ops=$name ${cov} --soc=$soc_version" |
| 87 | - bash build.sh -u --opkernel --ops=$name --cov --soc=$soc_version --cann_3rd_lib_path=${ASCEND_3RD_LIB_PATH} -j16 | 99 | + bash build.sh -u --opkernel --ops=$name ${cov} --soc=$soc_version --cann_3rd_lib_path=${ASCEND_3RD_LIB_PATH} -j16 |
| 88 | status=$? | 100 | status=$? |
| 89 | if [ $status -ne 0 ]; then | 101 | if [ $status -ne 0 ]; then |
| 90 | echo "${name} kernel ut fail" | 102 | echo "${name} kernel ut fail" |
| @@ -224,25 +224,24 @@ echo "related op: ${builtin_ops_name[*]}" | |||
| 224 | echo "related experimental op: ${experimental_ops_name[*]}" | 224 | echo "related experimental op: ${experimental_ops_name[*]}" |
| 225 | echo "need build all: ${build_all}" | 225 | echo "need build all: ${build_all}" |
| 226 | 226 | ||
| 227 | -cann_3rd_lib_path="/home/jenkins/opensource" | ||
| 228 | a5_soc="ascend950" | 227 | a5_soc="ascend950" |
| 229 | 228 | ||
| 230 | if [[ ${#builtin_ops_name[@]} -gt 0 && "$force_jit" = "false" ]]; then | 229 | if [[ ${#builtin_ops_name[@]} -gt 0 && "$force_jit" = "false" ]]; then |
| 231 | builtin_ops_str=$(IFS=,; echo "${builtin_ops_name[*]}") | 230 | builtin_ops_str=$(IFS=,; echo "${builtin_ops_name[*]}") |
| 232 | - build_cmd="bash build.sh --pkg --ops=$builtin_ops_str --soc=$a5_soc -j16 --cann_3rd_lib_path=$cann_3rd_lib_path" | 231 | + build_cmd="bash build.sh --pkg --ops=$builtin_ops_str --soc=$a5_soc -j16 --cann_3rd_lib_path=${ASCEND_3RD_LIB_PATH}" |
| 233 | run_build_command "$build_cmd" | 232 | run_build_command "$build_cmd" |
| 234 | execute_run_file "custom" | 233 | execute_run_file "custom" |
| 235 | fi | 234 | fi |
| 236 | 235 | ||
| 237 | if [[ ${#experimental_ops_name[@]} -gt 0 && "$force_jit" = "false" ]]; then | 236 | if [[ ${#experimental_ops_name[@]} -gt 0 && "$force_jit" = "false" ]]; then |
| 238 | experimental_ops_str=$(IFS=,; echo "${experimental_ops_name[*]}") | 237 | experimental_ops_str=$(IFS=,; echo "${experimental_ops_name[*]}") |
| 239 | - build_cmd="bash build.sh --pkg --experimental --ops=$experimental_ops_str --soc=$a5_soc -j16 --cann_3rd_lib_path=$cann_3rd_lib_path" | 238 | + build_cmd="bash build.sh --pkg --experimental --ops=$experimental_ops_str --soc=$a5_soc -j16 --cann_3rd_lib_path=${ASCEND_3RD_LIB_PATH}" |
| 240 | run_build_command "$build_cmd" | 239 | run_build_command "$build_cmd" |
| 241 | execute_run_file "custom" | 240 | execute_run_file "custom" |
| 242 | fi | 241 | fi |
| 243 | 242 | ||
| 244 | if [ ${build_all} -eq 1 ]; then | 243 | if [ ${build_all} -eq 1 ]; then |
| 245 | - build_cmd="bash build.sh --pkg --jit --soc=$a5_soc -j16 --cann_3rd_lib_path=$cann_3rd_lib_path" | 244 | + build_cmd="bash build.sh --pkg --jit --soc=$a5_soc -j16 --cann_3rd_lib_path=${ASCEND_3RD_LIB_PATH}" |
| 246 | run_build_command "$build_cmd" | 245 | run_build_command "$build_cmd" |
| 247 | execute_run_file "builtin" | 246 | execute_run_file "builtin" |
| 248 | fi | 247 | fi |
| @@ -0,0 +1,198 @@ | |||
| 1 | +#!/bin/bash | ||
| 2 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | +# ============================================================================ | ||
| 10 | + | ||
| 11 | +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" | ||
| 12 | +workspace_dir="$(dirname "$(dirname "$SCRIPT_DIR")")" | ||
| 13 | +export WORKSPACE=${workspace_dir} | ||
| 14 | +export ASCEND_3RD_LIB_PATH=${workspace_dir}/third_party | ||
| 15 | + | ||
| 16 | +get_pkg_name(){ | ||
| 17 | + compile_package_name=$(ls "${workspace_dir}/build_out/" | grep -E "*.run$" | head -n1) | ||
| 18 | + echo "${compile_package_name}" | ||
| 19 | + return 0 | ||
| 20 | +} | ||
| 21 | + | ||
| 22 | +set -e | ||
| 23 | +TARGET_URL="https://gitcode.com/cann/ops-nn.git" | ||
| 24 | +BASE_BRANCH_NAME=${1:-master} # 默认对比 master 分支 | ||
| 25 | +LOCAL_BRANCH=${2:-HEAD} # 默认对比当前本地分支 | ||
| 26 | + | ||
| 27 | +REMOTE_NAME="" | ||
| 28 | +for remote in $(git remote); do | ||
| 29 | + url=$(git remote get-url "$remote" 2>/dev/null) | ||
| 30 | + # 支持 https 和 git 协议，以及去掉末尾的 .git 进行模糊匹配 | ||
| 31 | + if [[ "$url" == *"$TARGET_URL"* ]] || [[ "$url" == *"${TARGET_URL%.git}"* ]]; then | ||
| 32 | + REMOTE_NAME="$remote" | ||
| 33 | + break | ||
| 34 | + fi | ||
| 35 | +done | ||
| 36 | + | ||
| 37 | +# 如果没找到匹配的 URL，默认使用 'origin' (兼容直接 clone 主仓的情况) | ||
| 38 | +if [ -z "$REMOTE_NAME" ]; then | ||
| 39 | + echo "Warning: Specific remote URL not found. Defaulting to 'origin'." | ||
| 40 | + if ! git remote | grep -q "^origin$"; then | ||
| 41 | + echo "Error: No 'origin' remote found either. Please check your remotes." | ||
| 42 | + exit 1 | ||
| 43 | + fi | ||
| 44 | + REMOTE_NAME="origin" | ||
| 45 | +fi | ||
| 46 | + | ||
| 47 | +echo "Detected Remote: $REMOTE_NAME (URL: $(git remote get-url $REMOTE_NAME))" | ||
| 48 | + | ||
| 49 | +echo "Fetching latest from $REMOTE_NAME..." | ||
| 50 | +git fetch "$REMOTE_NAME" --quiet --prune | ||
| 51 | + | ||
| 52 | +# 构建远程分支引用名 (例如: origin/master) | ||
| 53 | +REMOTE_REF="${REMOTE_NAME}/${BASE_BRANCH_NAME}" | ||
| 54 | + | ||
| 55 | +# 检查远程分支是否存在 | ||
| 56 | +if ! git rev-parse --verify "$REMOTE_REF" >/dev/null 2>&1; then | ||
| 57 | + echo "Error: Remote branch '$REMOTE_REF' does not exist." | ||
| 58 | + echo "Available branches from $REMOTE_NAME:" | ||
| 59 | + git branch -r --list "${REMOTE_NAME}/*" | ||
| 60 | + exit 1 | ||
| 61 | +fi | ||
| 62 | + | ||
| 63 | +MERGE_BASE_COMMIT=$(git merge-base "$REMOTE_REF" "$LOCAL_BRANCH") | ||
| 64 | + | ||
| 65 | +if [ -z "$MERGE_BASE_COMMIT" ]; then | ||
| 66 | + echo "Error: Could not find a common ancestor between $REMOTE_REF and $LOCAL_BRANCH." | ||
| 67 | + exit 1 | ||
| 68 | +fi | ||
| 69 | + | ||
| 70 | +TARGET_COMMIT=$(git rev-parse "$LOCAL_BRANCH") | ||
| 71 | + | ||
| 72 | +echo "Calculating changed files..." | ||
| 73 | +CHANGED_FILES=$(git diff --name-only "$MERGE_BASE_COMMIT" "$TARGET_COMMIT" | sort -u) | ||
| 74 | + | ||
| 75 | +if [ -z "$CHANGED_FILES" ]; then | ||
| 76 | + echo "[warning] The file for the change cannot be found. Please check whether the code has been committed." | ||
| 77 | + exit 0 | ||
| 78 | +fi | ||
| 79 | + | ||
| 80 | +# 输出结果 | ||
| 81 | +echo "$CHANGED_FILES" > pr_filelist.txt | ||
| 82 | + | ||
| 83 | +echo "Done! Changed files saved to pr_filelist.txt" | ||
| 84 | +echo "Total: $(wc -l < pr_filelist.txt) files" | ||
| 85 | +echo "" | ||
| 86 | +echo "Preview:" | ||
| 87 | +cat pr_filelist.txt | ||
| 88 | + | ||
| 89 | +export BASE_PATH=$(pwd) | ||
| 90 | +export BUILD_PATH="${BASE_PATH}/build" | ||
| 91 | +rm -rf $BUILD_PATH | ||
| 92 | +rm -rf $BASE_PATH/build_out | ||
| 93 | + | ||
| 94 | +THREAD_NUM=$(grep -c ^processor /proc/cpuinfo) | ||
| 95 | +echo "==============================pre build=========================================" | ||
| 96 | +rm -rf build && rm -rf build_out | ||
| 97 | +echo "exec cmd: [bash build.sh --pkg --ops=fatrelu_mul --vendor_name=fatrelu_mul -j${THREAD_NUM}]" | ||
| 98 | +bash build.sh --pkg --ops="fatrelu_mul" --vendor_name="fatrelu_mul" | ||
| 99 | +mkdir -p ${SCRIPT_DIR}/tmp && rm -rf ${script_dir}/tmp/* | ||
| 100 | +mv build_out/cann-ops-nn-fatrelu_mul*.run ${SCRIPT_DIR}/tmp/cann-ops-nn-fatrelu_mul_linux-aarch64.run | ||
| 101 | +rm -rf build && rm -rf build_out | ||
| 102 | + | ||
| 103 | +echo "==============================build jit=============================================" | ||
| 104 | +echo "exec cmd: [bash build.sh --pkg --jit -j${THREAD_NUM}]" | ||
| 105 | +bash build.sh --pkg --jit -j ${THREAD_NUM} | ||
| 106 | +rm -rf build && rm -rf build_out | ||
| 107 | + | ||
| 108 | +echo "==============================build single op====================================" | ||
| 109 | +SINGLE_FILE="single.tar.gz" | ||
| 110 | +need_check_example="false" | ||
| 111 | +rm -rf single/* | ||
| 112 | +echo "exec cmd: [bash scripts/ci/check_pkg.sh pr_filelist.txt]" | ||
| 113 | +bash scripts/ci/check_pkg.sh "pr_filelist.txt" | ||
| 114 | +if [[ -f "$SINGLE_FILE" && -s "$SINGLE_FILE" ]]; then | ||
| 115 | + need_check_example="true" | ||
| 116 | + rm single.tar.gz | ||
| 117 | +fi | ||
| 118 | +rm -rf build && rm -rf build_out | ||
| 119 | + | ||
| 120 | +echo "==============================compile ascend950===================================" | ||
| 121 | +# 获取机器架构 | ||
| 122 | +ARCH=$(uname -m) | ||
| 123 | +# 判断是 x86 还是 ARM | ||
| 124 | +if [[ "$ARCH" == "x86_64" ]]; then | ||
| 125 | + echo "exec cmd: [bash scripts/ci/compile_ascend950_pkg.sh pr_filelist.txt]" | ||
| 126 | + bash scripts/ci/compile_ascend950_pkg.sh | ||
| 127 | +elif [[ "$ARCH" == "aarch64" ]]; then | ||
| 128 | + echo "exec cmd: [bash scripts/ci/compile_ascend950_pkg.sh pr_filelist.txt -force_jit]" | ||
| 129 | + bash scripts/ci/compile_ascend950_pkg.sh "-force_jit" | ||
| 130 | +fi | ||
| 131 | +check_res=$? | ||
| 132 | +if [[ $check_res -ne 0 ]]; then | ||
| 133 | + echo "[ERROR] compile ascend950 failed" | ||
| 134 | + exit $check_res | ||
| 135 | +fi | ||
| 136 | +rm -rf build && rm -rf build_out | ||
| 137 | + | ||
| 138 | +echo "==============================build utest start======================================" | ||
| 139 | +echo "--------------------------build ophost ut start-----------------------------------" | ||
| 140 | +echo "exec cmd: [bash build.sh -u --ophost -f pr_filelist.txt -j${THREAD_NUM}]" | ||
| 141 | +bash build.sh -u --ophost -f "pr_filelist.txt" -j ${THREAD_NUM} | ||
| 142 | +echo "--------------------------build opapi ut start------------------------------------" | ||
| 143 | +echo "exec cmd: [bash build.sh -u --opapi -f pr_filelist.txt -j${THREAD_NUM}]" | ||
| 144 | +bash build.sh -u --opapi -f "pr_filelist.txt" -j ${THREAD_NUM} | ||
| 145 | +if [ "$BASE_BRANCH_NAME" = "master" ]; then | ||
| 146 | + echo "--------------------------build opgraph ut start-----------------------------------" | ||
| 147 | + echo "exec cmd: [bash build.sh -u --opgraph -f pr_filelist.txt -j${THREAD_NUM}]" | ||
| 148 | + bash build.sh -u --opgraph -f "pr_filelist.txt" -j ${THREAD_NUM} | ||
| 149 | + echo "--------------------------build opkernel ut start-----------------------------------" | ||
| 150 | + echo "exec cmd: [bash scripts/ci/check_kernel_ut.sh pr_filelist.txt --no_cov]" | ||
| 151 | + bash scripts/ci/check_kernel_ut.sh pr_filelist.txt --no_cov | tee output.txt | ||
| 152 | + if grep -q "error happened" output.txt; then | ||
| 153 | + echo "[ERROR] Error happened in output check log" | ||
| 154 | + exit 1 | ||
| 155 | + fi | ||
| 156 | +fi | ||
| 157 | +rm -rf build && rm -rf build_out | ||
| 158 | +echo "==============================build utest end====================================" | ||
| 159 | + | ||
| 160 | +if ! asys info -r=status | grep "Ascend 910B"; then | ||
| 161 | + echo "[Warning] The current platform does not support smoke tests, and the process ends" | ||
| 162 | + exit 0 | ||
| 163 | +fi | ||
| 164 | + | ||
| 165 | +rm -f run_test.log | ||
| 166 | +if [[ ${need_check_example} == "true" ]]; then | ||
| 167 | + echo "==============================build A2 smoke===================================" | ||
| 168 | + # 执行受影响的算子 | ||
| 169 | + echo "exec cmd: [bash scripts/ci/check_example.sh pr_filelist.txt]" | ||
| 170 | + bash scripts/ci/check_example.sh pr_filelist.txt 2>&1 | tee -a ./run_test.log | ||
| 171 | + # 单独执行nn仓的fatrelu_mul | ||
| 172 | + chmod +x ${SCRIPT_DIR}/tmp/* && ${SCRIPT_DIR}/tmp/*.run 2>&1 | tee -a ./run_test.log | ||
| 173 | + echo "exec cmd: [bash build.sh --run_example fatrelu_mul eager cust --vendor_name=fatrelu_mul]" | ||
| 174 | + bash build.sh --run_example fatrelu_mul eager cust --vendor_name=fatrelu_mul 2>&1 | tee -a ./run_test.log | ||
| 175 | + if grep -w -e "FAIL" -e "errors" -e "fail" -e "failed" -e "error" -e "ERROR:" -e "Error" -e "error:" "./run_test.log"; then | ||
| 176 | + echo "[error] run test case failed" | ||
| 177 | + exit 1 | ||
| 178 | + fi | ||
| 179 | +fi | ||
| 180 | + | ||
| 181 | +echo "==============================check experimental====================================" | ||
| 182 | +rm -rf build && rm -rf build_out | ||
| 183 | +echo "exec cmd: [bash scripts/ci/check_experimental_pkg.sh pr_filelist.txt]" | ||
| 184 | +bash scripts/ci/check_experimental_pkg.sh "pr_filelist.txt" | ||
| 185 | +if [ -f "${workspace_dir}/build_out/"*.run ]; then | ||
| 186 | + compile_package_name=$(get_pkg_name) | ||
| 187 | + if [[ -z "${compile_package_name}" ]]; then | ||
| 188 | + echo "[ERROR] Not find *.run in ${workspace_dir}/build_out !" | ||
| 189 | + exit 1 | ||
| 190 | + fi | ||
| 191 | + chmod +x ./build_out/${compile_package_name} | ||
| 192 | + echo "exec cmd: [bash scripts/ci/check_experimental_example.sh pr_filelist.txt]" | ||
| 193 | + echo 'y' | bash ${compile_package_name} --quiet && bash scripts/ci/check_experimental_example.sh pr_filelist.txt 2>&1 | tee -a ./run_test.log | ||
| 194 | + if grep -w -e "FAIL" -e "errors" -e "fail" -e "failed" -e "error" -e "ERROR:" -e "Error" -e "error:" "./run_test.log"; then | ||
| 195 | + echo "[ERROR] run test case failed" | ||
| 196 | + exit 1 | ||
| 197 | + fi | ||
| 198 | +fi | ||