已关闭
[Requirement|需求建议]: 本地可以执行CI流程的命令 #1261
丛吉钰创建于  3月2日关闭于  3月4日
丛吉钰
丛吉钰成员
3月2日 创建

Thanks for sending an requirement! Please fill in the following template to help quickly solve your problem.

Backgroud(背景信息)

当开发者代码开发完毕后,应在本地先执行构建流程,提前发现代码问题,提升整仓构建质量。

Origin(信息来源)

开发者

Benefit / Necessity (价值/作用)

减少失败构建竞争的服务器资源。提升整体构建上库效率。

Design(设计方案)

#!/bin/bash
export WORKSPACE=(pwd)exportASCEND3RDLIBPATH=(pwd) export ASCEND_3RD_LIB_PATH=(pwd)/third_party

get_pkg_name(){
compile_package_name=(ls"(ls "{WORKSPACE}/build_out/" | grep -E "*.run"headn1)echo"" | head -n1) echo "{compile_package_name}"
return 0
}

set -e
TARGET_URL="https://gitcode.com/cann/ops-nn.git"
BASE_BRANCH_NAME={1:-master} # 默认对比 master 分支 LOCAL_BRANCH={2:-HEAD} # 默认对比当前本地分支

REMOTE_NAME=""
for remote in (gitremote);dourl=(git remote); do url=(git remote get-url "remote" 2>/dev/null) # 支持 https 和 git 协议,以及去掉末尾的 .git 进行模糊匹配 if [[ "url" == *"TARGETURL"]][["TARGET_URL"* ]] || [[ "url" == *"{TARGET_URL%.git}"* ]]; then REMOTE_NAME="remote"
break
fi
done

如果没找到匹配的 URL,默认使用 'origin' (兼容直接 clone 主仓的情况)

if [ -z "REMOTENAME"];thenecho"Warning:SpecificremoteURLnotfound.Defaultingtoorigin."if!gitremotegrepq"originREMOTE_NAME" ]; then echo "Warning: Specific remote URL not found. Defaulting to 'origin'." if ! git remote | grep -q "^origin"; then
echo "Error: No 'origin' remote found either. Please check your remotes."
exit 1
fi
REMOTE_NAME="origin"
fi

echo "Detected Remote: $REMOTE_NAME (URL: $(git remote get-url $REMOTE_NAME))"

echo "Fetching latest from REMOTENAME..."gitfetch"REMOTE_NAME..." git fetch "REMOTE_NAME" --quiet --prune

构建远程分支引用名 (例如: origin/master)

REMOTE_REF="${REMOTE_NAME}/${BASE_BRANCH_NAME}"

检查远程分支是否存在

if ! git rev-parse --verify "REMOTE_REF" >/dev/null 2>&1; then echo "Error: Remote branch 'REMOTE_REF' does not exist."
echo "Available branches from REMOTENAME:"gitbranchrlist"REMOTE_NAME:" git branch -r --list "{REMOTE_NAME}/*"
exit 1
fi

MERGE_BASE_COMMIT=(gitmergebase"(git merge-base "REMOTE_REF" "$LOCAL_BRANCH")

if [ -z "$MERGE_BASE_COMMIT" ]; then
echo "Error: Could not find a common ancestor between $REMOTE_REF and $LOCAL_BRANCH."
echo "Are these branches completely unrelated?"
exit 1
fi

TARGET_COMMIT=(gitrevparse"(git rev-parse "LOCAL_BRANCH")

echo "---------------------------------------------------------"
echo "Base Branch Ref : $REMOTE_REF -> $(git rev-parse $REMOTE_REF)"
echo "Local Branch : $LOCAL_BRANCH -> $TARGET_COMMIT"
echo "Merge Base (Start Point of PR): $MERGE_BASE_COMMIT"
echo "---------------------------------------------------------"

echo "Calculating changed files..."
CHANGED_FILES=(gitdiffnameonly"(git diff --name-only "MERGE_BASE_COMMIT" "$TARGET_COMMIT" | sort -u)

if [ -z "$CHANGED_FILES" ]; then
echo "No local commits found relative to the merge base. No changes to report."
exit 0
fi

输出结果

echo "$CHANGED_FILES" > pr_filelist.txt

echo "Done! Changed files saved to pr_filelist.txt"
echo "Total: $(wc -l < pr_filelist.txt) files"
echo ""
echo "Preview:"
cat pr_filelist.txt

export BASE_PATH=(pwd)exportBUILDPATH="(pwd) export BUILD_PATH="{BASE_PATH}/build"
rm -rf $BUILD_PATH
rm -rf $BASE_PATH/build_out

THREAD_NUM=$(grep -c ^processor /proc/cpuinfo)
echo "==============================pre build========================================="
rm -rf build && rm -rf build_out
bash build.sh --pkg --ops="fatrelu_mul" --vendor_name="fatrelu_mul"
mkdir -p nn_a900
mv build_out/cann-ops-nn-fatrelu_mul*.run nn_a900/cann-ops-nn-fatrelu_mul_linux-aarch64.run

echo "==============================build single op===================================="
SINGLE_FILE="single.tar.gz"
need_check_example="false"
rm -rf single
echo "exec cmd: [bash scripts/ci/check_pkg.sh pr_filelist.txt]"
bash scripts/ci/check_pkg.sh "pr_filelist.txt" "-j 64"
if [[ -f "SINGLE_FILE" && -s "SINGLE_FILE" ]]; then
need_check_example="true"
rm single.tar.gz
fi

echo "==============================compile ascend950==================================="

获取机器架构

ARCH=$(uname -m)

判断是 x86 还是 ARM

if [[ "ARCH"=="x8664"]];thenecho"execcmd:[bashscripts/ci/compileascend950pkg.shprfilelist.txt]"bashscripts/ci/compileascend950pkg.shelif[["ARCH" == "x86_64" ]]; then echo "exec cmd: [bash scripts/ci/compile_ascend950_pkg.sh pr_filelist.txt]" bash scripts/ci/compile_ascend950_pkg.sh elif [[ "ARCH" == "aarch64" ]]; then
echo "exec cmd: [bash scripts/ci/compile_ascend950_pkg.sh pr_filelist.txt -force_jit]"
bash scripts/ci/compile_ascend950_pkg.sh "-force_jit"
fi

echo "==============================build jit============================================="
echo "exec cmd: [bash build.sh --pkg --jit -j ${THREAD_NUM}]"
bash build.sh --pkg --jit -j ${THREAD_NUM}

echo "==============================exec utest start======================================"
sh build.sh -u --ophost -f "pr_filelist.txt" -j ${THREAD_NUM}
echo "--------------------------build ophost ut end-----------------------------------"
sh build.sh -u --opapi -f "pr_filelist.txt" -j ${THREAD_NUM}
echo "--------------------------build opapi ut end------------------------------------"
if [ "$BASE_BRANCH" = "master" ]; then
sh build.sh -u --opgraph -f "pr_filelist.txt" -j ${THREAD_NUM}
echo "--------------------------build opgraph ut end-----------------------------------"
sh scripts/ci/check_kernel_ut.sh "pr_filelist.txt" "ops-nn" "-j ${THREAD_NUM}" | tee output.txt
echo "--------------------------build opkernel ut end-----------------------------------"
if grep -q "error happened" output.txt; then
echo "Error happened in output check log"
exit 1
fi
fi
echo "==============================exec utest end===================================="
rm -f run_test.log
if [[ {need_check_example} ]]; then echo "==============================exec A900 smoke===================================" # 执行受影响的算子 bash scripts/ci/check_example.sh pr_filelist.txt 2>&1 | tee -a ./run_test.log # 单独执行nn仓的fatrelu_mul chmod +x ./nn_a900/* && ./nn_a900/*.run 2>&1 | tee -a ./run_test.log bash build.sh --run_example "fatrelu_mul" "eager" cust --vendor_name="fatrelu_mul" 2>&1 | tee -a ./run_test.log if grep -w -e "FAIL" -e "errors" -e "fail" -e "failed" -e "error" -e "ERROR:" -e "Error" -e "error:" "./run_test.log"; then echo "date_time : run test case failed"
exit 1
fi
fi
echo "==============================check experimental===================================="
echo "exec cmd: [sh scripts/ci/check_experimental_pkg.sh pr_filelist.txt]"
rm -rf build && rm -rf build_out
bash scripts/ci/check_experimental_pkg.sh "pr_filelist.txt"
if [ -f "WORKSPACE/buildout/".run];thencompilepackagename={WORKSPACE}/build_out/"*.run ]; then compile_package_name=(get_pkg_name)
if [[ -z "${compile_package_name}" ]]; then
echo "ERROR: Not find *.run in ${WORKSPACE}/build_out !"
exit 1
fi
chmod +x ./build_out/${compile_package_name}
echo 'y' | bash ${compile_package_name} --quiet && bash scripts/ci/check_experimental_example.sh ${WORKSPACE}/pr_filelist.txt 2>&1 | tee -a ./run_test.log
fi

likedislike
丛吉钰丛吉钰成员
3月2日 修改了issue 的描述
丛吉钰丛吉钰成员
3月2日 修改了issue 的描述
丛吉钰丛吉钰成员
3月2日 将 cong-jiyu 设为负责人
CANN-robotCANN-robot成员
3月4日 关闭了 issue
CANN-robotCANN-robot成员
3月20日 添加了label:resolved