enhancement(ci): 自动检测变更 patch 并优化 NPU 测试流水线-pytorch-AtomGit

M.github/scripts/collect_all_cases.py+1-1

	@@ -505,7 +505,7 @@ def parse_args():
505	parser = argparse.ArgumentParser(description="Collect and shard test cases")	505	parser = argparse.ArgumentParser(description="Collect and shard test cases")
506	parser.add_argument("--test-dir", required=True, help="PyTorch test directory")	506	parser.add_argument("--test-dir", required=True, help="PyTorch test directory")
507	parser.add_argument("--case-paths-config", help="case_paths_ci.yml path")	507	parser.add_argument("--case-paths-config", help="case_paths_ci.yml path")
508	- parser.add_argument("--distributed-shards", type=int, default=2, help="Distributed test shards")	508	+ parser.add_argument("--distributed-shards", type=int, default=5, help="Distributed test shards")
509	parser.add_argument("--regular-shards", type=int, default=5, help="Regular test shards")	509	parser.add_argument("--regular-shards", type=int, default=5, help="Regular test shards")
510	parser.add_argument("--output-dir", required=True, help="Output directory for shard JSONs")	510	parser.add_argument("--output-dir", required=True, help="Output directory for shard JSONs")
511	parser.add_argument("--error-log-dir", help="Output directory for collection error logs (default: output-dir/collection_errors)")	511	parser.add_argument("--error-log-dir", help="Output directory for collection error logs (default: output-dir/collection_errors)")

A.github/scripts/detect_changed_patches.sh+140-0

@@ -0,0 +1,140 @@
	1	+#!/bin/bash
	2	+# ==============================================================================
	3	+# detect_changed_patches.sh
	4	+#
	5	+# Detect changed patch files in test_upstream/ and derive corresponding test files.
	6	+#
	7	+# Environment inputs (set by GitHub Actions workflow):
	8	+# EVENT_NAME - "pull_request" or "workflow_dispatch"
	9	+# BASE_SHA - PR base commit SHA (pull_request only)
	10	+# HEAD_SHA - PR head commit SHA (pull_request only)
	11	+# BASE_REF - PR target branch ref (pull_request only)
	12	+# INPUT_PATCH_FILES - comma-separated patch paths (workflow_dispatch only)
	13	+#
	14	+# Outputs (written to $GITHUB_OUTPUT):
	15	+# test_patches - comma-separated test_upstream/test/ patch paths
	16	+# torch_patches - comma-separated test_upstream/torch/ patch paths
	17	+# test_files - comma-separated derived test file names
	18	+# has_test_changes - "true" or "false"
	19	+# has_torch_changes - "true" or "false"
	20	+# changed_summary - one of: test+torch, test-only, torch-only, none
	21	+# ==============================================================================
	22	+set -euo pipefail
	23	+
	24	+# ------------------------------------------------------------------
	25	+# Step 1: Collect changed files from the trigger source
	26	+# ------------------------------------------------------------------
	27	+if [ "${EVENT_NAME}" = "pull_request" ]; then
	28	+ echo "=== PR Event: detecting changes ==="
	29	+ echo "Base SHA: ${BASE_SHA:-unknown}"
	30	+ echo "Head SHA: ${HEAD_SHA:-unknown}"
	31	+
	32	+ # HEAD is the PR merge commit (checked out by actions/checkout).
	33	+ # HEAD^1 = base branch, HEAD^2 = PR head branch.
	34	+ # Use three-dot (...) to show only PR-side changes relative to merge-base,
	35	+ # excluding upstream changes that happened after the fork point.
	36	+ if git cat-file -e HEAD^2 2>/dev/null; then
	37	+ echo "Using merge commit parents: HEAD^1...HEAD^2 (PR-side changes only)"
	38	+ CHANGED_FILES=$(git diff --name-only HEAD^1...HEAD^2 -- 'test_upstream/' 2>/dev/null \|\| true)
	39	+ else
	40	+ echo "Merge parents not available, falling back to base/head diff"
	41	+ git fetch --no-tags origin "${BASE_REF}" 2>/dev/null \|\| true
	42	+ CHANGED_FILES=$(git diff --name-only \
	43	+ "${BASE_SHA}" "${HEAD_SHA}" \
	44	+ -- 'test_upstream/' 2>/dev/null \|\| true)
	45	+ fi
	46	+else
	47	+ echo "=== Manual Dispatch: using input ==="
	48	+ CHANGED_FILES="${INPUT_PATCH_FILES:-}"
	49	+fi
	50	+
	51	+echo ""
	52	+echo "Raw changed files:"
	53	+echo "${CHANGED_FILES}" \| sed 's/^/ /'
	54	+
	55	+# ------------------------------------------------------------------
	56	+# Step 2: Normalize (handle comma-separated input from dispatch)
	57	+# ------------------------------------------------------------------
	58	+CHANGED_FILES=$(echo "${CHANGED_FILES}" \| tr ',' '\n' \| sed 's/^[[:space:]]//; s/[[:space:]]$//')
	59	+
	60	+# ------------------------------------------------------------------
	61	+# Step 3: Classify patches and derive test files
	62	+# ------------------------------------------------------------------
	63	+TEST_PATCHES=""
	64	+TORCH_PATCHES=""
	65	+TEST_FILES=""
	66	+
	67	+while IFS= read -r f; do
	68	+ [ -z "$f" ] && continue
	69	+
	70	+ case "$f" in
	71	+ test_upstream/test/.patch\|test_upstream/test/.diff)
	72	+ # Derive test file by stripping prefix + suffix:
	73	+ # test_upstream/test/test_autograd.py.patch → test_autograd.py
	74	+ # test_upstream/test/ao/test_foo.py.patch → ao/test_foo.py
	75	+ # test_upstream/test/inductor/test_minifer.diff → inductor/test_minifer.py
	76	+ TEST_FILE=$(echo "$f" \| sed 's\|^test_upstream/test/\|\|; s\|\.patch$\|\|; s\|\.diff$\|\|')
	77	+ # Ensure .py extension for cases where patch suffix was on bare name (e.g. test_foo.diff)
	78	+ [[ "$TEST_FILE" != *.py ]] && TEST_FILE="${TEST_FILE}.py"
	79	+ TEST_PATCHES="${TEST_PATCHES}${f},"
	80	+ TEST_FILES="${TEST_FILES}${TEST_FILE},"
	81	+ echo " → test patch: $f → test file: ${TEST_FILE}"
	82	+ ;;
	83	+ test_upstream/torch/.patch\|test_upstream/torch/.diff)
	84	+ TORCH_PATCHES="${TORCH_PATCHES}${f},"
	85	+ echo " → torch patch: $f (no direct test mapping)"
	86	+ ;;
	87	+ *)
	88	+ echo " → skipped: $f (not a patch file)"
	89	+ ;;
	90	+ esac
	91	+done <<< "${CHANGED_FILES}"
	92	+
	93	+# Remove trailing commas
	94	+TEST_PATCHES="${TEST_PATCHES%,}"
	95	+TORCH_PATCHES="${TORCH_PATCHES%,}"
	96	+TEST_FILES="${TEST_FILES%,}"
	97	+
	98	+# Determine change type flags
	99	+HAS_TEST="false"
	100	+HAS_TORCH="false"
	101	+[ -n "${TEST_PATCHES}" ] && HAS_TEST="true"
	102	+[ -n "${TORCH_PATCHES}" ] && HAS_TORCH="true"
	103	+
	104	+# Determine summary string
	105	+if [ "${HAS_TEST}" = "true" ] && [ "${HAS_TORCH}" = "true" ]; then
	106	+ CHANGED_SUMMARY="test+torch"
	107	+elif [ "${HAS_TEST}" = "true" ]; then
	108	+ CHANGED_SUMMARY="test-only"
	109	+elif [ "${HAS_TORCH}" = "true" ]; then
	110	+ CHANGED_SUMMARY="torch-only"
	111	+else
	112	+ CHANGED_SUMMARY="none"
	113	+fi
	114	+
	115	+# ------------------------------------------------------------------
	116	+# Step 4: Report and write outputs
	117	+# ------------------------------------------------------------------
	118	+echo ""
	119	+echo "=== Detection Result ==="
	120	+echo "test_patches=${TEST_PATCHES}"
	121	+echo "torch_patches=${TORCH_PATCHES}"
	122	+echo "test_files=${TEST_FILES}"
	123	+echo "has_test_changes=${HAS_TEST}"
	124	+echo "has_torch_changes=${HAS_TORCH}"
	125	+echo "changed_summary=${CHANGED_SUMMARY}"
	126	+
	127	+{
	128	+ echo "test_patches=${TEST_PATCHES}"
	129	+ echo "torch_patches=${TORCH_PATCHES}"
	130	+ echo "test_files=${TEST_FILES}"
	131	+ echo "has_test_changes=${HAS_TEST}"
	132	+ echo "has_torch_changes=${HAS_TORCH}"
	133	+ echo "changed_summary=${CHANGED_SUMMARY}"
	134	+} >> "${GITHUB_OUTPUT}"
	135	+
	136	+if [ "${HAS_TEST}" = "false" ] && [ "${HAS_TORCH}" = "false" ]; then
	137	+ echo ""
	138	+ echo "WARNING: No patch files detected in changed files."
	139	+ echo "If this is a PR, ensure it modifies .patch or .diff files under test_upstream/."
	140	+fi

M.github/scripts/run_npu_test_shard.py+15-4

	@@ -347,22 +347,33 @@ def parse_junit_xml_status(xml_file: Path) -> Dict:
347	# Check <skipped>	347	# Check <skipped>
348	skipped_elem = testcase.find("skipped")	348	skipped_elem = testcase.find("skipped")
349	if skipped_elem is not None:	349	if skipped_elem is not None:
		350	+ skip_type = skipped_elem.get("type", "")
		351	+ if skip_type == "pytest.xfail":
		352	+ result["status"] = "passed"
		353	+ result["message"] = "xfailed: expected failure"
		354	+ return result
350	result["status"] = "skipped"	355	result["status"] = "skipped"
351	- result["message"] = skipped_elem.get("message", "")	356	+ attr_msg = skipped_elem.get("message", "")
		357	+ text_msg = (skipped_elem.text or "").strip()
		358	+ result["message"] = attr_msg + ("\n" + text_msg if text_msg else "")
352	return result	359	return result
353		360
354	# Check <failure>	361	# Check <failure>
355	failure_elem = testcase.find("failure")	362	failure_elem = testcase.find("failure")
356	if failure_elem is not None:	363	if failure_elem is not None:
357	result["status"] = "failed"	364	result["status"] = "failed"
358	- result["message"] = failure_elem.get("message", "")	365	+ attr_msg = failure_elem.get("message", "")
		366	+ text_msg = (failure_elem.text or "").strip()
		367	+ result["message"] = attr_msg + ("\n" + text_msg if text_msg else "")
359	return result	368	return result
360		369
361	# Check <error>	370	# Check <error>
362	error_elem = testcase.find("error")	371	error_elem = testcase.find("error")
363	if error_elem is not None:	372	if error_elem is not None:
364	result["status"] = "error"	373	result["status"] = "error"
365	- result["message"] = error_elem.get("message", "")	374	+ attr_msg = error_elem.get("message", "")
		375	+ text_msg = (error_elem.text or "").strip()
		376	+ result["message"] = attr_msg + ("\n" + text_msg if text_msg else "")
366	return result	377	return result
367		378
368	# No failure/error/skipped = passed	379	# No failure/error/skipped = passed
	@@ -1700,4 +1711,4 @@ def main():
1700		1711
1701		1712
1702	if __name__ == "__main__":	1713	if __name__ == "__main__":
1703	- main()	1714	+ main()

M.github/workflows/_torch-npu-upstream-build.yml+1-2

	@@ -46,8 +46,7 @@ jobs:
46	- name: Checkout repository	46	- name: Checkout repository
47	uses: actions/checkout@v4	47	uses: actions/checkout@v4
48	with:	48	with:
49	- repository: Ascend/pytorch	49	+ ref: ${{ github.sha }}
50	- ref: v2.7.1
51	fetch-depth: 1	50	fetch-depth: 1
52	submodules: recursive	51	submodules: recursive
53		52

M.github/workflows/_torch-npu-upstream-collect.yml+2-1

	@@ -22,7 +22,7 @@ on:
22	distributed_shards:	22	distributed_shards:
23	required: false	23	required: false
24	type: string	24	type: string
25	- default: '2'	25	+ default: '5'
26	description: Number of shards for distributed tests	26	description: Number of shards for distributed tests
27	regular_shards:	27	regular_shards:
28	required: false	28	required: false
	@@ -65,6 +65,7 @@ jobs:
65	total_cases: ${{ steps.collect_and_shard.outputs.total_cases }}	65	total_cases: ${{ steps.collect_and_shard.outputs.total_cases }}
66		66
67	steps:	67	steps:
		68	+
68	- name: Setup NPU test environment	69	- name: Setup NPU test environment
69	uses: Ascend/pytorch/.github/actions/setup-npu-test-env@v2.7.1	70	uses: Ascend/pytorch/.github/actions/setup-npu-test-env@v2.7.1
70	with:	71	with:

M.github/workflows/_torch-npu-upstream-prepare.yml+1-2

	@@ -21,8 +21,7 @@ jobs:
21	- name: Checkout repository	21	- name: Checkout repository
22	uses: actions/checkout@v4	22	uses: actions/checkout@v4
23	with:	23	with:
24	- repository: Ascend/pytorch	24	+ ref: ${{ github.sha }}
25	- ref: v2.7.1
26	fetch-depth: 1	25	fetch-depth: 1
27		26
28	- name: Clone PyTorch v2.7.1 (for test source)	27	- name: Clone PyTorch v2.7.1 (for test source)

M.github/workflows/_torch-npu-upstream-test-custom.yml+3-1

	@@ -31,13 +31,14 @@ defaults:
31	jobs:	31	jobs:
32	run_tests:	32	run_tests:
33	name: test_custom	33	name: test_custom
34	- runs-on: linux-aarch64-a3-8	34	+ runs-on: linux-aarch64-a3-16
35	timeout-minutes: 1800	35	timeout-minutes: 1800
36	container:	36	container:
37	image: ${{ inputs.docker_image }}	37	image: ${{ inputs.docker_image }}
38	options: --user root	38	options: --user root
39		39
40	steps:	40	steps:
		41	+
41	- name: Setup NPU test environment	42	- name: Setup NPU test environment
42	uses: Ascend/pytorch/.github/actions/setup-npu-test-env@v2.7.1	43	uses: Ascend/pytorch/.github/actions/setup-npu-test-env@v2.7.1
43	with:	44	with:
	@@ -62,6 +63,7 @@ jobs:
62	--disabled-testcases pytorch-test-src/test_upstream/disabled_testcases.json \	63	--disabled-testcases pytorch-test-src/test_upstream/disabled_testcases.json \
63	--report-dir ${REPORT_DIR} \	64	--report-dir ${REPORT_DIR} \
64	--timeout 1200 \	65	--timeout 1200 \
		66	+ --max-workers 16 \
65	--verbose \	67	--verbose \
66	2>&1 \| tee /tmp/test_custom.log	68	2>&1 \| tee /tmp/test_custom.log
67		69

M.github/workflows/_torch-npu-upstream-test-dist.yml+1-0

	@@ -46,6 +46,7 @@ jobs:
46	fail-fast: false	46	fail-fast: false
47		47
48	steps:	48	steps:
		49	+
49	- name: Setup NPU test environment	50	- name: Setup NPU test environment
50	uses: Ascend/pytorch/.github/actions/setup-npu-test-env@v2.7.1	51	uses: Ascend/pytorch/.github/actions/setup-npu-test-env@v2.7.1
51	with:	52	with:

M.github/workflows/_torch-npu-upstream-test-regular.yml+2-1

	@@ -46,6 +46,7 @@ jobs:
46	fail-fast: false	46	fail-fast: false
47		47
48	steps:	48	steps:
		49	+
49	- name: Setup NPU test environment	50	- name: Setup NPU test environment
50	uses: Ascend/pytorch/.github/actions/setup-npu-test-env@v2.7.1	51	uses: Ascend/pytorch/.github/actions/setup-npu-test-env@v2.7.1
51	with:	52	with:
	@@ -80,7 +81,7 @@ jobs:
80	echo "Runner: linux-aarch64-a3-16 (16-card NPU)"	81	echo "Runner: linux-aarch64-a3-16 (16-card NPU)"
81	echo "Execution mode: CONCURRENT (16 workers)"	82	echo "Execution mode: CONCURRENT (16 workers)"
82		83
83	- # Regular tests: pre-collected cases, 16 concurrent workers	84	+ # Regular tests: pre-collected cases, 16 concurrent workers (one per NPU device)
84	set +e	85	set +e
85	$PYTHON ascend_pytorch/.github/scripts/run_npu_test_shard.py \	86	$PYTHON ascend_pytorch/.github/scripts/run_npu_test_shard.py \
86	--cases-json "${CASES_JSON}" \	87	--cases-json "${CASES_JSON}" \

M.github/workflows/_torch-npu-upstream-test.yml+1-1

	@@ -26,7 +26,7 @@ on:
26	distributed_shards:	26	distributed_shards:
27	required: false	27	required: false
28	type: string	28	type: string
29	- default: '2'	29	+ default: '5'
30	description: Number of shards for distributed tests	30	description: Number of shards for distributed tests
31	regular_shards:	31	regular_shards:
32	required: false	32	required: false

M.github/workflows/torch-npu-upstream-test-trigger.yml+37-20

	@@ -4,28 +4,45 @@ on:
4	pull_request:	4	pull_request:
5	paths:	5	paths:
6	- '.github/**'	6	- '.github/**'
7	- workflow_dispatch:	7	+ - 'test_upstream/**'
8	- inputs:
9	- distributed_shards:
10	- description: 'Number of shards for distributed tests (default 2)'
11	- required: false
12	- default: '2'
13	- type: string
14	- regular_shards:
15	- description: 'Number of shards for regular tests (default 5)'
16	- required: false
17	- default: '5'
18	- type: string
19	- test_files:
20	- description: 'Test files to run directly (comma-separated, e.g., "test_meta.py,test_nn.py"). Skip shard assignment if set.'
21	- required: false
22	- default: ''
23	- type: string
24		8
25	jobs:	9	jobs:
		10	+ # ============================================================================
		11	+ # 1. Detect Changed Patches
		12	+ # ============================================================================
		13	+ detect:
		14	+ name: Detect changed patches
		15	+ runs-on: ubuntu-latest
		16	+ outputs:
		17	+ test_files: ${{ steps.detect.outputs.test_files }}
		18	+ has_test_changes: ${{ steps.detect.outputs.has_test_changes }}
		19	+ has_torch_changes: ${{ steps.detect.outputs.has_torch_changes }}
		20	+ steps:
		21	+ - name: Checkout repository
		22	+ uses: actions/checkout@v4
		23	+ with:
		24	+ fetch-depth: 0
		25	+
		26	+ - name: Detect changed patch files
		27	+ id: detect
		28	+ env:
		29	+ EVENT_NAME: ${{ github.event_name }}
		30	+ BASE_SHA: ${{ github.event.pull_request.base.sha }}
		31	+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}
		32	+ BASE_REF: ${{ github.base_ref }}
		33	+ INPUT_PATCH_FILES: ''
		34	+ run: \|
		35	+ chmod +x .github/scripts/detect_changed_patches.sh
		36	+ .github/scripts/detect_changed_patches.sh
		37	+
		38	+ # ============================================================================
		39	+ # 2. Trigger Tests
		40	+ # ============================================================================
26	trigger_test:	41	trigger_test:
		42	+ needs: detect
		43	+ if: always()
27	uses: ./.github/workflows/_torch-npu-upstream-test.yml	44	uses: ./.github/workflows/_torch-npu-upstream-test.yml
28	with:	45	with:
29	- distributed_shards: ${{ github.event.inputs.distributed_shards \|\| '2' }}	46	+ distributed_shards: '5'
30	- regular_shards: ${{ github.event.inputs.regular_shards \|\| '5' }}	47	+ regular_shards: '5'
31	- test_files: ${{ github.event.inputs.test_files \|\| '' }}	48	+ test_files: ${{ needs.detect.outputs.test_files \|\| '' }}