Star844
1306
代码
代码
Issues1403
Pull Requests2252
流水线
项目讨论
Wiki
分析
Star844
1306
已合并
workflow触发上游pytorch全量用例测试 #35958
kerer-sk创建于 5月18日
workflow触发上游pytorch全量用例测试 #35958
已合并
kerer-sk创建于 5月18日
从kerer-sk/pytorchv2.7.1合入到Ascend/pytorchv2.7.1
共 18 个文件变更+5313-1
A.github/actions/setup-npu-test-env/action.yml+195-0
@@ -0,0 +1,195 @@
1+name: 'Setup NPU Test Environment'
2+description: 'Common environment setup for NPU upstream tests - checkout, cache, install PyTorch/torch_npu/triton-ascend, test dependencies'
3+ 
4+inputs:
5+ python_version:
6+ required: true
7+ type: string
8+ description: Python version to use
9+ pytorch_version:
10+ required: true
11+ type: string
12+ description: PyTorch version to install
13+ torch_npu_wheel_artifact:
14+ required: true
15+ type: string
16+ description: Name of the torch_npu wheel artifact
17+ prepared_test_src_artifact:
18+ required: true
19+ type: string
20+ description: Name of the prepared test source artifact
21+ cache_key_prefix:
22+ required: false
23+ type: string
24+ default: 'pip-py'
25+ description: Prefix for cache key (allows different cache strategies)
26+ patch_log_suffix:
27+ required: false
28+ type: string
29+ default: 'setup'
30+ description: Suffix for torch_env_patch log filename
31+ 
32+runs:
33+ using: 'composite'
34+ steps:
35+ - name: Checkout repository
36+ uses: actions/checkout@v4
37+ with:
38+ repository: Ascend/pytorch
39+ ref: v2.7.1
40+ fetch-depth: 1
41+ path: ascend_pytorch
42+ 
43+ - name: Setup cache directories
44+ shell: bash
45+ run: |
46+ mkdir -p /github/home/.cache/pip
47+ chmod -R 777 /github/home/.cache
48+ 
49+ - name: Cache pip (shared with build)
50+ uses: actions/cache@v4
51+ with:
52+ path: /github/home/.cache/pip
53+ key: ${{ inputs.cache_key_prefix }}${{ inputs.python_version }}-torch${{ inputs.pytorch_version }}-shared
54+ restore-keys: |
55+ ${{ inputs.cache_key_prefix }}${{ inputs.python_version }}-torch${{ inputs.pytorch_version }}-
56+ ${{ inputs.cache_key_prefix }}${{ inputs.python_version }}-
57+ 
58+ - name: Download built torch_npu wheel
59+ uses: actions/download-artifact@v4
60+ with:
61+ name: ${{ inputs.torch_npu_wheel_artifact }}
62+ path: torch-npu-wheel-artifact
63+ 
64+ - name: Uninstall pre-installed torch/torchvision
65+ shell: bash
66+ run: |
67+ pip${{ inputs.python_version }} uninstall -y torch torchvision || true
68+ echo "Pre-installed torch/torchvision uninstalled"
69+ 
70+ - name: Install PyTorch ${{ inputs.pytorch_version }} and built torch_npu
71+ shell: bash
72+ run: |
73+ source /usr/local/Ascend/cann/set_env.sh 2>/dev/null || true
74+ source /usr/local/Ascend/nnal/atb/set_env.sh 2>/dev/null || true
75+ 
76+ PIP=pip${{ inputs.python_version }}
77+ PYTHON=python${{ inputs.python_version }}
78+ export PIP_CACHE_DIR=/github/home/.cache/pip
79+ 
80+ $PIP install --upgrade pip setuptools wheel
81+ $PIP install torch==${{ inputs.pytorch_version }} --index-url https://download.pytorch.org/whl/cpu
82+ 
83+ TORCH_NPU_WHL=$(ls torch-npu-wheel-artifact/*.whl | head -1)
84+ $PIP install "${TORCH_NPU_WHL}"
85+ 
86+ echo "PyTorch ${{ inputs.pytorch_version }} and torch_npu installed from ${TORCH_NPU_WHL}"
87+ 
88+ - name: Verify NPU device
89+ shell: bash
90+ run: |
91+ source /usr/local/Ascend/cann/set_env.sh 2>/dev/null || true
92+ source /usr/local/Ascend/nnal/atb/set_env.sh 2>/dev/null || true
93+ 
94+ echo "=== NPU Device Information ==="
95+ npu-smi info
96+ echo "=== End of NPU Device Information ==="
97+ 
98+ - name: Install triton-ascend
99+ shell: bash
100+ run: |
101+ PIP=pip${{ inputs.python_version }}
102+ PYTHON=python${{ inputs.python_version }}
103+ 
104+ # Map Python version to triton-ascend wheel naming
105+ PY_VER=$(echo "${{ inputs.python_version }}" | tr -d '.')
106+ TRITON_ASCEND_WHL="triton_ascend-3.2.1-cp${PY_VER}-cp${PY_VER}-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl"
107+ TRITON_ASCEND_URL="https://gitcode.com/Ascend/triton-ascend/releases/download/v3.2.1/${TRITON_ASCEND_WHL}"
108+ 
109+ echo "=== Installing triton-ascend for Python ${{ inputs.python_version }} ==="
110+ echo "Download URL: ${TRITON_ASCEND_URL}"
111+ curl -sL "${TRITON_ASCEND_URL}" -o "/tmp/${TRITON_ASCEND_WHL}" || echo "triton-ascend wheel download failed"
112+ if [ -f "/tmp/${TRITON_ASCEND_WHL}" ]; then
113+ if $PIP install "/tmp/${TRITON_ASCEND_WHL}"; then
114+ echo "triton-ascend installed successfully"
115+ else
116+ echo "triton-ascend installation failed"
117+ fi
118+ fi
119+ 
120+ - name: Download prepared test source
121+ uses: actions/download-artifact@v4
122+ with:
123+ name: ${{ inputs.prepared_test_src_artifact }}
124+ path: prepared-test-src-artifact
125+ 
126+ - name: Extract prepared test source
127+ shell: bash
128+ run: |
129+ tar -xzf prepared-test-src-artifact/pytorch-test-src.tar.gz
130+ 
131+ - name: Verify NPU availability
132+ shell: bash
133+ run: |
134+ source /usr/local/Ascend/cann/set_env.sh 2>/dev/null || true
135+ source /usr/local/Ascend/nnal/atb/set_env.sh 2>/dev/null || true
136+ 
137+ PYTHON=python${{ inputs.python_version }}
138+ $PYTHON -c "
139+ import torch
140+ print(f'torch: {torch.__version__}')
141+ import torch_npu
142+ print(f'torch_npu: {torch_npu.__version__}')
143+ print(f'NPU available: {torch.npu.is_available()}')
144+ print(f'NPU count: {torch.npu.device_count()}')
145+ "
146+ 
147+ - name: Install test dependencies
148+ shell: bash
149+ run: |
150+ PIP=pip${{ inputs.python_version }}
151+ export PIP_CACHE_DIR=/github/home/.cache/pip
152+ 
153+ echo "=== Installing PyTorch requirements.txt ==="
154+ cd pytorch-test-src
155+ if [ -f requirements.txt ]; then
156+ $PIP install -r requirements.txt || echo "Some PyTorch dev dependencies may not be available"
157+ fi
158+ 
159+ echo "=== Installing PyTorch CI requirements ==="
160+ if [ -f .ci/docker/requirements-ci.txt ]; then
161+ $PIP install -r .ci/docker/requirements-ci.txt || echo "Some CI dependencies may not be available on this platform"
162+ fi
163+ 
164+ echo "=== Installing ascend_pytorch test requirements ==="
165+ cd ../ascend_pytorch
166+ if [ -f test/requirements.txt ]; then
167+ $PIP install -r test/requirements.txt || echo "Some torch_npu test dependencies may not be available"
168+ fi
169+ 
170+ cd ../pytorch-test-src
171+ 
172+ echo "=== Upgrading ml-dtypes to 0.5.4 ==="
173+ $PIP install ml-dtypes==0.5.4 || echo "ml-dtypes upgrade failed"
174+ 
175+ echo "=== Installed test dependencies ==="
176+ $PIP list | grep -E "pytest|onnx|z3|tensorboard|expecttest|hypothesis|torchvision|torch_geometric|ml-dtypes" | head -30
177+ 
178+ - name: Apply torch environment patches
179+ shell: bash
180+ run: |
181+ source /usr/local/Ascend/cann/set_env.sh 2>/dev/null || true
182+ source /usr/local/Ascend/nnal/atb/set_env.sh 2>/dev/null || true
183+ 
184+ PYTHON=python${{ inputs.python_version }}
185+ cd ascend_pytorch/test_upstream
186+ chmod +x torch_env_patch.sh
187+ 
188+ echo "=== Applying torch environment patches ==="
189+ ./torch_env_patch.sh --python=${{ inputs.python_version }} --verbose 2>&1 | tee /tmp/torch_env_patch_${{ inputs.patch_log_suffix }}.log
190+ PATCH_STATUS=$?
191+ 
192+ if [ ${PATCH_STATUS} -ne 0 ]; then
193+ echo "WARNING: Torch environment patch application returned non-zero status: ${PATCH_STATUS}"
194+ echo "Tests will continue, but some may fail due to missing patches"
195+ fi
A.github/scripts/collect_all_cases.py+499-0
@@ -0,0 +1,499 @@
1+#!/usr/bin/env python3
2+"""
3+Collect all test cases and split into shards.
4+ 
5+This script runs in prepare job (once) to:
6+1. Discover test files by type (distributed/regular)
7+2. Collect all test cases via pytest --collect-only
8+3. Split cases evenly into N shards
9+4. Output shard JSON files for each type
10+5. Save collection error logs for failed files
11+ 
12+Usage:
13+ python collect_all_cases.py \
14+ --test-dir /path/to/pytorch/test \
15+ --case-paths-config /path/to/case_paths_ci.yml \
16+ --distributed-shards 2 \
17+ --regular-shards 5 \
18+ --output-dir /path/to/output \
19+ --error-log-dir /path/to/error_logs \
20+ --parallel 16
21+"""
22+ 
23+import argparse
24+import json
25+import os
26+import subprocess
27+import sys
28+from concurrent.futures import ThreadPoolExecutor, as_completed
29+from pathlib import Path
30+from typing import Dict, List, Tuple
31+ 
32+# Import discover_test_files module
33+import discover_test_files
34+ 
35+ 
36+def _normalize_test_file_path(test_file: str) -> str:
37+ """
38+ Remove 'test/' prefix from test file path if present.
39+ 
40+ Args:
41+ test_file: Test file path (e.g., "test/distributed/pipelining/test_backward.py")
42+ 
43+ Returns:
44+ Relative path without 'test/' prefix
45+ """
46+ if test_file.startswith("test/"):
47+ return test_file[5:]
48+ return test_file
49+ 
50+ 
51+def get_test_file_parent_dir(test_file: str, test_dir: Path) -> Path:
52+ """
53+ Get the parent directory of a test file.
54+ 
55+ This directory should be added to PYTHONPATH to enable
56+ imports of sibling modules (e.g., model_registry.py).
57+ 
58+ Args:
59+ test_file: Test file path (e.g., "test/distributed/pipelining/test_backward.py")
60+ test_dir: Path to PyTorch test directory
61+ 
62+ Returns:
63+ Path to the test file's parent directory
64+ """
65+ test_file_rel = _normalize_test_file_path(test_file)
66+ test_file_path = Path(test_file_rel)
67+ return test_dir / test_file_path.parent
68+ 
69+ 
70+def collect_cases_for_file(test_file: str, test_dir: Path) -> Tuple[str, str, List[str], bool, str]:
71+ """
72+ Collect test cases from a single file.
73+ 
74+ Adds test file's parent directory to PYTHONPATH to enable
75+ imports of sibling modules (e.g., 'from model_registry import MLPModule').
76+ 
77+ Returns:
78+ Tuple of (test_file, display_name, nodeids, success, error_message)
79+ - test_file: Original test file path
80+ - display_name: Short name for logging (remove test/ prefix and .py suffix)
81+ - nodeids: List of collected test case nodeids
82+ - success: True if collection succeeded without errors
83+ - error_message: Error details if collection failed, empty string otherwise
84+ """
85+ test_file_rel = _normalize_test_file_path(test_file)
86+ 
87+ # Extract display name (remove .py suffix)
88+ display_name = test_file_rel
89+ if display_name.endswith(".py"):
90+ display_name = display_name[:-3]
91+ 
92+ # Get test file's parent directory for PYTHONPATH
93+ test_file_dir = get_test_file_parent_dir(test_file, test_dir)
94+ 
95+ # Build environment with test file directory in PYTHONPATH
96+ env = os.environ.copy()
97+ existing_pythonpath = env.get("PYTHONPATH", "")
98+ env["PYTHONPATH"] = str(test_file_dir) + (":" + existing_pythonpath if existing_pythonpath else "")
99+ 
100+ command = [
101+ sys.executable,
102+ "-m",
103+ "pytest",
104+ "--collect-only",
105+ "--quiet",
106+ test_file_rel,
107+ ]
108+ 
109+ try:
110+ result = subprocess.run(
111+ command,
112+ cwd=str(test_dir),
113+ env=env,
114+ capture_output=True,
115+ text=True,
116+ encoding="utf-8",
117+ errors="replace",
118+ timeout=120,
119+ )
120+ 
121+ nodeids = []
122+ for line in result.stdout.splitlines():
123+ stripped = line.strip()
124+ # pytest --collect-only -q outputs clean nodeids, one per line
125+ # Filter rules:
126+ # 1. Skip empty lines
127+ # 2. Skip summary lines (contain "collected" or "selected")
128+ # 3. Skip separator lines (start with "=")
129+ # 4. Must contain ".py::" to ensure it's a Python test file nodeid
130+ if not stripped:
131+ continue
132+ if "collected" in stripped or "selected" in stripped:
133+ continue
134+ if stripped.startswith("="):
135+ continue
136+ if ".py::" in stripped:
137+ nodeids.append(stripped)
138+ 
139+ # Check for collection errors based on pytest exit codes:
140+ # 0: all passed (success)
141+ # 2: pytest error (includes collection errors like ImportError)
142+ # 3: all skipped (success)
143+ # 4: command line error (error)
144+ # 5: no tests collected (ERROR - test file should have cases)
145+ # Key insight: if a test file is selected for execution, it should have cases.
146+ # returncode 5 means 0 cases collected, which indicates a problem.
147+ if result.returncode in (0, 3):
148+ # Normal: passed or skipped
149+ return (test_file, display_name, nodeids, True, "")
150+ else:
151+ # returncode 2, 4, 5: real collection error
152+ # returncode 5 specifically means no tests collected - a problem for selected files
153+ error_msg = result.stdout.strip()
154+ if result.stderr.strip():
155+ error_msg += "\n--- stderr ---\n" + result.stderr.strip()
156+ return (test_file, display_name, nodeids, False, error_msg)
157+ 
158+ except subprocess.TimeoutExpired:
159+ error_msg = f"TIMEOUT: Collection took >120s for {display_name}"
160+ return (test_file, display_name, [], False, error_msg)
161+ except Exception as e:
162+ error_msg = f"ERROR: {e}"
163+ return (test_file, display_name, [], False, error_msg)
164+ 
165+ 
166+def collect_all_cases(
167+ test_files: List[str],
168+ test_dir: Path,
169+ error_log_dir: Path,
170+ parallel: int = 16,
171+) -> List[Dict]:
172+ """
173+ Collect all cases from all files.
174+ 
175+ Args:
176+ test_files: List of test file paths
177+ test_dir: Path to PyTorch test directory
178+ error_log_dir: Directory to save error logs for failed collections
179+ parallel: Number of parallel workers
180+ 
181+ Returns:
182+ List of dicts with nodeid and file for each collected case
183+ """
184+ all_cases = []
185+ failed_files = [] # Track files with collection errors for logging
186+ 
187+ print(f"Collecting cases from {len(test_files)} files with {parallel} workers...")
188+ print("=" * 60)
189+ 
190+ # Create error log directory
191+ error_log_dir.mkdir(parents=True, exist_ok=True)
192+ 
193+ with ThreadPoolExecutor(max_workers=parallel) as executor:
194+ futures = {
195+ executor.submit(collect_cases_for_file, f, test_dir): f
196+ for f in test_files
197+ }
198+ 
199+ completed = 0
200+ successful_count = 0
201+ failed_count = 0
202+ total_cases = 0
203+ 
204+ for future in as_completed(futures):
205+ test_file, display_name, nodeids, success, error_msg = future.result()
206+ completed += 1
207+ 
208+ if success:
209+ successful_count += 1
210+ # Print concise log for successful files
211+ print(f" {display_name}: {len(nodeids)} cases")
212+ for nodeid in nodeids:
213+ all_cases.append({
214+ "nodeid": nodeid,
215+ "file": test_file,
216+ })
217+ else:
218+ failed_count += 1
219+ # Print concise log for failed files
220+ print(f" [FAILED] {display_name}: {len(nodeids)} cases")
221+ # Save error details to log file
222+ failed_files.append({
223+ "file": display_name,
224+ "error": error_msg,
225+ "cases": len(nodeids),
226+ "test_file": test_file,
227+ })
228+ # Still add any cases that were collected despite errors
229+ for nodeid in nodeids:
230+ all_cases.append({
231+ "nodeid": nodeid,
232+ "file": test_file,
233+ })
234+ 
235+ # Update total cases count for progress display
236+ total_cases += len(nodeids)
237+ 
238+ # Print progress summary every 100 files
239+ if completed % 100 == 0:
240+ print(f" [Progress: {completed}/{len(test_files)} files, {successful_count} ok, {failed_count} failed, {total_cases} cases]")
241+ 
242+ print("=" * 60)
243+ 
244+ # Save error logs to files
245+ if failed_files:
246+ save_error_logs(failed_files, error_log_dir)
247+ 
248+ # Final summary
249+ print(f"Collection complete: {len(all_cases)} cases from {successful_count}/{len(test_files)} files")
250+ if failed_count > 0:
251+ print(f" WARNING: {failed_count} files had collection errors (logs saved to {error_log_dir})")
252+ 
253+ return all_cases
254+ 
255+ 
256+def save_error_logs(failed_files: List[Dict], error_log_dir: Path) -> None:
257+ """
258+ Save collection error logs to individual files and create a summary.
259+ 
260+ Args:
261+ failed_files: List of dicts with file, error, cases info
262+ error_log_dir: Directory to save error logs
263+ """
264+ print(f"Saving error logs for {len(failed_files)} failed files...")
265+ 
266+ # Save individual error log files
267+ for failed in failed_files:
268+ # Create safe filename from display name (replace / with _)
269+ safe_name = failed['file'].replace('/', '_')
270+ log_file = error_log_dir / f"{safe_name}.log"
271+ 
272+ # Write error log
273+ with open(log_file, 'w', encoding='utf-8') as f:
274+ f.write(f"File: {failed['file']}\n")
275+ f.write(f"Cases collected: {failed['cases']}\n")
276+ f.write(f"Test file path: {failed['test_file']}\n")
277+ f.write("=" * 80 + "\n")
278+ f.write("Collection Error:\n")
279+ f.write("=" * 80 + "\n")
280+ f.write(failed['error'])
281+ f.write("\n")
282+ 
283+ # Save summary JSON
284+ summary_file = error_log_dir / "collection_errors_summary.json"
285+ summary_data = {
286+ "total_failed": len(failed_files),
287+ "failed_files": [
288+ {
289+ "file": f['file'],
290+ "cases": f['cases'],
291+ "test_file": f['test_file'],
292+ "log_file": f"{f['file'].replace('/', '_')}.log",
293+ }
294+ for f in failed_files
295+ ],
296+ }
297+ summary_file.write_text(json.dumps(summary_data, indent=2), encoding='utf-8')
298+ 
299+ print(f" Error logs saved to {error_log_dir}")
300+ print(f" Summary: {summary_file}")
301+ 
302+ 
303+def split_cases_into_shards(cases: List[Dict], num_shards: int) -> List[List[Dict]]:
304+ """Split cases evenly into shards."""
305+ total = len(cases)
306+ base_size = total // num_shards
307+ remainder = total % num_shards
308+ 
309+ shards = []
310+ start = 0
311+ for i in range(num_shards):
312+ size = base_size + (1 if i < remainder else 0)
313+ shards.append(cases[start:start + size])
314+ start += size
315+ 
316+ return shards
317+ 
318+ 
319+def save_cases_by_file(
320+ cases: List[Dict],
321+ test_files: List[str],
322+ test_type: str,
323+ output_dir: Path,
324+) -> Dict:
325+ """
326+ Save cases grouped by file in JSONL format.
327+ 
328+ Includes all test files, even those with 0 cases collected.
329+ 
330+ Output format (JSONL, one JSON object per line):
331+ Line 1: {"total_file":<count>,"total_cases":<count>}
332+ Line 2+: {"file_path":"...","case_count":<count>,"cases":["nodeid1","nodeid2",...]}
333+ """
334+ # Group cases by file
335+ file_groups: Dict[str, List[str]] = {}
336+ for case in cases:
337+ file_path = case["file"]
338+ if file_path not in file_groups:
339+ file_groups[file_path] = []
340+ file_groups[file_path].append(case["nodeid"])
341+ 
342+ output_file = output_dir / f"{test_type}_cases_by_file.jsonl"
343+ with open(output_file, 'w', encoding='utf-8') as f:
344+ # Line 1: summary
345+ summary_line = json.dumps({
346+ "total_file": len(test_files),
347+ "total_cases": len(cases),
348+ }, separators=(',', ':'))
349+ f.write(summary_line + '\n')
350+ 
351+ # Line 2+: file data (sorted by file path)
352+ for file_path in sorted(test_files):
353+ nodeids = file_groups.get(file_path, [])
354+ file_line = json.dumps({
355+ "file_path": file_path,
356+ "case_count": len(nodeids),
357+ "cases": nodeids,
358+ }, separators=(',', ':'))
359+ f.write(file_line + '\n')
360+ 
361+ print(f" Cases by file (JSONL): {len(test_files)} files -> {output_file}")
362+ 
363+ return {
364+ "test_type": test_type,
365+ "total_files": len(test_files),
366+ "total_cases": len(cases),
367+ }
368+ 
369+ 
370+def save_shards(
371+ cases: List[Dict],
372+ num_shards: int,
373+ test_type: str,
374+ output_dir: Path,
375+) -> Dict:
376+ """Save shard JSONs and return summary."""
377+ shards = split_cases_into_shards(cases, num_shards)
378+ 
379+ print(f"\nSaving {test_type} shards...")
380+ for i, shard_cases in enumerate(shards, 1):
381+ shard_file = output_dir / f"{test_type}_cases_shard_{i}.json"
382+ shard_data = {
383+ "shard": i,
384+ "num_shards": num_shards,
385+ "test_type": test_type,
386+ "total_cases": len(shard_cases),
387+ "cases": shard_cases,
388+ }
389+ shard_file.write_text(json.dumps(shard_data, indent=2), encoding="utf-8")
390+ print(f" Shard {i}: {len(shard_cases)} cases -> {shard_file}")
391+ 
392+ return {
393+ "test_type": test_type,
394+ "num_shards": num_shards,
395+ "total_cases": len(cases),
396+ "shard_sizes": [len(s) for s in shards],
397+ }
398+ 
399+ 
400+def main():
401+ args = parse_args()
402+ 
403+ test_dir = Path(args.test_dir).resolve()
404+ output_dir = Path(args.output_dir).resolve()
405+ output_dir.mkdir(parents=True, exist_ok=True)
406+ 
407+ # Error log directory for failed collections
408+ error_log_dir = Path(args.error_log_dir).resolve() if args.error_log_dir else output_dir / "collection_errors"
409+ error_log_dir.mkdir(parents=True, exist_ok=True)
410+ 
411+ # ========================================
412+ # Step 1: Collect distributed test cases
413+ # ========================================
414+ print("=" * 80)
415+ print("Collecting distributed test cases")
416+ print("=" * 80)
417+ 
418+ dist_files, dist_meta = discover_test_files.discover_test_files(
419+ test_dir=test_dir,
420+ test_type="distributed",
421+ case_paths_config=args.case_paths_config,
422+ )
423+ print(f"Found {len(dist_files)} distributed test files")
424+ 
425+ dist_cases = collect_all_cases(dist_files, test_dir, error_log_dir / "distributed", args.parallel)
426+ print(f"Total distributed cases: {len(dist_cases)}")
427+ 
428+ dist_summary = save_shards(dist_cases, args.distributed_shards, "distributed", output_dir)
429+ save_cases_by_file(dist_cases, dist_files, "distributed", output_dir)
430+ 
431+ # ========================================
432+ # Step 2: Collect regular test cases
433+ # ========================================
434+ print("\n" + "=" * 80)
435+ print("Collecting regular test cases")
436+ print("=" * 80)
437+ 
438+ reg_files, reg_meta = discover_test_files.discover_test_files(
439+ test_dir=test_dir,
440+ test_type="regular",
441+ case_paths_config=args.case_paths_config,
442+ )
443+ print(f"Found {len(reg_files)} regular test files")
444+ 
445+ reg_cases = collect_all_cases(reg_files, test_dir, error_log_dir / "regular", args.parallel)
446+ print(f"Total regular cases: {len(reg_cases)}")
447+ 
448+ reg_summary = save_shards(reg_cases, args.regular_shards, "regular", output_dir)
449+ save_cases_by_file(reg_cases, reg_files, "regular", output_dir)
450+ 
451+ # ========================================
452+ # Step 3: Save overall summary
453+ # ========================================
454+ # Calculate file counts (distributed + regular = total_files, no overlap)
455+ dist_selected = dist_meta.get("type_selected", 0)
456+ reg_selected = reg_meta.get("type_selected", 0)
457+ # total_files is same for both (all test_*.py files), use dist_meta
458+ total_files = dist_meta.get("total_files", 0)
459+ 
460+ overall_summary = {
461+ "distributed": {
462+ "cases_summary": dist_summary,
463+ "discovery_metadata": dist_meta,
464+ },
465+ "regular": {
466+ "cases_summary": reg_summary,
467+ "discovery_metadata": reg_meta,
468+ },
469+ "total_cases": len(dist_cases) + len(reg_cases),
470+ "total_files_scanned": total_files,
471+ "distributed_files": dist_selected,
472+ "regular_files": reg_selected,
473+ }
474+ summary_file = output_dir / "cases_collection_summary.json"
475+ summary_file.write_text(json.dumps(overall_summary, indent=2), encoding="utf-8")
476+ print(f"\nOverall summary saved to {summary_file}")
477+ 
478+ print("\n" + "=" * 80)
479+ print("Collection Complete")
480+ print("=" * 80)
481+ print(f"Distributed: {len(dist_cases)} cases -> {args.distributed_shards} shards (serial execution)")
482+ print(f"Regular: {len(reg_cases)} cases -> {args.regular_shards} shards (parallel execution)")
483+ print(f"Total: {len(dist_cases) + len(reg_cases)} cases")
484+ 
485+ 
486+def parse_args():
487+ parser = argparse.ArgumentParser(description="Collect and shard test cases")
488+ parser.add_argument("--test-dir", required=True, help="PyTorch test directory")
489+ parser.add_argument("--case-paths-config", help="case_paths_ci.yml path")
490+ parser.add_argument("--distributed-shards", type=int, default=2, help="Distributed test shards")
491+ parser.add_argument("--regular-shards", type=int, default=5, help="Regular test shards")
492+ parser.add_argument("--output-dir", required=True, help="Output directory for shard JSONs")
493+ parser.add_argument("--error-log-dir", help="Output directory for collection error logs (default: output-dir/collection_errors)")
494+ parser.add_argument("--parallel", type=int, default=16, help="Parallel collection workers")
495+ return parser.parse_args()
496+ 
497+ 
498+if __name__ == "__main__":
499+ main()
A.github/scripts/discover_test_files.py+341-0
@@ -0,0 +1,341 @@
1+#!/usr/bin/env python3
2+"""
3+Discover test files for PyTorch NPU testing.
4+ 
5+This script integrates 3 steps:
6+ Step 1: Test file discovery (scan all test_*.py)
7+ Step 2: Shard type filtering (distributed/regular)
8+ Step 3: Whitelist/blacklist filtering (case_paths_ci.yml)
9+ 
10+Output: Sorted list of test file paths (with 'test/' prefix)
11+ 
12+Usage:
13+ python discover_test_files.py \
14+ --test-dir /path/to/pytorch/test \
15+ --test-type distributed \
16+ --case-paths-config /path/to/case_paths_ci.yml \
17+ --output /path/to/output_file.txt
18+ 
19+ # Or output to stdout:
20+ python discover_test_files.py \
21+ --test-dir /path/to/pytorch/test \
22+ --test-type regular \
23+ --case-paths-config /path/to/case_paths_ci.yml
24+"""
25+ 
26+import argparse
27+import json
28+import sys
29+from pathlib import Path
30+from typing import Dict, List, Optional, Tuple
31+ 
32+try:
33+ import yaml
34+except ImportError:
35+ yaml = None
36+ 
37+ 
38+# ==============================================================================
39+# Path Normalization Functions
40+# ==============================================================================
41+ 
42+ 
43+def normalize_path(value: str) -> str:
44+ """Normalize path: convert backslashes, remove ./ prefix."""
45+ normalized = value.replace("\\", "/").strip()
46+ while normalized.startswith("./"):
47+ normalized = normalized[2:]
48+ return normalized.strip("/")
49+ 
50+ 
51+def normalize_rule_path(rule: str) -> str:
52+ """Normalize rule path: ensure it has 'test/' prefix."""
53+ normalized = normalize_path(rule)
54+ if not normalized:
55+ return ""
56+ if normalized == "test" or normalized.startswith("test/"):
57+ return normalized.rstrip("/")
58+ return f"test/{normalized}".rstrip("/")
59+ 
60+ 
61+# ==============================================================================
62+# YAML Parsing Functions
63+# ==============================================================================
64+ 
65+ 
66+def parse_simple_yaml_lists(raw_text: str) -> Dict[str, List[str]]:
67+ """Parse YAML file for whitelist/blacklist without yaml library."""
68+ parsed = {"whitelist": [], "blacklist": []}
69+ current_key = None
70+ 
71+ for raw_line in raw_text.splitlines():
72+ without_comment = raw_line.split("#", 1)[0].rstrip()
73+ if not without_comment.strip():
74+ continue
75+ 
76+ stripped = without_comment.lstrip()
77+ if not raw_line.startswith((" ", "\t")) and stripped.endswith(":"):
78+ key = stripped[:-1].strip()
79+ current_key = key if key in parsed else None
80+ continue
81+ 
82+ if current_key and stripped.startswith("- "):
83+ value = stripped[2:].strip().strip("\"'")
84+ if value:
85+ parsed[current_key].append(value)
86+ 
87+ return parsed
88+ 
89+ 
90+def coerce_rule_list(value, key: str) -> List[str]:
91+ """Validate and normalize rule list."""
92+ if value is None:
93+ return []
94+ if not isinstance(value, list):
95+ raise ValueError(f"Expected '{key}' to be a list, got {type(value).__name__}")
96+ 
97+ normalized_values = []
98+ for item in value:
99+ if not isinstance(item, str):
100+ raise ValueError(f"Expected every '{key}' entry to be a string, got {type(item).__name__}")
101+ normalized = normalize_rule_path(item)
102+ if normalized:
103+ normalized_values.append(normalized)
104+ return normalized_values
105+ 
106+ 
107+def load_case_path_rules(config_file: Optional[str]) -> Tuple[str, List[str], List[str]]:
108+ """Load whitelist/blacklist rules from case_paths_ci.yml."""
109+ if not config_file:
110+ return "", [], []
111+ 
112+ config_path = Path(config_file).resolve()
113+ if not config_path.exists():
114+ raise FileNotFoundError(f"case_paths_ci config not found: {config_path}")
115+ 
116+ raw_text = config_path.read_text(encoding="utf-8")
117+ 
118+ if yaml is not None:
119+ payload = yaml.safe_load(raw_text) or {}
120+ else:
121+ payload = parse_simple_yaml_lists(raw_text)
122+ 
123+ if not isinstance(payload, dict):
124+ raise ValueError(f"Expected a YAML object in {config_path}, got {type(payload).__name__}")
125+ 
126+ whitelist = coerce_rule_list(payload.get("whitelist"), "whitelist")
127+ blacklist = coerce_rule_list(payload.get("blacklist"), "blacklist")
128+ return str(config_path), whitelist, blacklist
129+ 
130+ 
131+# ==============================================================================
132+# Test File Discovery (Step 1)
133+# ==============================================================================
134+ 
135+ 
136+def discover_raw_test_files(test_dir: Path) -> List[str]:
137+ """Scan all test_*.py files in test directory."""
138+ files = []
139+ for test_file in test_dir.rglob("test_*.py"):
140+ rel_path = test_file.relative_to(test_dir).as_posix()
141+ files.append(f"test/{rel_path}")
142+ return sorted(files)
143+ 
144+ 
145+# ==============================================================================
146+# Type Filtering (Step 2)
147+# ==============================================================================
148+ 
149+ 
150+def filter_tests_by_type(test_files: List[str], test_type: str) -> Tuple[List[str], List[str]]:
151+ """Filter test files by test type (distributed/regular)."""
152+ if test_type == "distributed":
153+ selected = [f for f in test_files if f.startswith("test/distributed/")]
154+ excluded = [f for f in test_files if not f.startswith("test/distributed/")]
155+ else:
156+ selected = [f for f in test_files if not f.startswith("test/distributed/")]
157+ excluded = [f for f in test_files if f.startswith("test/distributed/")]
158+ return selected, excluded
159+ 
160+ 
161+# ==============================================================================
162+# Path Rules Filtering (Step 3)
163+# ==============================================================================
164+ 
165+ 
166+def path_matches_rule(test_path: str, rule: str) -> bool:
167+ """Check if test path matches a rule (supports glob patterns)."""
168+ import fnmatch
169+ 
170+ normalized_path = normalize_path(test_path)
171+ normalized_rule = normalize_rule_path(rule)
172+ if not normalized_rule:
173+ return False
174+ 
175+ if any(char in normalized_rule for char in "*?[]"):
176+ return fnmatch.fnmatch(normalized_path, normalized_rule)
177+ 
178+ return normalized_path == normalized_rule or normalized_path.startswith(f"{normalized_rule}/")
179+ 
180+ 
181+def apply_case_path_rules(
182+ test_files: List[str], whitelist: List[str], blacklist: List[str]
183+) -> Tuple[List[str], List[str]]:
184+ """Apply whitelist and blacklist rules to filter test files."""
185+ # Apply whitelist (if empty, select all)
186+ if whitelist:
187+ selected = [path for path in test_files if any(path_matches_rule(path, rule) for rule in whitelist)]
188+ else:
189+ selected = list(test_files)
190+ 
191+ # Apply blacklist
192+ if blacklist:
193+ selected = [path for path in selected if not any(path_matches_rule(path, rule) for rule in blacklist)]
194+ 
195+ selected_set = set(selected)
196+ excluded = [path for path in test_files if path not in selected_set]
197+ return selected, excluded
198+ 
199+ 
200+# ==============================================================================
201+# Main Discovery Function
202+# ==============================================================================
203+ 
204+ 
205+def discover_test_files(
206+ test_dir: Path,
207+ test_type: str,
208+ case_paths_config: Optional[str],
209+) -> Tuple[List[str], Dict]:
210+ """
211+ Execute all 3 steps to discover test files.
212+ 
213+ Returns:
214+ Tuple of (selected_files, metadata_dict)
215+ """
216+ # Step 1: Discover all test files
217+ all_test_files = discover_raw_test_files(test_dir)
218+ total_count = len(all_test_files)
219+ 
220+ # Step 2: Filter by test type
221+ type_selected, type_excluded = filter_tests_by_type(all_test_files, test_type)
222+ 
223+ # Step 3: Apply whitelist/blacklist rules
224+ config_path, whitelist, blacklist = load_case_path_rules(case_paths_config)
225+ rules_selected, rules_excluded = apply_case_path_rules(type_selected, whitelist, blacklist)
226+ 
227+ # Metadata for reporting
228+ metadata = {
229+ "test_dir": str(test_dir),
230+ "test_type": test_type,
231+ "total_files": total_count,
232+ "type_selected": len(type_selected),
233+ "type_excluded": len(type_excluded),
234+ "whitelist_entries": len(whitelist),
235+ "blacklist_entries": len(blacklist),
236+ "rules_selected": len(rules_selected),
237+ "rules_excluded": len(rules_excluded),
238+ "case_paths_config": config_path,
239+ }
240+ 
241+ return rules_selected, metadata
242+ 
243+ 
244+# ==============================================================================
245+# CLI Interface
246+# ==============================================================================
247+ 
248+ 
249+def parse_args():
250+ parser = argparse.ArgumentParser(
251+ description="Discover test files for PyTorch NPU testing",
252+ formatter_class=argparse.RawDescriptionHelpFormatter,
253+ epilog=__doc__,
254+ )
255+ parser.add_argument(
256+ "--test-dir",
257+ type=str,
258+ required=True,
259+ help="Path to the PyTorch test directory",
260+ )
261+ parser.add_argument(
262+ "--test-type",
263+ type=str,
264+ choices=["distributed", "regular"],
265+ default="regular",
266+ help="Test type: 'distributed' for distributed tests, 'regular' for other tests",
267+ )
268+ parser.add_argument(
269+ "--case-paths-config",
270+ type=str,
271+ help="Path to case_paths_ci.yml for file-level whitelist/blacklist control",
272+ )
273+ parser.add_argument(
274+ "--output",
275+ type=str,
276+ help="Output file path for test file list (default: stdout)",
277+ )
278+ parser.add_argument(
279+ "--metadata-output",
280+ type=str,
281+ help="Output file path for metadata JSON (optional)",
282+ )
283+ parser.add_argument(
284+ "--verbose",
285+ "-v",
286+ action="store_true",
287+ help="Print verbose output including metadata",
288+ )
289+ return parser.parse_args()
290+ 
291+ 
292+def main():
293+ args = parse_args()
294+ 
295+ test_dir = Path(args.test_dir).resolve()
296+ if not test_dir.is_dir():
297+ raise FileNotFoundError(f"Test directory not found: {test_dir}")
298+ 
299+ # Execute discovery
300+ selected_files, metadata = discover_test_files(
301+ test_dir=test_dir,
302+ test_type=args.test_type,
303+ case_paths_config=args.case_paths_config,
304+ )
305+ 
306+ # Output test file list
307+ output_content = "\n".join(selected_files) + ("\n" if selected_files else "")
308+ 
309+ if args.output:
310+ output_path = Path(args.output).resolve()
311+ output_path.parent.mkdir(parents=True, exist_ok=True)
312+ output_path.write_text(output_content, encoding="utf-8")
313+ if args.verbose:
314+ print(f"Written {len(selected_files)} test files to: {output_path}")
315+ else:
316+ sys.stdout.write(output_content)
317+ 
318+ # Output metadata
319+ if args.metadata_output:
320+ metadata_path = Path(args.metadata_output).resolve()
321+ metadata_path.parent.mkdir(parents=True, exist_ok=True)
322+ metadata_path.write_text(json.dumps(metadata, indent=2), encoding="utf-8")
323+ if args.verbose:
324+ print(f"Written metadata to: {metadata_path}")
325+ 
326+ # Verbose summary
327+ if args.verbose:
328+ print(f"\nDiscovery Summary:")
329+ print(f" Test directory: {test_dir}")
330+ print(f" Test type: {args.test_type}")
331+ print(f" Total files scanned: {metadata['total_files']}")
332+ print(f" After type filter: {metadata['type_selected']} selected, {metadata['type_excluded']} excluded")
333+ if args.case_paths_config:
334+ print(f" Whitelist entries: {metadata['whitelist_entries']}")
335+ print(f" Blacklist entries: {metadata['blacklist_entries']}")
336+ print(f" After rules filter: {metadata['rules_selected']} selected, {metadata['rules_excluded']} excluded")
337+ print(f" Final selected files: {len(selected_files)}")
338+ 
339+ 
340+if __name__ == "__main__":
341+ main()
A.github/scripts/generate_npu_full_test_report.py+892-0
@@ -0,0 +1,892 @@
1+#!/usr/bin/env python3
2+"""
3+Generate a consolidated markdown/json report for the NPU full test workflow.
4+ 
5+Output files:
6+- npu-full-test-summary.json: Lightweight summary with aggregated stats only
7+- distributed_cases_results_by_file.jsonl: Case-level results grouped by file
8+- regular_cases_results_by_file.jsonl: Case-level results grouped by file
9+"""
10+ 
11+import argparse
12+import json
13+import re
14+from collections import Counter
15+from pathlib import Path
16+from typing import Dict, List, Optional, Tuple
17+ 
18+# Import aggregation function from parse_test_results.py
19+import parse_test_results
20+ 
21+ 
22+# ==============================================================================
23+# Status Constants
24+# ==============================================================================
25+ 
26+STATUS_MISSING = "MISSING"
27+STATUS_TIMEOUT = "TIMEOUT"
28+STATUS_INCOMPLETE = "INCOMPLETE"
29+STATUS_ERROR = "ERROR"
30+STATUS_FAILED = "FAILED"
31+STATUS_PASSED = "PASSED"
32+STATUS_NO_TESTS = "NO TESTS"
33+ 
34+ 
35+def parse_args():
36+ parser = argparse.ArgumentParser(description="Generate consolidated NPU full test report")
37+ parser.add_argument("--reports-root", required=True, help="Root directory containing shard report files")
38+ parser.add_argument("--output-markdown", required=True, help="Path to write markdown report")
39+ parser.add_argument("--output-json", required=True, help="Path to write JSON summary")
40+ parser.add_argument("--pytorch-version", required=True, help="PyTorch version string")
41+ parser.add_argument("--torch-npu-whl", required=True, help="torch_npu wheel URL")
42+ parser.add_argument("--patch-count", default="N/A", help="Applied patch count")
43+ parser.add_argument("--shard-matrix-json", required=True, help="JSON array of requested shard ids")
44+ parser.add_argument("--docker-image", default="N/A", help="Docker image used for test execution")
45+ parser.add_argument("--runner", default="N/A", help="Runner machine type")
46+ parser.add_argument("--special-reports-root", help="Root directory containing special test report files")
47+ parser.add_argument("--expected-special-tests-json", default="[]", help="JSON array of expected special test names")
48+ parser.add_argument("--cases-summary", help="Path to cases_collection_summary.json for file discovery stats")
49+ parser.add_argument("--cases-by-file-dir", help="Directory containing *_cases_by_file.jsonl files")
50+ return parser.parse_args()
51+ 
52+ 
53+def load_json_file(path: Path) -> Dict:
54+ """Load JSON file with error handling for malformed/truncated files."""
55+ try:
56+ content = path.read_text(encoding="utf-8")
57+ return json.loads(content)
58+ except json.JSONDecodeError as e:
59+ print(f"Warning: Invalid JSON in {path}: {e}")
60+ print(f" File size: {len(content)} bytes")
61+ # Show context around error position
62+ error_pos = e.pos if hasattr(e, 'pos') else 0
63+ start = max(0, error_pos - 100)
64+ end = min(len(content), error_pos + 100)
65+ print(f" Context around error (pos {error_pos}): ...{content[start:end]}...")
66+ return {}
67+ except Exception as e:
68+ print(f"Warning: Failed to load {path}: {e}")
69+ return {}
70+ 
71+ 
72+def parse_requested_shards(raw: str) -> List[Tuple[str, int]]:
73+ """
74+ Parse shard identifiers from JSON array.
75+ 
76+ Supports formats:
77+ - Integers: [1, 2, 3] -> [("regular", 1), ("regular", 2), ("regular", 3)]
78+ - Type-prefixed: ["dist-1", "reg-2", "custom-1"] -> [("distributed", 1), ("regular", 2), ("custom", 1)]
79+ 
80+ Returns list of (shard_type, shard_number) tuples.
81+ """
82+ try:
83+ value = json.loads(raw)
84+ except json.JSONDecodeError:
85+ return []
86+ 
87+ if not isinstance(value, list):
88+ return []
89+ 
90+ result = []
91+ for item in value:
92+ try:
93+ if isinstance(item, str):
94+ # Parse type-prefixed format: "dist-1", "reg-2", "custom-1"
95+ if "-" in item:
96+ type_prefix, num_str = item.split("-", 1)
97+ if type_prefix == "dist":
98+ shard_type = "distributed"
99+ elif type_prefix == "reg":
100+ shard_type = "regular"
101+ elif type_prefix == "custom":
102+ shard_type = "custom"
103+ else:
104+ # Unknown prefix, skip
105+ continue
106+ shard_num = int(num_str)
107+ result.append((shard_type, shard_num))
108+ else:
109+ # String without prefix, try to parse as int
110+ shard_num = int(item)
111+ result.append(("regular", shard_num))
112+ elif isinstance(item, int):
113+ # Plain integer, assume "regular" type
114+ result.append(("regular", item))
115+ except (TypeError, ValueError):
116+ continue
117+ # Sort by type then number
118+ return sorted(set(result), key=lambda x: (x[0], x[1]))
119+ 
120+ 
121+def parse_expected_special_tests(raw: str) -> List[str]:
122+ try:
123+ value = json.loads(raw)
124+ except json.JSONDecodeError:
125+ return []
126+ 
127+ if not isinstance(value, list):
128+ return []
129+ 
130+ result = []
131+ for item in value:
132+ if isinstance(item, str) and item:
133+ result.append(item)
134+ return sorted(set(result))
135+ 
136+ 
137+def discover_shard_files(
138+ reports_root: Path,
139+) -> Tuple[
140+ Dict[Tuple[str, int], Path], # stats_files
141+ Dict[Tuple[str, int], Path], # info_files
142+ Dict[Tuple[str, int], Path], # cases_files
143+]:
144+ """
145+ Discover all shard report files in the reports directory.
146+ 
147+ Returns dicts keyed by (shard_type, shard_number) tuples.
148+ 
149+ File name format: shard_{type}-{number}_{suffix}
150+ Examples:
151+ - shard_dist-1_stats.json
152+ - shard_reg-1_info.json
153+ - shard_dist-1_cases.json (case-level results)
154+ """
155+ stats_files = {}
156+ info_files = {}
157+ cases_files = {}
158+ 
159+ def parse_shard_filename(path: Path, suffix_pattern: str) -> Optional[Tuple[str, int]]:
160+ """
161+ Parse shard type and number from filename.
162+ 
163+ Filename format: shard_{type}-{number}_{suffix}
164+ e.g., shard_dist-1_stats.json -> ("distributed", 1)
165+ shard_reg-1_stats.json -> ("regular", 1)
166+ shard_custom-1_stats.json -> ("custom", 1)
167+ """
168+ stem = path.stem # filename without extension
169+ # Match pattern: shard_{type}-{number}_{suffix}
170+ match = re.match(r"shard_(dist|reg|custom)-(\d+)_" + suffix_pattern, stem)
171+ if match:
172+ type_prefix = match.group(1)
173+ shard_num = int(match.group(2))
174+ if type_prefix == "dist":
175+ return ("distributed", shard_num)
176+ elif type_prefix == "reg":
177+ return ("regular", shard_num)
178+ elif type_prefix == "custom":
179+ return ("custom", shard_num)
180+ return None
181+ 
182+ for path in reports_root.rglob("shard_*_stats.json"):
183+ key = parse_shard_filename(path, "stats")
184+ if key:
185+ stats_files[key] = path
186+ 
187+ for path in reports_root.rglob("shard_*_info.json"):
188+ key = parse_shard_filename(path, "info")
189+ if key:
190+ info_files[key] = path
191+ 
192+ # Discover case-level results files
193+ for path in reports_root.rglob("shard_*_cases.json"):
194+ key = parse_shard_filename(path, "cases")
195+ if key:
196+ cases_files[key] = path
197+ 
198+ return stats_files, info_files, cases_files
199+ 
200+ 
201+def build_file_to_shards_map(cases_shards_dir: Path) -> Dict[str, List[str]]:
202+ """
203+ Build a mapping from test file path to shard IDs.
204+ 
205+ Scans all shard JSON files in cases_shards_dir and extracts file->shard mapping.
206+ 
207+ Args:
208+ cases_shards_dir: Directory containing shard JSON files like
209+ distributed_cases_shard_1.json, regular_cases_shard_2.json
210+ 
211+ Returns:
212+ Dict mapping file path (e.g., "test/test_ops.py") to list of shard IDs
213+ (e.g., ["dist-1", "reg-2", "reg-3"])
214+ """
215+ file_to_shards = {}
216+ 
217+ if not cases_shards_dir or not cases_shards_dir.exists():
218+ return file_to_shards
219+ 
220+ # Pattern: {test_type}_cases_shard_{num}.json
221+ for shard_file in cases_shards_dir.glob("*_cases_shard_*.json"):
222+ try:
223+ data = load_json_file(shard_file)
224+ test_type = data.get("test_type", "regular")
225+ shard_num = data.get("shard", 0)
226+ 
227+ # Build shard ID: "dist-1" or "reg-2"
228+ shard_prefix = "dist" if test_type == "distributed" else "reg"
229+ shard_id = f"{shard_prefix}-{shard_num}"
230+ 
231+ # Extract file paths from cases
232+ cases = data.get("cases", [])
233+ for case in cases:
234+ file_path = case.get("file", "")
235+ if file_path:
236+ # Normalize file path (remove leading "test/" if present for consistency)
237+ normalized_file = file_path
238+ if normalized_file.startswith("test/"):
239+ normalized_file = normalized_file[5:]
240+ 
241+ if normalized_file not in file_to_shards:
242+ file_to_shards[normalized_file] = []
243+ if shard_id not in file_to_shards[normalized_file]:
244+ file_to_shards[normalized_file].append(shard_id)
245+ except Exception as e:
246+ print(f"Warning: Failed to parse shard file {shard_file}: {e}")
247+ continue
248+ 
249+ # Sort shard IDs for each file
250+ for file_path in file_to_shards:
251+ # Sort by type (dist first) then number
252+ file_to_shards[file_path].sort(key=lambda x: (0 if x.startswith("dist") else 1, int(x.split("-")[1])))
253+ 
254+ return file_to_shards
255+ 
256+ 
257+def get_shard_status(stats: Dict, present: bool) -> str:
258+ if not present:
259+ return STATUS_MISSING
260+ if stats.get("timed_out"):
261+ return STATUS_TIMEOUT
262+ if stats.get("incomplete"):
263+ return STATUS_INCOMPLETE
264+ if stats.get("errors", 0) > 0:
265+ return STATUS_ERROR
266+ if stats.get("failed", 0) > 0:
267+ return STATUS_FAILED
268+ if stats.get("total", 0) == 0:
269+ return STATUS_NO_TESTS
270+ return STATUS_PASSED
271+ 
272+ 
273+def get_overall_status(status_counts: Counter) -> str:
274+ if status_counts[STATUS_MISSING] > 0:
275+ return STATUS_FAILED
276+ if any(status_counts[key] > 0 for key in (STATUS_TIMEOUT, STATUS_INCOMPLETE, STATUS_ERROR, STATUS_FAILED)):
277+ return STATUS_FAILED
278+ if status_counts[STATUS_PASSED] > 0:
279+ return STATUS_PASSED
280+ return STATUS_NO_TESTS
281+ 
282+ 
283+def format_duration(seconds: float) -> str:
284+ seconds = float(seconds)
285+ hours = int(seconds // 3600)
286+ minutes = int((seconds % 3600) // 60)
287+ secs = seconds % 60
288+ if hours > 0:
289+ return f"{hours}h {minutes}m {secs:.1f}s"
290+ if minutes > 0:
291+ return f"{minutes}m {secs:.1f}s"
292+ return f"{secs:.1f}s"
293+ 
294+ 
295+def sanitize_markdown_cell(value: str) -> str:
296+ return value.replace("|", "\\|").replace("\n", "<br>")
297+ 
298+ 
299+def render_table(headers: List[str], rows: List[List[str]]) -> List[str]:
300+ lines = [
301+ "| " + " | ".join(headers) + " |",
302+ "| " + " | ".join(["---"] * len(headers)) + " |",
303+ ]
304+ for row in rows:
305+ lines.append("| " + " | ".join(row) + " |")
306+ return lines
307+ 
308+ 
309+def discover_special_test_files(reports_root: Path | None) -> Dict[str, Path]:
310+ if reports_root is None or not reports_root.exists():
311+ return {}
312+ 
313+ special_files = {}
314+ for path in reports_root.rglob("special_test_*.json"):
315+ try:
316+ payload = load_json_file(path)
317+ except Exception:
318+ continue
319+ name = payload.get("name")
320+ if isinstance(name, str) and name:
321+ special_files[name] = path
322+ return special_files
323+ 
324+ 
325+def load_cases_by_file_jsonl(jsonl_path: Path) -> Tuple[Dict, List[Dict]]:
326+ """
327+ Load cases_by_file.jsonl file.
328+ 
329+ Returns:
330+ Tuple of (summary_dict, file_data_list)
331+ - summary_dict: {"total_file": xxx, "total_cases": xxx}
332+ - file_data_list: [{"file_path": xxx, "case_count": xxx, "cases": [nodeid1, ...]}, ...]
333+ """
334+ if not jsonl_path or not jsonl_path.exists():
335+ return {}, []
336+ 
337+ summary_dict = {}
338+ file_data_list = []
339+ 
340+ try:
341+ with open(jsonl_path, 'r', encoding='utf-8') as f:
342+ for i, line in enumerate(f):
343+ line = line.strip()
344+ if not line:
345+ continue
346+ try:
347+ obj = json.loads(line)
348+ except json.JSONDecodeError:
349+ continue
350+ 
351+ if i == 0 and "total_file" in obj:
352+ # First line is summary
353+ summary_dict = obj
354+ elif "file_path" in obj:
355+ # File data line
356+ file_data_list.append(obj)
357+ except Exception as e:
358+ print(f"Warning: Failed to load {jsonl_path}: {e}")
359+ 
360+ return summary_dict, file_data_list
361+ 
362+ 
363+def build_nodeid_to_case_map(cases_results: Dict) -> Dict[str, Dict]:
364+ """
365+ Build a mapping from nodeid to case execution result.
366+ 
367+ Args:
368+ cases_results: Dict from shard_key -> cases_data
369+ 
370+ Returns:
371+ Dict mapping nodeid -> case result dict
372+ """
373+ nodeid_to_case = {}
374+ for shard_key, cases_data in cases_results.items():
375+ cases_list = cases_data.get("cases", [])
376+ for case in cases_list:
377+ nodeid = case.get("nodeid", "")
378+ if nodeid:
379+ nodeid_to_case[nodeid] = case
380+ return nodeid_to_case
381+ 
382+ 
383+def generate_cases_results_jsonl(
384+ test_type: str,
385+ file_data_list: List[Dict],
386+ summary_dict: Dict,
387+ nodeid_to_case: Dict,
388+ output_dir: Path,
389+) -> Path:
390+ """
391+ Generate JSONL file with case execution results grouped by file.
392+ 
393+ Format:
394+ Line 1: {"total_file":xxx,"total_cases":xxx}
395+ Line 2+: {"file_path":"xxx","case_count":xxx,"cases":[{"nodeid":"xxx","status":"passed",...},...]}
396+ 
397+ Args:
398+ test_type: "distributed" or "regular"
399+ file_data_list: List of file data dicts from *_cases_by_file.jsonl
400+ summary_dict: Summary dict from *_cases_by_file.jsonl
401+ nodeid_to_case: Mapping from nodeid to case execution result
402+ output_dir: Output directory
403+ 
404+ Returns:
405+ Path to generated JSONL file
406+ """
407+ output_file = output_dir / f"{test_type}_cases_results_by_file.jsonl"
408+ 
409+ with open(output_file, 'w', encoding='utf-8') as f:
410+ # Line 1: summary (use compact JSON)
411+ summary_line = json.dumps(summary_dict, separators=(',', ':'))
412+ f.write(summary_line + '\n')
413+ 
414+ # Line 2+: file data with enriched case results
415+ for file_data in file_data_list:
416+ file_path = file_data.get("file_path", "")
417+ nodeids = file_data.get("cases", [])
418+ 
419+ # Enrich nodeids with execution results
420+ enriched_cases = []
421+ for nodeid in nodeids:
422+ case_result = nodeid_to_case.get(nodeid, {})
423+ if case_result:
424+ # Case has execution result
425+ enriched_cases.append({
426+ "nodeid": case_result.get("nodeid", nodeid),
427+ "status": case_result.get("status", "unknown"),
428+ "duration": case_result.get("duration", 0.0),
429+ "returncode": case_result.get("returncode", 0),
430+ "message": case_result.get("message", ""),
431+ "command": case_result.get("command", ""),
432+ "file": case_result.get("file", file_path),
433+ "case_idx": case_result.get("case_idx", 0),
434+ })
435+ else:
436+ # Case not executed (missing from results)
437+ enriched_cases.append({
438+ "nodeid": nodeid,
439+ "status": "not_executed",
440+ "duration": 0.0,
441+ "returncode": 0,
442+ "message": "",
443+ "command": "",
444+ "file": file_path,
445+ "case_idx": 0,
446+ })
447+ 
448+ file_line = json.dumps({
449+ "file_path": file_path,
450+ "case_count": len(enriched_cases),
451+ "cases": enriched_cases,
452+ }, separators=(',', ':'))
453+ f.write(file_line + '\n')
454+ 
455+ print(f"Generated {test_type}_cases_results_by_file.jsonl: {len(file_data_list)} files -> {output_file}")
456+ return output_file
457+ 
458+ 
459+def main():
460+ args = parse_args()
461+ reports_root = Path(args.reports_root)
462+ output_markdown = Path(args.output_markdown)
463+ output_json = Path(args.output_json)
464+ requested_shards = parse_requested_shards(args.shard_matrix_json)
465+ expected_special_tests = parse_expected_special_tests(args.expected_special_tests_json)
466+ special_reports_root = Path(args.special_reports_root) if args.special_reports_root else None
467+ 
468+ # Load cases collection summary for file discovery stats
469+ cases_summary_data = None
470+ file_discovery_stats = {
471+ "total_files_scanned": 0,
472+ "distributed_files": 0,
473+ "regular_files": 0,
474+ }
475+ if args.cases_summary:
476+ cases_summary_path = Path(args.cases_summary)
477+ if cases_summary_path.exists():
478+ cases_summary_data = load_json_file(cases_summary_path)
479+ # Extract file discovery stats (正交: total = distributed + regular)
480+ if cases_summary_data:
481+ file_discovery_stats["total_files_scanned"] = cases_summary_data.get("total_files_scanned", 0)
482+ file_discovery_stats["distributed_files"] = cases_summary_data.get("distributed_files", 0)
483+ file_discovery_stats["regular_files"] = cases_summary_data.get("regular_files", 0)
484+ 
485+ stats_files, info_files, cases_files = discover_shard_files(reports_root)
486+ special_test_files = discover_special_test_files(special_reports_root)
487+ shard_ids = requested_shards or sorted(set(stats_files) | set(info_files) | set(cases_files))
488+ 
489+ # Build file to shards mapping from cases-shards directory
490+ cases_shards_dir = Path(args.cases_summary).parent if args.cases_summary else None
491+ file_to_shards_map = build_file_to_shards_map(cases_shards_dir)
492+ 
493+ status_counts = Counter()
494+ totals = {
495+ "total": 0,
496+ "passed": 0,
497+ "failed": 0,
498+ "errors": 0,
499+ "skipped": 0,
500+ "timeout": 0,
501+ "duration": 0.0,
502+ }
503+ shard_rows = []
504+ selection_modes = set()
505+ cases_results = {} # Store case-level results for each shard
506+ 
507+ for shard_type, shard_num in shard_ids:
508+ shard_key = (shard_type, shard_num)
509+ stats_path = stats_files.get(shard_key)
510+ info_path = info_files.get(shard_key)
511+ cases_path = cases_files.get(shard_key)
512+ stats = load_json_file(stats_path) if stats_path else {}
513+ info = load_json_file(info_path) if info_path else {}
514+ 
515+ # Load case-level results if available
516+ cases_data = load_json_file(cases_path) if cases_path else {}
517+ if cases_data:
518+ cases_results[shard_key] = cases_data
519+ # Override stats with case-level data
520+ stats["total"] = cases_data.get("total_cases", 0)
521+ stats["passed"] = cases_data.get("passed", 0)
522+ stats["failed"] = cases_data.get("failed", 0)
523+ stats["errors"] = cases_data.get("errors", 0)
524+ stats["skipped"] = cases_data.get("skipped", 0)
525+ stats["timeout"] = cases_data.get("timeout", 0)
526+ stats["duration"] = cases_data.get("duration", 0.0)
527+ # Update totals (正交累加: total = passed + failed + errors + skipped + timeout)
528+ totals["total"] += cases_data.get("total_cases", 0)
529+ totals["passed"] += cases_data.get("passed", 0)
530+ totals["failed"] += cases_data.get("failed", 0)
531+ totals["errors"] += cases_data.get("errors", 0)
532+ totals["skipped"] += cases_data.get("skipped", 0)
533+ totals["timeout"] += cases_data.get("timeout", 0)
534+ totals["duration"] += cases_data.get("duration", 0.0)
535+ 
536+ present = bool(stats_path or cases_path)
537+ 
538+ if info.get("selection_mode"):
539+ selection_modes.add(str(info.get("selection_mode")))
540+ 
541+ status = get_shard_status(stats, present)
542+ status_counts[status] += 1
543+ 
544+ # Convert shard_type to display prefix ("distributed" -> "dist", "regular" -> "reg", "custom" -> "custom")
545+ if shard_type == "distributed":
546+ shard_prefix = "dist"
547+ elif shard_type == "custom":
548+ shard_prefix = "custom"
549+ else:
550+ shard_prefix = "reg"
551+ shard_rows.append(
552+ {
553+ "shard": f"{shard_prefix}-{shard_num}", # "dist-1", "reg-1", or "custom-1"
554+ "shard_type": shard_type,
555+ "shard_num": shard_num,
556+ "status": status,
557+ "total": int(stats.get("total", 0)),
558+ "passed": int(stats.get("passed", 0)),
559+ "failed": int(stats.get("failed", 0)),
560+ "skipped": int(stats.get("skipped", 0)),
561+ "errors": int(stats.get("errors", 0)),
562+ "timeout": int(stats.get("timeout", 0)),
563+ "duration": float(stats.get("duration", 0.0)),
564+ }
565+ )
566+ 
567+ overall_status = get_overall_status(status_counts)
568+ whl_name = Path(args.torch_npu_whl).name
569+ received_reports = len(stats_files)
570+ expected_reports = len(shard_ids)
571+ selection_mode_display = ", ".join(sorted(selection_modes)) if selection_modes else "-"
572+ 
573+ # Show all shards in the detail table
574+ sorted_shards = sorted(shard_rows, key=lambda row: (row["shard_type"], row["shard_num"]))
575+ special_test_names = expected_special_tests or sorted(special_test_files)
576+ special_test_rows = []
577+ special_status_counts = Counter()
578+ 
579+ for test_name in special_test_names:
580+ payload = load_json_file(special_test_files[test_name]) if test_name in special_test_files else {}
581+ status = str(payload.get("status", "MISSING"))
582+ special_status_counts[status] += 1
583+ special_test_rows.append(
584+ {
585+ "name": test_name,
586+ "group": str(payload.get("group", "-")),
587+ "status": status,
588+ "duration": float(payload.get("duration", 0.0)),
589+ "returncode": payload.get("returncode", "-"),
590+ "note": str(payload.get("note", "") or "-"),
591+ }
592+ )
593+ 
594+ if any(row["status"] != STATUS_PASSED for row in special_test_rows):
595+ overall_status = STATUS_FAILED
596+ 
597+ include_special_tests = bool(special_test_names or special_test_rows)
598+ 
599+ # Build Selection row content based on available data
600+ if cases_summary_data:
601+ # Use file discovery stats from cases_collection_summary.json
602+ total_scanned = file_discovery_stats["total_files_scanned"]
603+ dist_files = file_discovery_stats["distributed_files"]
604+ reg_files = file_discovery_stats["regular_files"]
605+ selection_content = (
606+ f"扫描发现 {total_scanned} 个测试文件 "
607+ f"(distributed: {dist_files}, regular: {reg_files})"
608+ )
609+ else:
610+ # Fallback to original selection mode display
611+ selection_content = selection_mode_display
612+ 
613+ # Extract planned cases count from cases_collection_summary.json
614+ planned_total_cases = 0
615+ planned_dist_cases = 0
616+ planned_reg_cases = 0
617+ if cases_summary_data:
618+ planned_total_cases = cases_summary_data.get("total_cases", 0)
619+ planned_dist_cases = cases_summary_data.get("distributed", {}).get("cases_summary", {}).get("total_cases", 0)
620+ planned_reg_cases = cases_summary_data.get("regular", {}).get("cases_summary", {}).get("total_cases", 0)
621+ 
622+ overview_rows = [
623+ ["Overall result", overall_status],
624+ ["PyTorch", f"`v{args.pytorch_version}`"],
625+ ["torch_npu", f"`{whl_name}`"],
626+ ["Patches applied", str(args.patch_count)],
627+ ["Docker image", f"`{args.docker_image}`"],
628+ ["Runner", f"`{args.runner}`"],
629+ ["Shards", f"{received_reports} / {expected_reports} reported"],
630+ ["Selection", selection_content],
631+ [
632+ "实际执行用例",
633+ (
634+ f"{totals['total']} total; {totals['passed']} passed; {totals['failed']} failed; "
635+ f"{totals['errors']} errors; {totals['skipped']} skipped; "
636+ f"{totals['timeout']} timeout"
637+ ),
638+ ],
639+ ]
640+ # Add planned cases count row if available
641+ if planned_total_cases > 0:
642+ overview_rows.append([
643+ "规划用例总数",
644+ f"{planned_total_cases} (distributed: {planned_dist_cases}, regular: {planned_reg_cases})",
645+ ])
646+ overview_rows.append(["Duration", format_duration(totals["duration"])])
647+ if include_special_tests:
648+ overview_rows.append(["Special tests expected", str(len(special_test_names))])
649+ 
650+ markdown_lines = [
651+ "# PyTorch NPU Full Test Summary",
652+ "",
653+ "## Overview",
654+ ]
655+ markdown_lines.extend(
656+ render_table(
657+ ["Item", "Value"],
658+ overview_rows,
659+ )
660+ )
661+ 
662+ # Add case-level statistics table if available
663+ if cases_results:
664+ markdown_lines.extend(["", "## 用例级执行统计"])
665+ markdown_lines.extend(
666+ render_table(
667+ ["Shard", "总用例", "通过", "失败", "错误", "跳过", "超时", "Duration"],
668+ [
669+ [
670+ f"{row['shard']}",
671+ str(row["total"]),
672+ str(row["passed"]),
673+ str(row["failed"]),
674+ str(row["errors"]),
675+ str(row.get("skipped", 0)),
676+ str(row.get("timeout", 0)),
677+ format_duration(row["duration"]),
678+ ]
679+ for row in sorted_shards
680+ if (row["shard_type"], row["shard_num"]) in cases_results
681+ ],
682+ )
683+ )
684+ 
685+ # Build file-level statistics from jsonl (full file set) + execution results
686+ file_stats = parse_test_results.aggregate_all_cases_by_file(cases_results)
687+ 
688+ # Load all files from jsonl (includes files with 0 cases that weren't executed)
689+ all_files_from_jsonl = {}
690+ if args.cases_by_file_dir:
691+ cases_by_file_dir = Path(args.cases_by_file_dir)
692+ dist_jsonl_path = cases_by_file_dir / "distributed_cases_by_file.jsonl"
693+ reg_jsonl_path = cases_by_file_dir / "regular_cases_by_file.jsonl"
694+ 
695+ if dist_jsonl_path.exists():
696+ _, dist_file_data = load_cases_by_file_jsonl(dist_jsonl_path)
697+ for fd in dist_file_data:
698+ file_path = fd.get("file_path", "")
699+ all_files_from_jsonl[file_path] = {
700+ "file": file_path,
701+ "case_count": fd.get("case_count", 0),
702+ "test_type": "distributed",
703+ }
704+ 
705+ if reg_jsonl_path.exists():
706+ _, reg_file_data = load_cases_by_file_jsonl(reg_jsonl_path)
707+ for fd in reg_file_data:
708+ file_path = fd.get("file_path", "")
709+ all_files_from_jsonl[file_path] = {
710+ "file": file_path,
711+ "case_count": fd.get("case_count", 0),
712+ "test_type": "regular",
713+ }
714+ 
715+ # Merge execution results with full file set
716+ merged_file_stats = {}
717+ for file_path, file_info in all_files_from_jsonl.items():
718+ exec_stats = file_stats.get(file_path, {})
719+ merged_file_stats[file_path] = {
720+ "file": file_path,
721+ "total": exec_stats.get("total", 0),
722+ "passed": exec_stats.get("passed", 0),
723+ "failed": exec_stats.get("failed", 0),
724+ "errors": exec_stats.get("errors", 0),
725+ "timeout": exec_stats.get("timeout", 0),
726+ "skipped": exec_stats.get("skipped", 0),
727+ "duration": exec_stats.get("duration", 0.0),
728+ "case_count": file_info.get("case_count", 0), # 规划用例数（可能 > 执行用例数）
729+ "test_type": file_info.get("test_type", "unknown"),
730+ }
731+ 
732+ # Also add files that were executed but not in jsonl (edge case)
733+ for file_path, exec_stats in file_stats.items():
734+ if file_path not in merged_file_stats:
735+ merged_file_stats[file_path] = {
736+ "file": file_path,
737+ "total": exec_stats.get("total", 0),
738+ "passed": exec_stats.get("passed", 0),
739+ "failed": exec_stats.get("failed", 0),
740+ "errors": exec_stats.get("errors", 0),
741+ "timeout": exec_stats.get("timeout", 0),
742+ "skipped": exec_stats.get("skipped", 0),
743+ "duration": exec_stats.get("duration", 0.0),
744+ "case_count": exec_stats.get("total", 0),
745+ "test_type": "unknown",
746+ }
747+ 
748+ if merged_file_stats:
749+ # Sort files by total cases descending
750+ sorted_files = sorted(
751+ merged_file_stats.values(),
752+ key=lambda x: (-x["case_count"], x["file"])
753+ )
754+ 
755+ markdown_lines.extend(["", "## 测试文件结果汇总"])
756+ 
757+ file_rows = []
758+ for fs in sorted_files:
759+ # Calculate fail rate based on executed cases
760+ failed_total = fs["failed"] + fs["errors"] + fs["timeout"]
761+ fail_rate = f"{(failed_total / fs['total'] * 100):.1f}%" if fs["total"] > 0 else "0%"
762+ # Get shard info for this file
763+ file_path = fs["file"]
764+ # Normalize file path for lookup (remove leading "test/")
765+ lookup_path = file_path
766+ if lookup_path.startswith("test/"):
767+ lookup_path = lookup_path[5:]
768+ shards_for_file = file_to_shards_map.get(lookup_path, [])
769+ # If case_count is 0, no shard executed this file
770+ shard_info = ", ".join(shards_for_file) if shards_for_file else "-"
771+ file_rows.append([
772+ sanitize_markdown_cell(fs["file"]),
773+ shard_info,
774+ str(fs["case_count"]), # 规划用例数
775+ str(fs["passed"]),
776+ str(fs["failed"]),
777+ str(fs["errors"]),
778+ str(fs["skipped"]),
779+ str(fs["timeout"]),
780+ fail_rate,
781+ ])
782+ 
783+ markdown_lines.extend(
784+ render_table(
785+ ["测试文件", "分片", "规划用例", "通过", "失败", "错误", "跳过", "超时", "失败率"],
786+ file_rows,
787+ )
788+ )
789+ 
790+ if include_special_tests:
791+ markdown_lines.extend(["", "## Special Test Results"])
792+ markdown_lines.extend(
793+ render_table(
794+ ["Test", "Group", "Status", "Duration", "Return Code", "Note"],
795+ [
796+ [
797+ row["name"],
798+ row["group"],
799+ row["status"],
800+ format_duration(row["duration"]),
801+ str(row["returncode"]),
802+ sanitize_markdown_cell(row["note"]),
803+ ]
804+ for row in special_test_rows
805+ ] or [["-", "-", "-", "0.0s", "-", "-"]],
806+ )
807+ )
808+ 
809+ report_json = {
810+ "overall_status": overall_status,
811+ "requested_shards": shard_ids,
812+ "reports_collected": received_reports,
813+ "patch_count": args.patch_count,
814+ "pytorch_version": args.pytorch_version,
815+ "torch_npu_whl": whl_name,
816+ "docker_image": args.docker_image,
817+ "runner": args.runner,
818+ "status_counts": dict(status_counts),
819+ "totals": totals,
820+ "file_discovery_stats": file_discovery_stats,
821+ "planned_cases": {
822+ "total": planned_total_cases,
823+ "distributed": planned_dist_cases,
824+ "regular": planned_reg_cases,
825+ },
826+ "shards": shard_rows,
827+ }
828+ 
829+ # Add cases collection summary (lightweight metadata only for md rendering)
830+ if cases_summary_data:
831+ report_json["cases_collection_summary"] = {
832+ "total_cases": cases_summary_data.get("total_cases", 0),
833+ "total_files_scanned": cases_summary_data.get("total_files_scanned", 0),
834+ "distributed_files": cases_summary_data.get("distributed_files", 0),
835+ "regular_files": cases_summary_data.get("regular_files", 0),
836+ "distributed": {
837+ "total_cases": cases_summary_data.get("distributed", {}).get("cases_summary", {}).get("total_cases", 0),
838+ },
839+ "regular": {
840+ "total_cases": cases_summary_data.get("regular", {}).get("cases_summary", {}).get("total_cases", 0),
841+ },
842+ }
843+ 
844+ # Generate JSONL files with case-level results grouped by file
845+ if cases_results and args.cases_by_file_dir:
846+ cases_by_file_dir = Path(args.cases_by_file_dir)
847+ output_dir = output_json.parent
848+ 
849+ # Build nodeid to case result mapping
850+ nodeid_to_case = build_nodeid_to_case_map(cases_results)
851+ 
852+ # Process distributed cases
853+ dist_jsonl_path = cases_by_file_dir / "distributed_cases_by_file.jsonl"
854+ if dist_jsonl_path.exists():
855+ dist_summary, dist_file_data = load_cases_by_file_jsonl(dist_jsonl_path)
856+ generate_cases_results_jsonl(
857+ "distributed",
858+ dist_file_data,
859+ dist_summary,
860+ nodeid_to_case,
861+ output_dir,
862+ )
863+ 
864+ # Process regular cases
865+ reg_jsonl_path = cases_by_file_dir / "regular_cases_by_file.jsonl"
866+ if reg_jsonl_path.exists():
867+ reg_summary, reg_file_data = load_cases_by_file_jsonl(reg_jsonl_path)
868+ generate_cases_results_jsonl(
869+ "regular",
870+ reg_file_data,
871+ reg_summary,
872+ nodeid_to_case,
873+ output_dir,
874+ )
875+ 
876+ # Add special tests if applicable
877+ if include_special_tests:
878+ report_json["special_tests"] = {
879+ "expected": special_test_names,
880+ "status_counts": dict(special_status_counts),
881+ "results": special_test_rows,
882+ }
883+ 
884+ output_markdown.write_text("\n".join(markdown_lines) + "\n", encoding="utf-8")
885+ output_json.write_text(json.dumps(report_json, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
886+ 
887+ print(f"Generated markdown report: {output_markdown}")
888+ print(f"Generated json report: {output_json}")
889+ 
890+ 
891+if __name__ == "__main__":
892+ main()
A.github/scripts/parse_test_results.py+304-0
@@ -0,0 +1,304 @@
1+#!/usr/bin/env python3
2+"""
3+Utility functions for test result processing.
4+ 
5+This module provides file operations and summary printing for test execution:
6+ - Create shard info dictionaries
7+ - Save results to JSON files (stats, info, cases, test plan)
8+ - Print test summary to stdout
9+ - Aggregate case results by test file
10+ 
11+Usage as module:
12+ from parse_test_results import (
13+ create_shard_info,
14+ get_shard_log_file,
15+ save_stats_file,
16+ save_info_file,
17+ save_cases_file,
18+ save_test_plan_file,
19+ print_stats_summary,
20+ aggregate_all_cases_by_file,
21+ )
22+"""
23+ 
24+import json
25+import os
26+import sys
27+from pathlib import Path
28+from typing import Dict, List, Optional
29+ 
30+ 
31+# ==============================================================================
32+# Stats Processing
33+# ==============================================================================
34+ 
35+ 
36+def create_shard_info(shard: int, num_shards: int, timestamp: str) -> Dict:
37+ """Create shard info dictionary template."""
38+ return {
39+ "shard": shard,
40+ "num_shards": num_shards,
41+ "selection_mode": "pytest_direct",
42+ "total_files": 0,
43+ "selected_test_files": 0,
44+ "shard_files": 0,
45+ "path_filtered_out_files": 0,
46+ "excluded_test_files": 0,
47+ "disabled_count": 0,
48+ "whitelist_entries": 0,
49+ "blacklist_entries": 0,
50+ "junit_generated": False,
51+ "junit_xml_files": 0,
52+ "zero_item_test_files": 0,
53+ "startup_failures": 0,
54+ "import_failures": 0,
55+ "test_failures": 0,
56+ "timestamp": timestamp,
57+ }
58+ 
59+ 
60+# ==============================================================================
61+# Utility Functions
62+# ==============================================================================
63+ 
64+ 
65+def get_shard_type_prefix(shard_type: str) -> str:
66+ """Convert shard type to short prefix for file naming."""
67+ if shard_type == "distributed":
68+ return "dist"
69+ elif shard_type == "custom":
70+ return "custom"
71+ else:
72+ return "reg"
73+ 
74+ 
75+def get_shard_log_file(report_dir: Path, shard: int, shard_type: str = "regular") -> Path:
76+ """Get path for shard log file."""
77+ prefix = get_shard_type_prefix(shard_type)
78+ return report_dir / f"test_shard_{prefix}-{shard}.log"
79+ 
80+ 
81+def load_disabled_testcases_count(json_file: str) -> int:
82+ """Count entries in disabled_testcases.json."""
83+ if not json_file or not os.path.exists(json_file):
84+ return 0
85+ 
86+ with open(json_file, encoding="utf-8") as f:
87+ data = json.load(f)
88+ 
89+ if isinstance(data, (dict, list)):
90+ return len(data)
91+ return 0
92+ 
93+ 
94+# ==============================================================================
95+# File Save Functions
96+# ==============================================================================
97+ 
98+ 
99+def save_stats_file(report_dir: str, shard: int, stats: Dict, shard_type: str = "regular") -> str:
100+ """Save statistics to JSON file."""
101+ os.makedirs(report_dir, exist_ok=True)
102+ prefix = get_shard_type_prefix(shard_type)
103+ stats_file = os.path.join(report_dir, f"shard_{prefix}-{shard}_stats.json")
104+ with open(stats_file, "w", encoding="utf-8") as f:
105+ json.dump(stats, f, indent=2)
106+ return stats_file
107+ 
108+ 
109+def save_info_file(report_dir: str, shard: int, info: Dict, shard_type: str = "regular") -> str:
110+ """Save info to JSON file."""
111+ os.makedirs(report_dir, exist_ok=True)
112+ prefix = get_shard_type_prefix(shard_type)
113+ info_file = os.path.join(report_dir, f"shard_{prefix}-{shard}_info.json")
114+ with open(info_file, "w", encoding="utf-8") as f:
115+ json.dump(info, f, indent=2)
116+ return info_file
117+ 
118+ 
119+def save_test_plan_file(report_dir: str, shard: int, planned_tests: List[str], shard_type: str = "regular") -> str:
120+ """Save planned test files list."""
121+ os.makedirs(report_dir, exist_ok=True)
122+ prefix = get_shard_type_prefix(shard_type)
123+ plan_file = os.path.join(report_dir, f"shard_{prefix}-{shard}_planned_test_files.txt")
124+ with open(plan_file, "w", encoding="utf-8") as f:
125+ for target in planned_tests:
126+ f.write(f"{target}\n")
127+ return plan_file
128+ 
129+ 
130+def save_cases_file(report_dir: str, shard: int, cases_data: Dict, shard_type: str = "regular") -> str:
131+ """Save case-level results to JSON file."""
132+ os.makedirs(report_dir, exist_ok=True)
133+ prefix = get_shard_type_prefix(shard_type)
134+ cases_file = os.path.join(report_dir, f"shard_{prefix}-{shard}_cases.json")
135+ with open(cases_file, "w", encoding="utf-8") as f:
136+ json.dump(cases_data, f, indent=2, ensure_ascii=False)
137+ return cases_file
138+ 
139+ 
140+# ==============================================================================
141+# Case Aggregation by File
142+# ==============================================================================
143+ 
144+ 
145+def aggregate_cases_by_file(cases_list: List[Dict]) -> Dict[str, Dict]:
146+ """
147+ Aggregate case results by test file.
148+ 
149+ This function groups test cases by their source file and computes
150+ statistics (passed, failed, errors, etc.) per file. It also collects
151+ detailed failure information for reporting.
152+ 
153+ Args:
154+ cases_list: List of case result dicts with "nodeid", "file", "status" keys
155+ 
156+ Returns:
157+ Dict mapping test file path -> aggregated stats
158+ Each entry contains:
159+ - file: test file path
160+ - total: total cases in file
161+ - passed, failed, errors, crashed, timeout, skipped: counts
162+ - failed_cases: list of failed/error/crashed/timeout cases with details
163+ - duration: total execution time for file
164+ """
165+ file_stats = {}
166+ 
167+ for case in cases_list:
168+ test_file = case.get("file", "unknown")
169+ if not test_file:
170+ # Try to extract file from nodeid
171+ nodeid = case.get("nodeid", "")
172+ if "::" in nodeid:
173+ test_file = nodeid.split("::")[0]
174+ else:
175+ test_file = "unknown"
176+ 
177+ status = case.get("status", "error")
178+ duration = case.get("duration", 0.0)
179+ 
180+ if test_file not in file_stats:
181+ file_stats[test_file] = {
182+ "file": test_file,
183+ "total": 0,
184+ "passed": 0,
185+ "failed": 0,
186+ "errors": 0,
187+ "timeout": 0,
188+ "skipped": 0,
189+ "failed_cases": [],
190+ "duration": 0.0,
191+ }
192+ 
193+ stats = file_stats[test_file]
194+ stats["total"] += 1
195+ stats["duration"] += duration
196+ 
197+ if status == "passed":
198+ stats["passed"] += 1
199+ elif status == "failed":
200+ stats["failed"] += 1
201+ stats["failed_cases"].append({
202+ "nodeid": case.get("nodeid"),
203+ "status": "failed",
204+ "message": case.get("message", ""),
205+ "duration": duration,
206+ })
207+ elif status == "error":
208+ stats["errors"] += 1
209+ stats["failed_cases"].append({
210+ "nodeid": case.get("nodeid"),
211+ "status": "error",
212+ "message": case.get("message", ""),
213+ "duration": duration,
214+ })
215+ elif status == "timeout":
216+ stats["timeout"] += 1
217+ stats["failed_cases"].append({
218+ "nodeid": case.get("nodeid"),
219+ "status": "timeout",
220+ "message": f"Timeout after {duration}s",
221+ "duration": duration,
222+ })
223+ elif status == "skipped":
224+ stats["skipped"] += 1
225+ 
226+ return file_stats
227+ 
228+ 
229+def aggregate_all_cases_by_file(cases_results: Dict) -> Dict[str, Dict]:
230+ """
231+ Aggregate all cases from multiple shards by test file.
232+ 
233+ Args:
234+ cases_results: Dict mapping shard_key -> cases_data (from shard_*_cases.json)
235+ 
236+ Returns:
237+ Dict mapping test file -> aggregated stats across all shards
238+ """
239+ all_file_stats = {}
240+ 
241+ for shard_key, cases_data in cases_results.items():
242+ shard_cases = cases_data.get("cases", [])
243+ file_stats = aggregate_cases_by_file(shard_cases)
244+ 
245+ for test_file, stats in file_stats.items():
246+ if test_file not in all_file_stats:
247+ all_file_stats[test_file] = {
248+ "file": test_file,
249+ "total": 0,
250+ "passed": 0,
251+ "failed": 0,
252+ "errors": 0,
253+ "timeout": 0,
254+ "skipped": 0,
255+ "failed_cases": [],
256+ "duration": 0.0,
257+ }
258+ 
259+ existing = all_file_stats[test_file]
260+ existing["total"] += stats["total"]
261+ existing["passed"] += stats["passed"]
262+ existing["failed"] += stats["failed"]
263+ existing["errors"] += stats["errors"]
264+ existing["timeout"] += stats["timeout"]
265+ existing["skipped"] += stats["skipped"]
266+ existing["duration"] += stats["duration"]
267+ existing["failed_cases"].extend(stats["failed_cases"])
268+ 
269+ # Sort failed_cases within each file
270+ for test_file in all_file_stats:
271+ all_file_stats[test_file]["failed_cases"].sort(
272+ key=lambda x: x.get("nodeid", "")
273+ )
274+ 
275+ return all_file_stats
276+ 
277+ 
278+# ==============================================================================
279+# Summary Printing
280+# ==============================================================================
281+ 
282+ 
283+def print_stats_summary(shard: int, stats: Dict, shard_type: str = "regular") -> None:
284+ """Print statistics summary to stdout."""
285+ prefix = get_shard_type_prefix(shard_type)
286+ print(f"\n{'=' * 60}")
287+ print(f"Test Results for Shard {prefix}-{shard}")
288+ print(f"{'=' * 60}")
289+ print(f"Total: {stats['total']}")
290+ print(f"Passed: {stats['passed']}")
291+ print(f"Failed: {stats['failed']}")
292+ print(f"Skipped: {stats['skipped']}")
293+ print(f"Errors: {stats['errors']}")
294+ print(f"Duration: {stats['duration']:.2f}s")
295+ if stats.get("missing_files_count"):
296+ print(f"Missing files: {stats['missing_files_count']}")
297+ if stats.get("crashed"):
298+ print(f"Crash signal: {stats.get('crash_signal', 'unknown')}")
299+ print(f"{'=' * 60}")
300+ 
301+ 
302+if __name__ == "__main__":
303+ # Module only, no CLI functionality
304+ pass
A.github/scripts/run_npu_test_shard.py+1396-0
@@ -0,0 +1,1396 @@
1+#!/usr/bin/env python3
2+"""
3+Run PyTorch NPU tests via per-case isolation pytest execution.
4+ 
5+This script executes pre-collected test cases or specified test files
6+with per-case subprocess isolation for crash safety.
7+ 
8+Execution modes:
9+ - Pre-collected cases (--cases-json): Execute cases from JSON file
10+ - Custom test files (--test-files): Execute specified test files
11+ 
12+Each case runs in its own pytest subprocess for isolation:
13+ - NPU kernel crashes won't cascade to other cases
14+ - Results recorded in cases.json file
15+ 
16+Test types:
17+ - distributed: Serial execution (one case at a time)
18+ - regular: Concurrent execution (multiple workers)
19+ 
20+Usage:
21+ # Pre-collected cases mode (primary usage):
22+ python run_npu_test_shard.py \
23+ --cases-json distributed_cases_shard_1.json \
24+ --test-dir /path/to/pytorch/test \
25+ --disabled-testcases /path/to/disabled_testcases.json \
26+ --report-dir test-reports \
27+ --timeout 1200 \
28+ --max-workers 64 \
29+ --verbose
30+ 
31+ # Custom test files mode:
32+ python run_npu_test_shard.py \
33+ --test-files test_meta.py,test_nn.py \
34+ --test-dir /path/to/pytorch/test \
35+ --disabled-testcases /path/to/disabled_testcases.json \
36+ --report-dir test-reports \
37+ --timeout 1200 \
38+ --max-workers 4 \
39+ --verbose
40+ 
41+Note: Shard discovery mode (--shard/--num-shards/--test-type) has been removed.
42+ Use collect_all_cases.py for case discovery and sharding.
43+"""
44+ 
45+import argparse
46+import dataclasses
47+import importlib.util
48+import json
49+import os
50+import subprocess
51+import sys
52+import threading
53+import xml.etree.ElementTree as ET
54+from concurrent.futures import ThreadPoolExecutor, as_completed
55+from datetime import datetime
56+from pathlib import Path
57+from queue import Queue, Empty
58+from time import monotonic
59+from typing import Dict, List, Optional, Tuple
60+ 
61+import collect_all_cases
62+ 
63+ 
64+# ==============================================================================
65+# NPU Device Detection
66+# ==============================================================================
67+ 
68+ 
69+def get_npu_device_count() -> int:
70+ """
71+ Detect NPU device count via libascend_hal.so.
72+ 
73+ Returns the number of available NPU devices. Falls back to 8 if detection fails.
74+ """
75+ try:
76+ from ctypes import byref, c_int, CDLL
77+ ascend_hal = CDLL("libascend_hal.so")
78+ dev_count = c_int(-1)
79+ rc = ascend_hal.drvGetDevNum(byref(dev_count))
80+ if rc == 0 and dev_count.value > 0:
81+ return dev_count.value
82+ except OSError:
83+ print("Warning: Failed to load libascend_hal.so, using default 8 NPU devices")
84+ except AttributeError:
85+ print("Warning: drvGetDevNum not found in libascend_hal.so, using default 8 NPU devices")
86+ return 8 # Default: typical node has 8 NPU cards
87+ 
88+ 
89+# ==============================================================================
90+# Import Result Parser Module
91+# ==============================================================================
92+ 
93+ 
94+def load_parse_test_results_module(script_dir: Path):
95+ """Load parse_test_results module dynamically."""
96+ module_path = script_dir / "parse_test_results.py"
97+ if not module_path.exists():
98+ raise FileNotFoundError(f"parse_test_results.py not found at {module_path}")
99+ 
100+ spec = importlib.util.spec_from_file_location("parse_test_results", str(module_path))
101+ module = importlib.util.module_from_spec(spec)
102+ spec.loader.exec_module(module)
103+ return module
104+ 
105+ 
106+# ==============================================================================
107+# Data Classes
108+# ==============================================================================
109+ 
110+ 
111+@dataclasses.dataclass
112+class CaseExecutionTask:
113+ """Task for concurrent case execution."""
114+ case_idx: int
115+ nodeid: str
116+ test_file: str
117+ 
118+ 
119+@dataclasses.dataclass
120+class ConcurrentExecutionConfig:
121+ """Configuration for concurrent execution."""
122+ max_workers: int = 4
123+ per_case_timeout: int = 1200
124+ verbose: bool = False
125+ 
126+ 
127+# ==============================================================================
128+# Case Log Saving Functions
129+# ==============================================================================
130+ 
131+ 
132+def sanitize_nodeid_for_filename(nodeid: str) -> str:
133+ """
134+ Convert nodeid to a safe filename.
135+ 
136+ Replaces special characters with underscores and truncates if too long.
137+ Invalid characters for NTFS/filesystems: " : < > | * ? \r \n
138+ """
139+ # Replace special characters (including NTFS-invalid chars)
140+ safe_name = nodeid.replace("::", "_").replace("/", "_").replace("\\", "_")
141+ safe_name = safe_name.replace("(", "_").replace(")", "_").replace("[", "_").replace("]", "_")
142+ # NTFS-invalid characters that GitHub Actions artifact upload rejects
143+ safe_name = safe_name.replace("<", "_lt_").replace(">", "_gt_")
144+ safe_name = safe_name.replace('"', "_quot_").replace("|", "_pipe_")
145+ safe_name = safe_name.replace("*", "_star_").replace("?", "_q_")
146+ safe_name = safe_name.replace(":", "_colon_")
147+ safe_name = safe_name.replace(" ", "_")
148+ safe_name = safe_name.replace(".", "_")
149+ 
150+ # Remove leading underscores and collapse multiple underscores
151+ while safe_name.startswith("_"):
152+ safe_name = safe_name[1:]
153+ while "__" in safe_name:
154+ safe_name = safe_name.replace("__", "_")
155+ 
156+ # Truncate if too long (max 200 chars for filesystem compatibility)
157+ if len(safe_name) > 200:
158+ safe_name = safe_name[:200]
159+ 
160+ return safe_name or "unknown_case"
161+ 
162+ 
163+def save_case_log(
164+ report_dir: Path,
165+ shard: int,
166+ shard_type: str,
167+ nodeid: str,
168+ case_idx: int,
169+ status: str,
170+ stdout: str,
171+ stderr: str,
172+ duration: float,
173+ returncode: int,
174+ command: str,
175+ npu_device_id: Optional[int] = None,
176+) -> Path:
177+ """
178+ Save complete execution log for all test cases.
179+ 
180+ Creates a dedicated log file containing:
181+ - Case metadata (nodeid, status, duration, returncode)
182+ - Full stdout and stderr output
183+ - Execution command
184+ 
185+ Returns:
186+ Path to the saved log file
187+ """
188+ # Create cases log directory
189+ cases_logs_dir = report_dir / "cases_logs"
190+ cases_logs_dir.mkdir(parents=True, exist_ok=True)
191+ 
192+ # Generate safe filename
193+ safe_name = sanitize_nodeid_for_filename(nodeid)
194+ prefix = "dist" if shard_type == "distributed" else "reg"
195+ log_filename = f"{prefix}-{shard}_{case_idx}_{safe_name}.log"
196+ log_path = cases_logs_dir / log_filename
197+ 
198+ # Write log content
199+ content_lines = [
200+ "=" * 80,
201+ f"CASE LOG",
202+ "=" * 80,
203+ f"Shard: {prefix}-{shard}",
204+ f"Case Index: {case_idx}",
205+ f"Nodeid: {nodeid}",
206+ f"Status: {status}",
207+ f"Duration: {duration:.2f}s",
208+ f"Return Code: {returncode}",
209+ f"Command: {command}",
210+ ]
211+ if npu_device_id is not None:
212+ content_lines.append(f"NPU Device: {npu_device_id}")
213+ content_lines.extend([
214+ "=" * 80,
215+ "",
216+ "STDOUT:",
217+ "-" * 80,
218+ stdout or "(empty)",
219+ "",
220+ "STDERR:",
221+ "-" * 80,
222+ stderr or "(empty)",
223+ "",
224+ "=" * 80,
225+ ])
226+ 
227+ log_path.write_text("\n".join(content_lines), encoding="utf-8")
228+ return log_path
229+ 
230+ 
231+class ConcurrentResultAggregator:
232+ """Thread-safe result aggregator for concurrent execution."""
233+ 
234+ def __init__(self):
235+ self._lock = threading.Lock()
236+ self._cases_list: List[Dict] = []
237+ self._worst_returncode: int = 0
238+ self._passed_count: int = 0
239+ self._failed_count: int = 0
240+ self._error_count: int = 0
241+ self._skipped_count: int = 0
242+ self._timeout_count: int = 0
243+ self._total_cases: int = 0
244+ 
245+ def add_case_result(self, case_result: Dict) -> None:
246+ """Thread-safe add case result."""
247+ with self._lock:
248+ self._cases_list.append(case_result)
249+ self._total_cases += 1
250+ 
251+ status = case_result.get("status", "error")
252+ if status == "passed":
253+ self._passed_count += 1
254+ elif status == "failed":
255+ self._failed_count += 1
256+ elif status == "skipped":
257+ self._skipped_count += 1
258+ elif status == "timeout":
259+ self._timeout_count += 1
260+ else:
261+ # error
262+ self._error_count += 1
263+ 
264+ # Track worst returncode (largest non-zero value)
265+ # Negative returncodes (signal crashes) have larger absolute values
266+ rc = case_result.get("returncode", 1)
267+ if rc != 0:
268+ # Keep the "worst" returncode: max of current worst and new rc
269+ # This captures both high positive codes and severe crashes (negative)
270+ self._worst_returncode = max(self._worst_returncode, rc)
271+ 
272+ def get_sorted_cases(self) -> List[Dict]:
273+ """Get cases sorted by case_idx."""
274+ with self._lock:
275+ return sorted(self._cases_list, key=lambda x: x.get("case_idx", 0))
276+ 
277+ def get_summary(self) -> Dict:
278+ """Get execution summary."""
279+ with self._lock:
280+ return {
281+ "total_cases": self._total_cases,
282+ "passed_count": self._passed_count,
283+ "failed_count": self._failed_count,
284+ "error_count": self._error_count,
285+ "skipped_count": self._skipped_count,
286+ "timeout_count": self._timeout_count,
287+ "worst_returncode": self._worst_returncode,
288+ }
289+ 
290+ 
291+class ProgressTracker:
292+ """Thread-safe progress tracker with real-time output."""
293+ 
294+ def __init__(self, total_tasks: int):
295+ self._total_tasks = total_tasks
296+ self._completed_tasks = 0
297+ self._lock = threading.Lock()
298+ self._start_time = monotonic()
299+ 
300+ def mark_completed(self, nodeid: str, status: str, duration: float) -> None:
301+ """Mark task completed and print progress."""
302+ with self._lock:
303+ self._completed_tasks += 1
304+ elapsed = monotonic() - self._start_time
305+ progress_pct = (self._completed_tasks / self._total_tasks) * 100
306+ 
307+ # Status indicator
308+ status_icon = {
309+ "passed": "[PASS]",
310+ "failed": "[FAIL]",
311+ "error": "[ERR]",
312+ "timeout": "[TIME]",
313+ "skipped": "[SKIP]",
314+ }.get(status, "[?]")
315+ 
316+ # Truncate nodeid for display
317+ display_nodeid = nodeid[:60] + "..." if len(nodeid) > 60 else nodeid
318+ 
319+ print(f"[{self._completed_tasks}/{self._total_tasks}] {progress_pct:.1f}% "
320+ f"{status_icon} {display_nodeid} ({duration:.1f}s) "
321+ f"[elapsed: {elapsed:.0f}s]", flush=True)
322+ 
323+ 
324+# ==============================================================================
325+# JUnit XML Parsing for Accurate Status Detection
326+# ==============================================================================
327+ 
328+ 
329+def parse_junit_xml_status(xml_file: Path) -> Dict:
330+ """
331+ 解析 JUnit XML 报告，获取测试状态。
332+ 
333+ Args:
334+ xml_file: JUnit XML 文件路径
335+ 
336+ Returns:
337+ Dict: {"status": "passed" | "skipped" | "failed" | "error" | "no_xml", "message": str}
338+ """
339+ if not xml_file.exists():
340+ return {"status": "no_xml", "message": "XML file not generated"}
341+ 
342+ try:
343+ tree = ET.parse(str(xml_file))
344+ root = tree.getroot()
345+ 
346+ for testcase in root.iter("testcase"):
347+ result = {"status": "passed", "message": ""}
348+ 
349+ # Check <skipped>
350+ skipped_elem = testcase.find("skipped")
351+ if skipped_elem is not None:
352+ result["status"] = "skipped"
353+ result["message"] = skipped_elem.get("message", "")
354+ return result
355+ 
356+ # Check <failure>
357+ failure_elem = testcase.find("failure")
358+ if failure_elem is not None:
359+ result["status"] = "failed"
360+ result["message"] = failure_elem.get("message", "")
361+ return result
362+ 
363+ # Check <error>
364+ error_elem = testcase.find("error")
365+ if error_elem is not None:
366+ result["status"] = "error"
367+ result["message"] = error_elem.get("message", "")
368+ return result
369+ 
370+ # No failure/error/skipped = passed
371+ return result
372+ 
373+ return {"status": "error", "message": "No testcase in XML"}
374+ 
375+ except Exception:
376+ return {"status": "no_xml", "message": "XML parse failed"}
377+ 
378+ 
379+# ==============================================================================
380+# Utility Functions
381+# ==============================================================================
382+ 
383+ 
384+def strip_test_prefix_and_suffix(test_path: str) -> str:
385+ """Remove 'test/' prefix and '.py' suffix from path."""
386+ path = test_path
387+ if path.startswith("test/"):
388+ path = path[5:]
389+ if path.endswith(".py"):
390+ path = path[:-3]
391+ return path
392+ 
393+ 
394+def load_installed_torch_root() -> str:
395+ """Get installed torch root directory."""
396+ try:
397+ import torch
398+ return str(Path(torch.__file__).resolve().parent.parent)
399+ except Exception as exc:
400+ print(f"Warning: Failed to import torch: {exc}")
401+ return ""
402+ 
403+ 
404+# ==============================================================================
405+# Concurrent Case Execution
406+# ==============================================================================
407+ 
408+ 
409+def run_single_case_concurrent(
410+ task: CaseExecutionTask,
411+ test_dir: Path,
412+ merged_env: Dict[str, str],
413+ config: ConcurrentExecutionConfig,
414+ result_aggregator: ConcurrentResultAggregator,
415+ progress_tracker: ProgressTracker,
416+ log_queue: Queue,
417+ report_dir: Path,
418+ shard: int,
419+ shard_type: str,
420+ npu_device_id: Optional[int] = None,
421+) -> Dict:
422+ """
423+ Execute a single test case in subprocess (for concurrent execution).
424+ 
425+ This function runs in ThreadPoolExecutor threads. Each call spawns
426+ an independent subprocess for the test case. Core dumps and crashes
427+ in the subprocess do NOT affect the main Python process or other
428+ concurrent tasks.
429+ 
430+ CRITICAL: This function must catch ALL exceptions and return a result
431+ dict. It should NEVER raise exceptions to ThreadPoolExecutor level.
432+ 
433+ Args:
434+ task: Case execution task with nodeid and metadata
435+ test_dir: PyTorch test directory
436+ merged_env: Environment variables
437+ config: Execution configuration
438+ result_aggregator: Thread-safe result collector
439+ progress_tracker: Thread-safe progress tracker
440+ log_queue: Queue for log messages
441+ 
442+ Returns:
443+ Dict with case result (never raises exception)
444+ """
445+ start_time = monotonic()
446+ original_nodeid = task.nodeid
447+ case_nodeid = task.nodeid
448+ 
449+ # Strip test/ prefix for pytest execution
450+ if case_nodeid.startswith("test/"):
451+ case_nodeid = case_nodeid[5:]
452+ 
453+ # Generate XML file path with descriptive name
454+ prefix = "dist" if shard_type == "distributed" else "reg"
455+ safe_case_name = sanitize_nodeid_for_filename(original_nodeid)
456+ xml_filename = f"{prefix}-{shard}_{task.case_idx}_{safe_case_name}.xml"
457+ xml_file = report_dir / "junit_xmls" / xml_filename
458+ 
459+ command = [
460+ sys.executable,
461+ "-m",
462+ "pytest",
463+ "--color=no",
464+ "-ra",
465+ "--tb=short",
466+ case_nodeid,
467+ f"--junitxml={xml_file}",
468+ ]
469+ 
470+ if config.per_case_timeout > 0:
471+ command.append(f"--timeout={config.per_case_timeout}")
472+ 
473+ if config.verbose:
474+ command.append("-vv")
475+ else:
476+ command.append("-v")
477+ 
478+ command_str = " ".join(command)
479+ 
480+ # Build per-case environment with test file directory in PYTHONPATH
481+ # This enables imports of sibling modules (e.g., 'from model_registry import MLPModule')
482+ case_env = merged_env.copy()
483+ test_file = task.test_file
484+ if test_file.startswith("test/"):
485+ test_file_rel = test_file[5:]
486+ else:
487+ test_file_rel = test_file
488+ 
489+ test_file_path = Path(test_file_rel)
490+ test_file_dir = test_dir / test_file_path.parent
491+ 
492+ existing_pythonpath = case_env.get("PYTHONPATH", "")
493+ case_env["PYTHONPATH"] = str(test_file_dir) + (":" + existing_pythonpath if existing_pythonpath else "")
494+ 
495+ # Set NPU device for regular tests (round-robin allocation)
496+ # distributed tests do not set ASCEND_RT_VISIBLE_DEVICES to allow using all devices
497+ if npu_device_id is not None:
498+ case_env["ASCEND_RT_VISIBLE_DEVICES"] = str(npu_device_id)
499+ 
500+ # Print start log to stdout (before execution)
501+ # Truncate nodeid for display
502+ display_nodeid = original_nodeid[:70] + "..." if len(original_nodeid) > 70 else original_nodeid
503+ print(f"[{task.case_idx}] Starting: {display_nodeid}", flush=True)
504+ 
505+ # Log start
506+ log_queue.put({
507+ "type": "case_start",
508+ "case_idx": task.case_idx,
509+ "nodeid": original_nodeid,
510+ "file": task.test_file,
511+ "command": command_str,
512+ })
513+ 
514+ # Execute subprocess - CRITICAL: catch ALL exceptions
515+ try:
516+ result = subprocess.run(
517+ command,
518+ cwd=str(test_dir),
519+ env=case_env, # Use per-case environment with test file directory in PYTHONPATH
520+ capture_output=True,
521+ text=True,
522+ encoding="utf-8",
523+ errors="replace",
524+ timeout=config.per_case_timeout + 30, # Extra 30s buffer for pytest startup overhead
525+ )
526+ 
527+ duration = monotonic() - start_time
528+ returncode = result.returncode
529+ 
530+ # Parse JUnit XML for status
531+ # - Has XML: use XML status
532+ # - No XML: error
533+ xml_result = parse_junit_xml_status(xml_file)
534+ xml_status = xml_result.get("status")
535+ 
536+ if xml_status == "no_xml":
537+ # No XML → error
538+ status = "error"
539+ message = xml_result.get("message")
540+ else:
541+ # Has XML → use XML status
542+ status = xml_status
543+ message = xml_result.get("message", "")
544+ 
545+ # Save logs for all cases
546+ save_case_log(
547+ report_dir=report_dir,
548+ shard=shard,
549+ shard_type=shard_type,
550+ nodeid=original_nodeid,
551+ case_idx=task.case_idx,
552+ status=status,
553+ stdout=result.stdout,
554+ stderr=result.stderr,
555+ duration=duration,
556+ returncode=returncode,
557+ command=command_str,
558+ npu_device_id=npu_device_id,
559+ )
560+ 
561+ case_result = {
562+ "nodeid": original_nodeid,
563+ "status": status,
564+ "duration": duration,
565+ "returncode": returncode,
566+ "message": message,
567+ "command": command_str,
568+ "file": task.test_file,
569+ "case_idx": task.case_idx,
570+ }
571+ 
572+ except subprocess.TimeoutExpired:
573+ # Timeout → no XML, status = timeout
574+ duration = monotonic() - start_time
575+ status = "timeout"
576+ case_result = {
577+ "nodeid": original_nodeid,
578+ "status": status,
579+ "duration": duration,
580+ "returncode": -1,
581+ "message": f"Timeout after {config.per_case_timeout}s",
582+ "command": command_str,
583+ "file": task.test_file,
584+ "case_idx": task.case_idx,
585+ }
586+ 
587+ # Save log for timeout
588+ save_case_log(
589+ report_dir=report_dir,
590+ shard=shard,
591+ shard_type=shard_type,
592+ nodeid=original_nodeid,
593+ case_idx=task.case_idx,
594+ status=status,
595+ stdout="(process timed out, no output captured)",
596+ stderr="(process timed out, no output captured)",
597+ duration=duration,
598+ returncode=-1,
599+ command=command_str,
600+ npu_device_id=npu_device_id,
601+ )
602+ 
603+ except Exception as e:
604+ # Any other exception - return result, don't raise
605+ duration = monotonic() - start_time
606+ case_result = {
607+ "nodeid": original_nodeid,
608+ "status": "error",
609+ "duration": duration,
610+ "returncode": 1,
611+ "message": f"Unexpected error: {str(e)[:200]}",
612+ "command": command_str,
613+ "file": task.test_file,
614+ "case_idx": task.case_idx,
615+ }
616+ 
617+ # Save error case log
618+ save_case_log(
619+ report_dir=report_dir,
620+ shard=shard,
621+ shard_type=shard_type,
622+ nodeid=original_nodeid,
623+ case_idx=task.case_idx,
624+ status="error",
625+ stdout="(exception occurred before execution)",
626+ stderr=str(e),
627+ duration=duration,
628+ returncode=1,
629+ command=command_str,
630+ npu_device_id=npu_device_id,
631+ )
632+ 
633+ # Log finish
634+ log_queue.put({
635+ "type": "case_finish",
636+ "case_idx": task.case_idx,
637+ "nodeid": original_nodeid,
638+ "status": case_result["status"],
639+ "duration": case_result["duration"],
640+ "message": case_result["message"][:200] if case_result["message"] else "",
641+ })
642+ 
643+ # Update aggregator (thread-safe)
644+ result_aggregator.add_case_result(case_result)
645+ 
646+ # Update progress (thread-safe)
647+ progress_tracker.mark_completed(original_nodeid, case_result["status"], duration)
648+ 
649+ return case_result
650+ 
651+ 
652+def log_writer_thread(log_queue: Queue, log_file: Path, stop_event: threading.Event) -> None:
653+ """
654+ Background thread for writing logs.
655+ 
656+ Ensures thread-safe log file writes while concurrent tasks run.
657+ """
658+ with log_file.open("w", encoding="utf-8") as log_handle:
659+ while not stop_event.is_set() or not log_queue.empty():
660+ try:
661+ log_entry = log_queue.get(timeout=0.5)
662+ except Empty:
663+ continue
664+ 
665+ if log_entry.get("type") == "header":
666+ log_handle.write(log_entry.get("content", ""))
667+ log_handle.flush()
668+ elif log_entry.get("type") == "case_start":
669+ log_handle.write(f"\n[{log_entry['case_idx']}] {log_entry['nodeid']}\n")
670+ log_handle.write(f" File: {log_entry.get('file', '')}\n")
671+ log_handle.write(f" Command: {log_entry.get('command', '')}\n")
672+ log_handle.flush()
673+ elif log_entry.get("type") == "case_finish":
674+ status_str = log_entry.get("status", "")
675+ duration_str = f"{log_entry.get('duration', 0):.2f}s"
676+ log_handle.write(f" Status: {status_str}, Duration: {duration_str}\n")
677+ if log_entry.get("message"):
678+ log_handle.write(f" Message: {log_entry['message']}\n")
679+ log_handle.flush()
680+ elif log_entry.get("type") == "summary":
681+ log_handle.write(log_entry.get("content", ""))
682+ log_handle.flush()
683+ 
684+ 
685+def run_tests_with_tasks_concurrent(
686+ tasks: List[CaseExecutionTask],
687+ shard: int,
688+ test_dir: Path,
689+ report_dir: Path,
690+ env_updates: Dict[str, str],
691+ timeout: int,
692+ verbose: bool,
693+ shard_type: str,
694+ max_workers: int,
695+ result_module,
696+ quick_test: int = None,
697+) -> Tuple[int, float, List[Dict]]:
698+ """
699+ Execute pre-collected test cases with concurrent per-case isolation.
700+ 
701+ This function takes CaseExecutionTask objects directly (pre-collected cases)
702+ and executes them concurrently without the file-level case collection phase.
703+ 
704+ Args:
705+ tasks: List of CaseExecutionTask objects (pre-collected cases)
706+ shard: Shard number
707+ test_dir: PyTorch test directory
708+ report_dir: Report output directory
709+ env_updates: Environment variable updates
710+ timeout: Per-case timeout in seconds
711+ verbose: Verbose output
712+ shard_type: "distributed" or "regular"
713+ max_workers: Maximum concurrent subprocesses
714+ result_module: parse_test_results module
715+ quick_test: Maximum number of cases to execute (None = all cases)
716+ 
717+ Returns:
718+ Tuple of (worst_returncode, duration, cases_list_sorted)
719+ """
720+ start = monotonic()
721+ log_file = result_module.get_shard_log_file(report_dir, shard, shard_type)
722+ 
723+ # Create junit_xmls directory for XML reports
724+ junit_xml_dir = report_dir / "junit_xmls"
725+ junit_xml_dir.mkdir(parents=True, exist_ok=True)
726+ 
727+ merged_env = os.environ.copy()
728+ merged_env.update(env_updates)
729+ 
730+ # Detect NPU device count and allocate devices
731+ # distributed tests do not set ASCEND_RT_VISIBLE_DEVICES to allow using all devices
732+ if shard_type == "distributed":
733+ num_npu_devices = None
734+ print("NPU device allocation: DISABLED (distributed test uses all devices)")
735+ else:
736+ num_npu_devices = get_npu_device_count()
737+ print(f"NPU device allocation: {num_npu_devices} devices detected (round-robin)")
738+ 
739+ config = ConcurrentExecutionConfig(
740+ max_workers=max_workers,
741+ per_case_timeout=timeout,
742+ verbose=verbose,
743+ )
744+ 
745+ # Thread-safe result aggregator
746+ result_aggregator = ConcurrentResultAggregator()
747+ 
748+ # Log queue and writer thread
749+ log_queue = Queue()
750+ stop_event = threading.Event()
751+ log_thread = threading.Thread(
752+ target=log_writer_thread,
753+ args=(log_queue, log_file, stop_event),
754+ daemon=True,
755+ )
756+ 
757+ # Write log header
758+ log_queue.put({
759+ "type": "header",
760+ "content": (
761+ "=" * 80 + "\n"
762+ f"Pre-collected cases concurrent execution ({shard_type} shard)\n"
763+ "=" * 80 + "\n"
764+ f"Total cases: {len(tasks)}\n"
765+ f"Max concurrent workers: {max_workers}\n"
766+ "Execution mode: concurrent subprocess, each case isolated\n"
767+ "=" * 80 + "\n\n"
768+ ),
769+ })
770+ 
771+ log_thread.start()
772+ 
773+ # Quick test: limit number of cases to execute
774+ if quick_test and len(tasks) > quick_test:
775+ tasks = tasks[:quick_test]
776+ print(f"\nQuick test mode: executing only {quick_test} cases", flush=True)
777+ 
778+ print(f"\n{'=' * 80}", flush=True)
779+ print(f"Pre-collected cases: {len(tasks)} cases", flush=True)
780+ print(f"Execution mode: {max_workers} workers concurrent, each case in subprocess", flush=True)
781+ print(f"{'=' * 80}\n", flush=True)
782+ 
783+ total_cases = len(tasks)
784+ print(f"Phase 1: Executing {total_cases} pre-collected cases...", flush=True)
785+ 
786+ # Phase 2: Concurrent execution via ThreadPoolExecutor
787+ progress_tracker = ProgressTracker(total_cases)
788+ 
789+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
790+ # Submit all tasks with device allocation (round-robin)
791+ future_to_task = {}
792+ for task in tasks:
793+ # Calculate device ID (round-robin allocation)
794+ if num_npu_devices is not None:
795+ device_id = task.case_idx % num_npu_devices
796+ else:
797+ device_id = None
798+ 
799+ future = executor.submit(
800+ run_single_case_concurrent,
801+ task,
802+ test_dir,
803+ merged_env,
804+ config,
805+ result_aggregator,
806+ progress_tracker,
807+ log_queue,
808+ report_dir,
809+ shard,
810+ shard_type,
811+ device_id,
812+ )
813+ future_to_task[future] = task
814+ 
815+ # Wait for completion (as_completed gives results as they finish)
816+ for future in as_completed(future_to_task):
817+ task = future_to_task[future]
818+ try:
819+ # Result already collected in aggregator
820+ _ = future.result()
821+ except Exception as e:
822+ # Should never happen (run_single_case_concurrent catches all)
823+ # But as safety, create error result
824+ case_result = {
825+ "nodeid": task.nodeid,
826+ "status": "error",
827+ "duration": 0.0,
828+ "returncode": 1,
829+ "message": f"Future error: {str(e)[:200]}",
830+ "file": task.test_file,
831+ "case_idx": task.case_idx,
832+ }
833+ result_aggregator.add_case_result(case_result)
834+ progress_tracker.mark_completed(task.nodeid, "error", 0.0)
835+ 
836+ # Stop log thread
837+ elapsed = monotonic() - start
838+ summary = result_aggregator.get_summary()
839+ 
840+ log_queue.put({
841+ "type": "summary",
842+ "content": (
843+ f"\n{'=' * 80}\n"
844+ f"Summary: {summary['total_cases']} cases executed\n"
845+ f" Passed: {summary['passed_count']}\n"
846+ f" Failed: {summary['failed_count']}\n"
847+ f" Errors: {summary['error_count']}\n"
848+ f" Timeout: {summary['timeout_count']}\n"
849+ f" Skipped: {summary['skipped_count']}\n"
850+ f" Duration: {elapsed:.2f}s\n"
851+ f" Concurrent workers: {max_workers}\n"
852+ f"{'=' * 80}\n"
853+ ),
854+ })
855+ 
856+ stop_event.set()
857+ log_thread.join(timeout=5)
858+ 
859+ # Print final summary
860+ print(f"\n{'=' * 80}", flush=True)
861+ print(f"Summary: {summary['total_cases']} cases executed", flush=True)
862+ print(f" Passed: {summary['passed_count']}", flush=True)
863+ print(f" Failed: {summary['failed_count']}", flush=True)
864+ print(f" Errors: {summary['error_count']}", flush=True)
865+ print(f" Timeout: {summary['timeout_count']}", flush=True)
866+ print(f" Skipped: {summary['skipped_count']}", flush=True)
867+ print(f" Duration: {elapsed:.2f}s", flush=True)
868+ print(f"{'=' * 80}", flush=True)
869+ 
870+ return summary["worst_returncode"], elapsed, result_aggregator.get_sorted_cases()
871+ 
872+ 
873+def build_execution_env(
874+ test_dir: Path,
875+ script_dir: Path,
876+ disabled_testcases_file: str,
877+ shard: int,
878+ shard_type: str,
879+) -> Dict[str, str]:
880+ """Build environment variables for test execution."""
881+ repo_root = test_dir.parent
882+ pythonpath_parts = [str(script_dir)]
883+ 
884+ torch_path = load_installed_torch_root()
885+ if torch_path:
886+ pythonpath_parts.append(torch_path)
887+ 
888+ pythonpath_parts.extend([str(repo_root), str(test_dir)])
889+ 
890+ existing_pythonpath = os.environ.get("PYTHONPATH", "")
891+ if existing_pythonpath:
892+ pythonpath_parts.append(existing_pythonpath)
893+ 
894+ updates = {
895+ "PYTHONPATH": os.pathsep.join(pythonpath_parts),
896+ "PYTORCH_TEST_NPU": "1",
897+ "TORCH_DEVICE_BACKEND_AUTOLOAD": "1",
898+ "NO_TD": "1",
899+ "PYTHONUNBUFFERED": "1",
900+ # Note: Do NOT set CI=true here, as some test files have conditional
901+ # test generation logic like:
902+ # if not (IS_CI and torch.cuda.is_available()):
903+ # globals().update(generate_tests(...))
904+ # Setting CI=true would prevent test case generation in those files.
905+ }
906+ 
907+ # Use PyTorch's built-in DISABLED_TESTS_FILE mechanism for skipping test cases
908+ if disabled_testcases_file:
909+ # The disabled_testcases.json format is similar to .pytorch-disabled-tests.json
910+ # Set DISABLED_TESTS_FILE to use PyTorch's built-in skip mechanism
911+ updates["DISABLED_TESTS_FILE"] = os.path.abspath(disabled_testcases_file)
912+ 
913+ return updates
914+ 
915+ 
916+def save_results_and_summary(
917+ result_module,
918+ report_dir: Path,
919+ shard: int,
920+ shard_type: str,
921+ cases_list: List[Dict],
922+ duration: float,
923+ returncode: int,
924+ info: Dict,
925+ execution_mode: Optional[str] = None,
926+ concurrent_workers: Optional[int] = None,
927+ has_distributed_files: Optional[bool] = None,
928+) -> None:
929+ """
930+ Save results and print summary.
931+ 
932+ This function handles the common result processing logic:
933+ - Calculate statistics (passed, failed, errors, etc.)
934+ - Build cases_data and stats dicts
935+ - Save cases.json, info, stats files
936+ - Print summary
937+ """
938+ # Calculate statistics
939+ passed_count = sum(1 for c in cases_list if c["status"] == "passed")
940+ failed_count = sum(1 for c in cases_list if c["status"] == "failed")
941+ error_count = sum(1 for c in cases_list if c["status"] == "error")
942+ timeout_count = sum(1 for c in cases_list if c["status"] == "timeout")
943+ skipped_count = sum(1 for c in cases_list if c["status"] == "skipped")
944+ 
945+ # Build cases.json data
946+ cases_data = {
947+ "shard": shard,
948+ "shard_type": shard_type,
949+ "execution_mode": execution_mode or info.get("execution_mode", "unknown"),
950+ "concurrent_workers": concurrent_workers or info.get("concurrent_workers", 1),
951+ "total_cases": len(cases_list),
952+ "passed": passed_count,
953+ "failed": failed_count,
954+ "errors": error_count,
955+ "timeout": timeout_count,
956+ "skipped": skipped_count,
957+ "duration": duration,
958+ "cases": cases_list,
959+ }
960+ if has_distributed_files is not None:
961+ cases_data["has_distributed_files"] = has_distributed_files
962+ 
963+ # Save cases.json
964+ result_module.save_cases_file(str(report_dir), shard, cases_data, shard_type)
965+ 
966+ # Save info file
967+ info["returncode"] = returncode
968+ info["duration"] = duration
969+ result_module.save_info_file(str(report_dir), shard, info, shard_type)
970+ 
971+ # Build and save stats
972+ stats = {
973+ "total": len(cases_list),
974+ "passed": passed_count,
975+ "failed": failed_count,
976+ "skipped": skipped_count,
977+ "errors": error_count,
978+ "timeout": timeout_count,
979+ "duration": duration,
980+ "returncode": returncode,
981+ "per_case_isolation": True,
982+ }
983+ if execution_mode:
984+ stats["execution_mode"] = execution_mode
985+ if concurrent_workers:
986+ stats["concurrent_workers"] = concurrent_workers
987+ if has_distributed_files is not None:
988+ stats["has_distributed_files"] = has_distributed_files
989+ 
990+ result_module.save_stats_file(str(report_dir), shard, stats, shard_type)
991+ 
992+ # Print summary
993+ result_module.print_stats_summary(shard, stats, shard_type)
994+ 
995+ 
996+def clean_existing_junit_xml(report_dir: Path) -> None:
997+ """Clean existing JUnit XML files."""
998+ if not report_dir.exists():
999+ return
1000+ for xml_file in report_dir.rglob("*.xml"):
1001+ xml_file.unlink(missing_ok=True)
1002+ 
1003+ 
1004+# ==============================================================================
1005+# Test Files Input Parser
1006+# ==============================================================================
1007+ 
1008+ 
1009+def has_distributed_test_files(test_files: List[str]) -> bool:
1010+ """
1011+ Check if any test file is a distributed test.
1012+ 
1013+ Distributed tests are identified by path starting with "test/distributed/".
1014+ 
1015+ Args:
1016+ test_files: List of test file paths (e.g., ["test/test_meta.py", "test/distributed/test_ddp.py"])
1017+ 
1018+ Returns:
1019+ True if any file is a distributed test, False otherwise
1020+ """
1021+ for f in test_files:
1022+ if f.startswith("test/distributed/"):
1023+ return True
1024+ return False
1025+ 
1026+ 
1027+def parse_test_files_input(test_files_str: str, test_dir: Path) -> List[str]:
1028+ """
1029+ Parse comma-separated test file input and return standardized test file paths.
1030+ 
1031+ Args:
1032+ test_files_str: Comma-separated test file paths (e.g., "test_meta.py,test_nn.py")
1033+ test_dir: Path to PyTorch test directory
1034+ 
1035+ Returns:
1036+ List of standardized test file paths (e.g., ["test/test_meta.py", "test/test_nn.py"])
1037+ 
1038+ Raises:
1039+ FileNotFoundError: If any specified test file does not exist
1040+ """
1041+ files = [f.strip() for f in test_files_str.split(",") if f.strip()]
1042+ result = []
1043+ 
1044+ for f in files:
1045+ # Normalize path format: ensure starts with "test/"
1046+ if not f.startswith("test/"):
1047+ f = "test/" + f
1048+ 
1049+ # Remove leading "test/" prefix if it's duplicated
1050+ if f.startswith("test/test/"):
1051+ f = f[5:]
1052+ 
1053+ # Verify file exists
1054+ full_path = test_dir.parent / f
1055+ if not full_path.exists():
1056+ # Try with .py extension if not provided
1057+ if not f.endswith(".py"):
1058+ f_with_ext = f + ".py"
1059+ full_path_with_ext = test_dir.parent / f_with_ext
1060+ if full_path_with_ext.exists():
1061+ f = f_with_ext
1062+ full_path = full_path_with_ext
1063+ else:
1064+ raise FileNotFoundError(f"Test file not found: {f} or {f_with_ext}")
1065+ else:
1066+ raise FileNotFoundError(f"Test file not found: {f}")
1067+ 
1068+ result.append(f)
1069+ 
1070+ return result
1071+ 
1072+ 
1073+# ==============================================================================
1074+# CLI
1075+# ==============================================================================
1076+ 
1077+ 
1078+def parse_args():
1079+ """Parse command line arguments."""
1080+ parser = argparse.ArgumentParser(
1081+ description="Run PyTorch NPU tests via per-case isolation pytest execution"
1082+ )
1083+ parser.add_argument("--test-files", type=str, help="Comma-separated test file paths to run directly (e.g., 'test_meta.py,test_nn.py')")
1084+ parser.add_argument("--cases-json", type=str, help="Path to pre-collected cases JSON file")
1085+ parser.add_argument("--test-dir", type=str, required=True, help="Path to PyTorch test directory")
1086+ parser.add_argument("--disabled-testcases", type=str, help="Path to disabled_testcases.json")
1087+ parser.add_argument("--report-dir", type=str, default="test-reports", help="Directory for reports")
1088+ parser.add_argument("--timeout", type=int, default=1200, help="Per-case timeout in seconds (default: 1200 = 20 minutes)")
1089+ parser.add_argument(
1090+ "--max-workers",
1091+ type=int,
1092+ default=4,
1093+ help="Maximum concurrent workers for regular tests (default: 4). Each worker runs one pytest subprocess.",
1094+ )
1095+ parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")
1096+ parser.add_argument("--quick-test", type=int, default=None, help="Quick test mode: execute only N cases for fast verification (default: None, run all cases)")
1097+ args = parser.parse_args()
1098+ 
1099+ # Validate required arguments: must specify either --test-files or --cases-json
1100+ if not args.test_files and not args.cases_json:
1101+ parser.error("Either --test-files or --cases-json must be specified")
1102+ 
1103+ # Validate max_workers
1104+ if args.max_workers < 1:
1105+ parser.error("--max-workers must be at least 1")
1106+ if args.max_workers > 128:
1107+ print(f"WARNING: --max-workers={args.max_workers} is very high, may cause resource contention")
1108+ 
1109+ return args
1110+ 
1111+ 
1112+def main():
1113+ """Main entry point."""
1114+ args = parse_args()
1115+ 
1116+ # Resolve paths
1117+ test_dir = Path(args.test_dir).resolve()
1118+ if not test_dir.is_dir():
1119+ raise FileNotFoundError(f"Test directory not found: {test_dir}")
1120+ 
1121+ repo_root = test_dir.parent
1122+ script_dir = Path(__file__).resolve().parent
1123+ report_dir = Path(args.report_dir).resolve()
1124+ report_dir.mkdir(parents=True, exist_ok=True)
1125+ 
1126+ # Load modules
1127+ result_module = load_parse_test_results_module(script_dir)
1128+ 
1129+ timestamp = datetime.now().isoformat()
1130+ 
1131+ # ==========================================================================
1132+ # Mode: Direct execution of specified test files
1133+ # ==========================================================================
1134+ if args.test_files:
1135+ print("=" * 80)
1136+ print("Custom Test Files Execution Mode")
1137+ print("=" * 80)
1138+ 
1139+ # Parse test files input
1140+ planned_tests = parse_test_files_input(args.test_files, test_dir)
1141+ 
1142+ # Use fixed shard number for custom mode
1143+ shard = 1
1144+ num_shards = 1
1145+ shard_type = "custom"
1146+ 
1147+ # Detect distributed test files and determine execution mode
1148+ has_distributed = has_distributed_test_files(planned_tests)
1149+ if has_distributed:
1150+ effective_workers = 1
1151+ execution_mode = "serial"
1152+ print(f"WARNING: Distributed test files detected, forcing serial execution")
1153+ else:
1154+ effective_workers = args.max_workers
1155+ execution_mode = "concurrent"
1156+ 
1157+ print(f"Test files specified: {len(planned_tests)}")
1158+ print(f"Test directory: {test_dir}")
1159+ print(f"Execution mode: {execution_mode} ({effective_workers} workers, per-case subprocess isolation)")
1160+ if has_distributed:
1161+ distributed_files = [f for f in planned_tests if f.startswith("test/distributed/")]
1162+ print(f" Distributed files: {len(distributed_files)}")
1163+ for df in distributed_files:
1164+ print(f" - {strip_test_prefix_and_suffix(df)}")
1165+ if args.disabled_testcases:
1166+ disabled_count = result_module.load_disabled_testcases_count(args.disabled_testcases)
1167+ print(f"Disabled testcase entries: {disabled_count}")
1168+ print(f"\n{'=' * 80}\n")
1169+ 
1170+ for index, target in enumerate(planned_tests, 1):
1171+ display_name = strip_test_prefix_and_suffix(target)
1172+ is_dist = target.startswith("test/distributed/")
1173+ dist_marker = " [distributed]" if is_dist else ""
1174+ print(f" [{index:03d}] {display_name}{dist_marker}")
1175+ 
1176+ # Create info dict for custom mode
1177+ info = result_module.create_shard_info(shard, num_shards, timestamp)
1178+ info["selection_mode"] = "custom_files"
1179+ info["shard_type"] = shard_type
1180+ info["shard_files"] = len(planned_tests)
1181+ info["total_files"] = len(planned_tests)
1182+ info["selected_test_files"] = len(planned_tests)
1183+ info["has_distributed_files"] = has_distributed
1184+ info["execution_mode"] = execution_mode
1185+ if args.disabled_testcases:
1186+ info["disabled_count"] = result_module.load_disabled_testcases_count(args.disabled_testcases)
1187+ 
1188+ # Save test plan
1189+ result_module.save_test_plan_file(str(report_dir), shard, planned_tests, shard_type)
1190+ 
1191+ # Clean old files
1192+ clean_existing_junit_xml(report_dir)
1193+ result_module.get_shard_log_file(report_dir, shard, shard_type).unlink(missing_ok=True)
1194+ 
1195+ # Build execution env
1196+ env_updates = build_execution_env(
1197+ test_dir, script_dir, args.disabled_testcases, shard, shard_type
1198+ )
1199+ 
1200+ # Execute tests (custom mode: auto-detect distributed files for execution mode)
1201+ cases_list = []
1202+ if planned_tests:
1203+ # Phase 1: Collect all test cases using collect_all_cases module
1204+ print("\nPhase 1: Collecting test cases...")
1205+ error_log_dir = report_dir / "collection_errors"
1206+ collected_cases = collect_all_cases.collect_all_cases(
1207+ planned_tests,
1208+ test_dir,
1209+ error_log_dir,
1210+ parallel=16, # 16 parallel collectors balance speed vs resource usage
1211+ )
1212+ 
1213+ # Apply quick_test limit if specified
1214+ if args.quick_test and len(collected_cases) > args.quick_test:
1215+ collected_cases = collected_cases[:args.quick_test]
1216+ print(f" Quick test mode: using only {args.quick_test} cases")
1217+ 
1218+ total_cases = len(collected_cases)
1219+ print(f"\nPhase 2: Executing {total_cases} cases with {effective_workers} workers")
1220+ 
1221+ # Build CaseExecutionTask list
1222+ tasks = []
1223+ for i, case in enumerate(collected_cases, 1):
1224+ tasks.append(CaseExecutionTask(
1225+ case_idx=i,
1226+ nodeid=case["nodeid"],
1227+ test_file=case["file"],
1228+ ))
1229+ 
1230+ # Phase 2: Execute cases using run_tests_with_tasks_concurrent
1231+ # Use effective_workers (1 for distributed files, args.max_workers otherwise)
1232+ # Note: quick_test already applied above, pass None to avoid redundant check
1233+ returncode, duration, cases_list = run_tests_with_tasks_concurrent(
1234+ tasks,
1235+ shard,
1236+ test_dir,
1237+ report_dir,
1238+ env_updates,
1239+ args.timeout,
1240+ args.verbose,
1241+ shard_type,
1242+ effective_workers,
1243+ result_module,
1244+ None, # quick_test already applied above
1245+ )
1246+ info["per_case_isolation"] = True
1247+ info["concurrent_workers"] = effective_workers
1248+ info["returncode"] = returncode
1249+ info["duration"] = duration
1250+ else:
1251+ returncode = 0
1252+ duration = 0.0
1253+ 
1254+ # Save results and print summary
1255+ save_results_and_summary(
1256+ result_module=result_module,
1257+ report_dir=report_dir,
1258+ shard=shard,
1259+ shard_type=shard_type,
1260+ cases_list=cases_list,
1261+ duration=duration,
1262+ returncode=returncode,
1263+ info=info,
1264+ execution_mode=execution_mode,
1265+ concurrent_workers=effective_workers,
1266+ has_distributed_files=has_distributed,
1267+ )
1268+ 
1269+ # Exit with 0 to allow step to succeed and report generation to proceed
1270+ # The actual test results are recorded in cases.json
1271+ sys.exit(0)
1272+ 
1273+ # ==========================================================================
1274+ # Mode: Pre-collected cases JSON execution
1275+ # ==========================================================================
1276+ if args.cases_json:
1277+ print("=" * 80)
1278+ print("Pre-collected Cases Execution Mode")
1279+ print("=" * 80)
1280+ 
1281+ cases_file = Path(args.cases_json).resolve()
1282+ if not cases_file.exists():
1283+ raise FileNotFoundError(f"Cases JSON file not found: {cases_file}")
1284+ 
1285+ cases_data = json.loads(cases_file.read_text(encoding="utf-8"))
1286+ 
1287+ shard = cases_data["shard"]
1288+ num_shards = cases_data["num_shards"]
1289+ shard_type = cases_data.get("test_type", "regular")
1290+ planned_cases = cases_data["cases"]
1291+ total_cases = len(planned_cases)
1292+ 
1293+ print(f"Cases JSON: {cases_file}")
1294+ print(f"Shard: {shard}/{num_shards}")
1295+ print(f"Test type: {shard_type}")
1296+ print(f"Total cases: {total_cases}")
1297+ print(f"Test directory: {test_dir}")
1298+ 
1299+ # Execution mode based on test_type
1300+ if shard_type == "distributed":
1301+ print(f"Execution mode: SERIAL (per-case subprocess isolation)")
1302+ else:
1303+ print(f"Execution mode: CONCURRENT ({args.max_workers} workers, per-case subprocess isolation)")
1304+ 
1305+ if args.disabled_testcases:
1306+ disabled_count = result_module.load_disabled_testcases_count(args.disabled_testcases)
1307+ print(f"Disabled testcase entries: {disabled_count}")
1308+ 
1309+ print(f"\n{'=' * 80}\n")
1310+ 
1311+ # Create info dict for cases-json mode
1312+ info = result_module.create_shard_info(shard, num_shards, timestamp)
1313+ info["selection_mode"] = "cases_json"
1314+ info["shard_type"] = shard_type
1315+ info["cases_json_file"] = str(cases_file)
1316+ info["total_cases"] = total_cases
1317+ info["per_case_isolation"] = True
1318+ if args.disabled_testcases:
1319+ info["disabled_count"] = result_module.load_disabled_testcases_count(args.disabled_testcases)
1320+ 
1321+ # Clean old files
1322+ clean_existing_junit_xml(report_dir)
1323+ result_module.get_shard_log_file(report_dir, shard, shard_type).unlink(missing_ok=True)
1324+ 
1325+ # Build execution env
1326+ env_updates = build_execution_env(
1327+ test_dir, script_dir, args.disabled_testcases, shard, shard_type
1328+ )
1329+ 
1330+ # Convert cases to CaseExecutionTask format
1331+ tasks = []
1332+ for i, case in enumerate(planned_cases, 1):
1333+ tasks.append(CaseExecutionTask(
1334+ case_idx=i,
1335+ nodeid=case["nodeid"],
1336+ test_file=case.get("file", ""),
1337+ ))
1338+ 
1339+ # Execute tests based on shard_type
1340+ cases_list = []
1341+ if tasks:
1342+ # Determine execution mode and worker count
1343+ if shard_type == "distributed":
1344+ # Distributed: serial execution (1 worker)
1345+ effective_workers = 1
1346+ print(f"\nExecution mode: SERIAL (distributed tests require sequential execution)")
1347+ else:
1348+ # Regular: concurrent execution
1349+ effective_workers = args.max_workers
1350+ print(f"\nExecution mode: CONCURRENT ({effective_workers} workers)")
1351+ 
1352+ # Execute tasks directly using the new function
1353+ returncode, duration, cases_list = run_tests_with_tasks_concurrent(
1354+ tasks,
1355+ shard,
1356+ test_dir,
1357+ report_dir,
1358+ env_updates,
1359+ args.timeout,
1360+ args.verbose,
1361+ shard_type,
1362+ effective_workers,
1363+ result_module,
1364+ args.quick_test,
1365+ )
1366+ info["execution_mode"] = "serial" if effective_workers == 1 else "concurrent"
1367+ info["concurrent_workers"] = effective_workers
1368+ 
1369+ else:
1370+ print("No cases to execute.")
1371+ returncode = 0
1372+ duration = 0.0
1373+ 
1374+ # Save results and print summary
1375+ save_results_and_summary(
1376+ result_module=result_module,
1377+ report_dir=report_dir,
1378+ shard=shard,
1379+ shard_type=shard_type,
1380+ cases_list=cases_list,
1381+ duration=duration,
1382+ returncode=returncode,
1383+ info=info,
1384+ )
1385+ 
1386+ # Exit with 0 to allow step to succeed and report generation to proceed
1387+ # The actual test results are recorded in cases.json
1388+ sys.exit(0)
1389+ 
1390+ # No valid mode specified (should not reach here due to argument validation)
1391+ print("ERROR: Either --test-files or --cases-json must be specified")
1392+ sys.exit(1)
1393+ 
1394+ 
1395+if __name__ == "__main__":
1396+ main()
A.github/workflows/_torch-npu-upstream-build.yml+227-0
@@ -0,0 +1,227 @@
1+name: Torch NPU Upstream Build
2+ 
3+on:
4+ workflow_call:
5+ inputs:
6+ python_version:
7+ required: true
8+ type: string
9+ description: Python version to use for building
10+ pytorch_version:
11+ required: true
12+ type: string
13+ description: PyTorch version to install
14+ torch_npu_wheel_artifact:
15+ required: true
16+ type: string
17+ description: Name of the artifact to upload the wheel
18+ max_jobs:
19+ required: false
20+ type: string
21+ default: '40'
22+ description: Maximum number of parallel build jobs
23+ outputs:
24+ wheel_name:
25+ description: Name of the built wheel file
26+ value: ${{ jobs.build_torch_npu.outputs.wheel }}
27+ build_status:
28+ description: Build status (0 for success, non-zero for failure)
29+ value: ${{ jobs.build_torch_npu.outputs.status }}
30+ 
31+jobs:
32+ build_torch_npu:
33+ runs-on: linux-aarch64-a3-2
34+ # NOTE: container.image cannot reference env variables in GitHub Actions.
35+ # The DOCKER_IMAGE env below is used for Build Summary display.
36+ # When updating the image, update BOTH container.image AND env.DOCKER_IMAGE.
37+ container:
38+ image: swr.cn-north-4.myhuaweicloud.com/frameworkptadapter/manylinux2_28_aarch64-builder:npu-20241225
39+ options: --user root
40+ outputs:
41+ wheel: ${{ steps.build.outputs.wheel }}
42+ status: ${{ steps.build.outputs.status }}
43+ env:
44+ DOCKER_IMAGE: swr.cn-north-4.myhuaweicloud.com/frameworkptadapter/manylinux2_28_aarch64-builder:npu-20241225
45+ PYTHON_VERSION: ${{ inputs.python_version }}
46+ 
47+ steps:
48+ - name: Checkout repository
49+ uses: actions/checkout@v4
50+ with:
51+ repository: Ascend/pytorch
52+ ref: v2.7.1
53+ fetch-depth: 1
54+ submodules: recursive
55+ 
56+ - name: Check image dependencies
57+ run: |
58+ echo "=== Python Version ==="
59+ python${{ inputs.python_version }} --version
60+ pip${{ inputs.python_version }} --version
61+ 
62+ echo "=== CMake Version ==="
63+ cmake --version | head -1
64+ 
65+ echo "=== GCC Version ==="
66+ gcc --version | head -1
67+ 
68+ echo "=== ccache Version ==="
69+ ccache --version | head -1 || echo "ccache not found"
70+ 
71+ echo "=== nproc ==="
72+ nproc
73+ 
74+ - name: Collect repository metadata
75+ id: repo_meta
76+ run: |
77+ COMMIT=$(git rev-parse HEAD)
78+ COMMIT_SHORT=$(git rev-parse --short HEAD)
79+ COMMIT_DATE=$(git log -1 --format='%ci')
80+ 
81+ echo "commit=${COMMIT}" >> $GITHUB_OUTPUT
82+ echo "commit_short=${COMMIT_SHORT}" >> $GITHUB_OUTPUT
83+ echo "commit_date=${COMMIT_DATE}" >> $GITHUB_OUTPUT
84+ 
85+ - name: Collect toolchain metadata
86+ id: toolchain_meta
87+ run: |
88+ CMAKE_VERSION=$(cmake --version | head -1)
89+ GCC_VERSION=$(gcc --version | head -1)
90+ 
91+ echo "cmake_version=${CMAKE_VERSION}" >> $GITHUB_OUTPUT
92+ echo "gcc_version=${GCC_VERSION}" >> $GITHUB_OUTPUT
93+ 
94+ - name: Setup cache directories
95+ run: |
96+ mkdir -p /github/home/.cache/pip
97+ mkdir -p /github/home/.cache/ccache
98+ chmod -R 777 /github/home/.cache
99+ 
100+ - name: Cache pip
101+ uses: actions/cache@v4
102+ with:
103+ path: /github/home/.cache/pip
104+ # Shared cache key - all workflows use same key to share downloaded packages
105+ key: pip-py${{ inputs.python_version }}-torch${{ inputs.pytorch_version }}-shared
106+ restore-keys: |
107+ pip-py${{ inputs.python_version }}-torch${{ inputs.pytorch_version }}-
108+ pip-py${{ inputs.python_version }}-
109+ 
110+ - name: Cache ccache
111+ uses: actions/cache@v4
112+ with:
113+ path: /github/home/.cache/ccache
114+ key: ccache-py${{ inputs.python_version }}-torch${{ inputs.pytorch_version }}-${{ github.sha }}
115+ restore-keys: |
116+ ccache-py${{ inputs.python_version }}-torch${{ inputs.pytorch_version }}-
117+ 
118+ - name: Install PyTorch ${{ inputs.pytorch_version }} and build dependencies
119+ id: install_torch
120+ run: |
121+ PYTHON=python${{ inputs.python_version }}
122+ PIP=pip${{ inputs.python_version }}
123+ export PIP_CACHE_DIR=/github/home/.cache/pip
124+ 
125+ $PIP install --upgrade pip setuptools wheel
126+ $PIP install torch==${{ inputs.pytorch_version }} --index-url https://download.pytorch.org/whl/cpu
127+ $PIP install pyyaml
128+ 
129+ TORCH_VER=$($PYTHON -c "import torch; print(torch.__version__)")
130+ echo "torch_version=${TORCH_VER}" >> $GITHUB_OUTPUT
131+ echo "PyTorch version: ${TORCH_VER}"
132+ 
133+ - name: Build torch_npu wheel
134+ id: build
135+ run: |
136+ PYTHON=python${{ inputs.python_version }}
137+ 
138+ # 配置 ccache
139+ if command -v ccache &> /dev/null; then
140+ echo "ccache found, enabling ccache"
141+ ccache -M 10G
142+ ccache -z || true
143+ export CC="ccache gcc"
144+ export CXX="ccache g++"
145+ export CCACHE_DIR=/github/home/.cache/ccache
146+ export CCACHE_COMPRESS=1
147+ export CCACHE_MAXSIZE=10G
148+ export CCACHE_BASEDIR="${PWD}"
149+ USE_CCACHE=1
150+ else
151+ echo "ccache not found, building without cache"
152+ USE_CCACHE=0
153+ fi
154+ 
155+ # 构建参数
156+ echo "nproc value: $(nproc)"
157+ echo "MAX_JOBS: ${{ inputs.max_jobs }}"
158+ export MAX_JOBS=${{ inputs.max_jobs }}
159+ export DISABLE_INSTALL_TORCHAIR=FALSE
160+ export BUILD_WITHOUT_SHA=1
161+ 
162+ # 使用 ci/build.sh 脚本
163+ bash ci/build.sh --python=${{ inputs.python_version }} 2>&1 | tee /tmp/build_torch_npu.log
164+ BUILD_STATUS=${PIPESTATUS[0]}
165+ 
166+ # ccache 统计
167+ if [ "${USE_CCACHE}" = "1" ]; then
168+ CCACHE_STATS=$(ccache -s | grep -E "cache hit|cache miss|cache size|hit rate" | tr '\n' ' ')
169+ echo "ccache_stats=${CCACHE_STATS}" >> $GITHUB_OUTPUT
170+ ccache -s
171+ fi
172+ 
173+ echo "status=${BUILD_STATUS}" >> $GITHUB_OUTPUT
174+ 
175+ if [ ${BUILD_STATUS} -eq 0 ]; then
176+ WHL=$(ls dist/*.whl 2>/dev/null | head -1)
177+ echo "wheel=${WHL}" >> $GITHUB_OUTPUT
178+ echo "Build succeeded: ${WHL}"
179+ fi
180+ 
181+ exit ${BUILD_STATUS}
182+ 
183+ - name: Upload build log
184+ if: always()
185+ uses: actions/upload-artifact@v4
186+ with:
187+ name: build-logs-torch-npu
188+ path: /tmp/build_torch_npu.log
189+ if-no-files-found: warn
190+ retention-days: 30
191+ 
192+ - name: Upload built torch_npu wheel
193+ if: steps.build.outputs.status == '0'
194+ uses: actions/upload-artifact@v4
195+ with:
196+ name: ${{ inputs.torch_npu_wheel_artifact }}
197+ path: dist/*.whl
198+ if-no-files-found: error
199+ retention-days: 60
200+ 
201+ - name: Build summary
202+ if: always()
203+ run: |
204+ BUILD_STATUS="${{ steps.build.outputs.status }}"
205+ if [ "${BUILD_STATUS}" = "0" ]; then
206+ BUILD_RESULT="SUCCESS"
207+ else
208+ BUILD_RESULT="FAILED"
209+ fi
210+ 
211+ cat >> $GITHUB_STEP_SUMMARY << EOF
212+ ## torch_npu Source Build
213+ 
214+ | Item | Value |
215+ |------|-------|
216+ | Build time | $(date -u '+%Y-%m-%d %H:%M UTC') |
217+ | Docker image | \`${{ env.DOCKER_IMAGE }}\` |
218+ | CMake | \`${{ steps.toolchain_meta.outputs.cmake_version }}\` |
219+ | GCC | \`${{ steps.toolchain_meta.outputs.gcc_version }}\` |
220+ | Source commit | [\`${{ steps.repo_meta.outputs.commit_short }}\`](${{ github.server_url }}/${{ github.repository }}/commit/${{ steps.repo_meta.outputs.commit }}) |
221+ | Commit time | ${{ steps.repo_meta.outputs.commit_date }} |
222+ | PyTorch | \`${{ steps.install_torch.outputs.torch_version }}\` |
223+ | ccache | ${{ steps.build.outputs.ccache_stats || 'N/A' }} |
224+ | Build result | ${BUILD_RESULT} |
225+ 
226+ $( [ "${BUILD_STATUS}" = "0" ] && echo "> Wheel: \`${{ steps.build.outputs.wheel }}\`" || echo "> See the build-logs-torch-npu artifact for failure details." )
227+ EOF
A.github/workflows/_torch-npu-upstream-collect.yml+154-0
@@ -0,0 +1,154 @@
1+name: Torch NPU Upstream Collect
2+ 
3+on:
4+ workflow_call:
5+ inputs:
6+ python_version:
7+ required: true
8+ type: string
9+ description: Python version to use
10+ pytorch_version:
11+ required: true
12+ type: string
13+ description: PyTorch version to install
14+ prepared_test_src_artifact:
15+ required: true
16+ type: string
17+ description: Name of the prepared test source artifact
18+ torch_npu_wheel_artifact:
19+ required: true
20+ type: string
21+ description: Name of the torch_npu wheel artifact
22+ docker_image:
23+ required: true
24+ type: string
25+ description: Docker image to use
26+ distributed_shards:
27+ required: false
28+ type: string
29+ default: '2'
30+ description: Number of shards for distributed tests
31+ regular_shards:
32+ required: false
33+ type: string
34+ default: '5'
35+ description: Number of shards for regular tests
36+ outputs:
37+ distributed_matrix:
38+ description: Distributed shard matrix JSON
39+ value: ${{ jobs.collect.outputs.distributed_matrix }}
40+ regular_matrix:
41+ description: Regular shard matrix JSON
42+ value: ${{ jobs.collect.outputs.regular_matrix }}
43+ distributed_shards:
44+ description: Number of distributed shards
45+ value: ${{ jobs.collect.outputs.distributed_shards }}
46+ regular_shards:
47+ description: Number of regular shards
48+ value: ${{ jobs.collect.outputs.regular_shards }}
49+ total_cases:
50+ description: Total number of test cases
51+ value: ${{ jobs.collect.outputs.total_cases }}
52+ 
53+jobs:
54+ collect:
55+ runs-on: linux-aarch64-a3-8
56+ timeout-minutes: 60
57+ container:
58+ image: ${{ inputs.docker_image }}
59+ options: --user root
60+ outputs:
61+ distributed_matrix: ${{ steps.collect_and_shard.outputs.distributed_matrix }}
62+ regular_matrix: ${{ steps.collect_and_shard.outputs.regular_matrix }}
63+ distributed_shards: ${{ steps.collect_and_shard.outputs.distributed_shards }}
64+ regular_shards: ${{ steps.collect_and_shard.outputs.regular_shards }}
65+ total_cases: ${{ steps.collect_and_shard.outputs.total_cases }}
66+ 
67+ steps:
68+ - name: Setup NPU test environment
69+ uses: Ascend/pytorch/.github/actions/setup-npu-test-env@v2.7.1
70+ with:
71+ python_version: ${{ inputs.python_version }}
72+ pytorch_version: ${{ inputs.pytorch_version }}
73+ torch_npu_wheel_artifact: ${{ inputs.torch_npu_wheel_artifact }}
74+ prepared_test_src_artifact: ${{ inputs.prepared_test_src_artifact }}
75+ patch_log_suffix: collect
76+ 
77+ - name: Collect all test cases and shard
78+ id: collect_and_shard
79+ run: |
80+ source /usr/local/Ascend/cann/set_env.sh 2>/dev/null || true
81+ source /usr/local/Ascend/nnal/atb/set_env.sh 2>/dev/null || true
82+ 
83+ PYTHON=python${{ inputs.python_version }}
84+ cd pytorch-test-src
85+ 
86+ # Case-level sharding
87+ DISTRIBUTED_SHARDS='${{ inputs.distributed_shards }}'
88+ REGULAR_SHARDS='${{ inputs.regular_shards }}'
89+ 
90+ echo "=== Collecting all test cases ==="
91+ echo "Distributed shards: ${DISTRIBUTED_SHARDS}"
92+ echo "Regular shards: ${REGULAR_SHARDS}"
93+ 
94+ $PYTHON ../ascend_pytorch/.github/scripts/collect_all_cases.py \
95+ --test-dir test \
96+ --case-paths-config test_upstream/case_paths_ci.yml \
97+ --distributed-shards ${DISTRIBUTED_SHARDS} \
98+ --regular-shards ${REGULAR_SHARDS} \
99+ --output-dir cases_shards \
100+ --error-log-dir collection_errors \
101+ --parallel 16 \
102+ 2>&1 | tee /tmp/collect_cases.log
103+ 
104+ # Verify output
105+ echo "=== Generated shard files ==="
106+ ls -la cases_shards/
107+ 
108+ echo "=== Collection summary ==="
109+ cat cases_shards/cases_collection_summary.json
110+ 
111+ # Extract total cases from summary
112+ TOTAL_CASES=$(python3 -c "import json; d=json.load(open('cases_shards/cases_collection_summary.json')); print(d['total_cases'])")
113+ 
114+ # Build shard matrices
115+ DIST_SHARDS=$(seq 1 ${DISTRIBUTED_SHARDS} | tr '\n' ',' | sed 's/,$//')
116+ REG_SHARDS=$(seq 1 ${REGULAR_SHARDS} | tr '\n' ',' | sed 's/,$//')
117+ 
118+ echo "distributed_matrix=[${DIST_SHARDS}]" >> $GITHUB_OUTPUT
119+ echo "distributed_shards=${DISTRIBUTED_SHARDS}" >> $GITHUB_OUTPUT
120+ echo "regular_matrix=[${REG_SHARDS}]" >> $GITHUB_OUTPUT
121+ echo "regular_shards=${REGULAR_SHARDS}" >> $GITHUB_OUTPUT
122+ echo "total_cases=${TOTAL_CASES}" >> $GITHUB_OUTPUT
123+ 
124+ echo "=== Shard configuration ==="
125+ echo "Distributed tests: ${DISTRIBUTED_SHARDS} shards (case-level, serial execution, linux-aarch64-a3-16)"
126+ echo "Regular tests: ${REGULAR_SHARDS} shards (case-level, 64 workers, linux-aarch64-a3-16)"
127+ echo "Total cases: ${TOTAL_CASES}"
128+ 
129+ # Package error logs if any
130+ if [ -d "collection_errors" ] && [ "$(ls -A collection_errors 2>/dev/null)" ]; then
131+ echo "=== Packaging collection error logs ==="
132+ tar -czf collection_errors.tar.gz collection_errors/
133+ echo "Error logs packaged: collection_errors.tar.gz"
134+ ls -la collection_errors.tar.gz
135+ fi
136+ 
137+ - name: Upload cases shard JSONs
138+ uses: actions/upload-artifact@v4
139+ with:
140+ name: cases-shards
141+ path: pytorch-test-src/cases_shards/
142+ retention-days: 60
143+ 
144+ - name: Upload collect logs
145+ if: always()
146+ uses: actions/upload-artifact@v4
147+ with:
148+ name: collect-cases-logs
149+ path: |
150+ /tmp/collect_cases.log
151+ /tmp/torch_env_patch_collect.log
152+ pytorch-test-src/collection_errors.tar.gz
153+ if-no-files-found: warn
154+ retention-days: 60
A.github/workflows/_torch-npu-upstream-prepare.yml+78-0
@@ -0,0 +1,78 @@
1+name: Torch NPU Upstream Prepare
2+ 
3+on:
4+ workflow_call:
5+ inputs:
6+ pytorch_version:
7+ required: true
8+ type: string
9+ description: PyTorch version to clone for test source
10+ prepared_test_src_artifact:
11+ required: true
12+ type: string
13+ description: Name of the artifact for prepared test source
14+ outputs:
15+ patch_count:
16+ description: Number of patches applied
17+ value: ${{ jobs.prepare.outputs.patch_count }}
18+ 
19+jobs:
20+ prepare:
21+ runs-on: ubuntu-latest
22+ outputs:
23+ patch_count: ${{ steps.apply_patches.outputs.patch_count }}
24+ steps:
25+ - name: Checkout repository
26+ uses: actions/checkout@v4
27+ with:
28+ repository: Ascend/pytorch
29+ ref: v2.7.1
30+ fetch-depth: 1
31+ 
32+ - name: Clone PyTorch v${{ inputs.pytorch_version }} (for test source)
33+ run: |
34+ git clone --depth=1 --branch v${{ inputs.pytorch_version }} \
35+ https://github.com/pytorch/pytorch.git pytorch-test-src
36+ 
37+ - name: Copy test_upstream patches
38+ run: |
39+ cp -r test_upstream pytorch-test-src/
40+ 
41+ - name: Apply NPU patches
42+ id: apply_patches
43+ run: |
44+ cd pytorch-test-src/test_upstream
45+ chmod +x apply_patch.sh
46+ # Count patch files before applying
47+ PATCH_COUNT=$(find . -name "*.patch" -o -name "*.diff" | wc -l)
48+ echo "Found ${PATCH_COUNT} patch files"
49+ ./apply_patch.sh 2>&1 | tee /tmp/patch.log
50+ APPLY_STATUS=$?
51+ # Use patch file count as the metric (more reliable than grep Chinese output)
52+ echo "patch_count=${PATCH_COUNT}" >> $GITHUB_OUTPUT
53+ echo "apply_status=${APPLY_STATUS}" >> $GITHUB_OUTPUT
54+ # Fail if apply_patch.sh returned non-zero
55+ if [ ${APPLY_STATUS} -ne 0 ]; then
56+ echo "Patch application failed!"
57+ exit 1
58+ fi
59+ 
60+ - name: Package prepared test source
61+ run: |
62+ tar -czf pytorch-test-src.tar.gz pytorch-test-src
63+ 
64+ - name: Upload prepared test source
65+ uses: actions/upload-artifact@v4
66+ with:
67+ name: ${{ inputs.prepared_test_src_artifact }}
68+ path: pytorch-test-src.tar.gz
69+ retention-days: 60
70+ 
71+ - name: Upload prepare logs
72+ if: always()
73+ uses: actions/upload-artifact@v4
74+ with:
75+ name: prepare-logs
76+ path: /tmp/patch.log
77+ if-no-files-found: warn
78+ retention-days: 60
A.github/workflows/_torch-npu-upstream-report.yml+131-0
@@ -0,0 +1,131 @@
1+name: Torch NPU Upstream Report
2+ 
3+on:
4+ workflow_call:
5+ inputs:
6+ python_version:
7+ required: true
8+ type: string
9+ description: Python version to use
10+ pytorch_version:
11+ required: true
12+ type: string
13+ description: PyTorch version
14+ torch_npu_wheel_name:
15+ required: false
16+ type: string
17+ default: 'source-build.whl'
18+ description: Name of the torch_npu wheel file
19+ patch_count:
20+ required: false
21+ type: string
22+ default: 'N/A'
23+ description: Number of patches applied
24+ docker_image:
25+ required: true
26+ type: string
27+ description: Docker image used for tests
28+ distributed_matrix:
29+ required: false
30+ type: string
31+ default: '[]'
32+ description: Distributed shard matrix JSON
33+ regular_matrix:
34+ required: false
35+ type: string
36+ default: '[]'
37+ description: Regular shard matrix JSON
38+ 
39+jobs:
40+ generate_report:
41+ runs-on: ubuntu-latest
42+ steps:
43+ - name: Checkout repository
44+ uses: actions/checkout@v4
45+ with:
46+ repository: Ascend/pytorch
47+ ref: v2.7.1
48+ fetch-depth: 1
49+ 
50+ - name: Setup Python ${{ inputs.python_version }}
51+ uses: actions/setup-python@v5
52+ with:
53+ python-version: ${{ inputs.python_version }}
54+ 
55+ - name: Download distributed shard reports
56+ uses: actions/download-artifact@v4
57+ with:
58+ pattern: test-reports-dist-*
59+ path: all-test-reports
60+ merge-multiple: true
61+ 
62+ - name: Download regular shard reports
63+ uses: actions/download-artifact@v4
64+ with:
65+ pattern: test-reports-reg-*
66+ path: all-test-reports
67+ merge-multiple: true
68+ 
69+ - name: Download custom test reports
70+ uses: actions/download-artifact@v4
71+ with:
72+ name: test-reports-custom
73+ path: all-test-reports
74+ merge-multiple: true
75+ continue-on-error: true
76+ 
77+ - name: Download cases collection summary
78+ uses: actions/download-artifact@v4
79+ with:
80+ name: cases-shards
81+ path: cases-shards
82+ continue-on-error: true
83+ 
84+ - name: Generate consolidated summary
85+ run: |
86+ PYTHON=python
87+ REPORT_MD=npu-full-test-summary.md
88+ REPORT_JSON=npu-full-test-summary.json
89+ 
90+ # Combine shard matrices for reporting
91+ # Include distributed, regular, and custom shards
92+ DIST_MATRIX='${{ inputs.distributed_matrix }}'
93+ REG_MATRIX='${{ inputs.regular_matrix }}'
94+ 
95+ # Check if custom test reports exist (test_files mode)
96+ CUSTOM_SHARDS="[]"
97+ if [ -d "all-test-reports" ]; then
98+ CUSTOM_FILES=$(find all-test-reports -name "shard_custom-*_stats.json" -o -name "shard_custom-*_cases.json" 2>/dev/null | head -1)
99+ if [ -n "$CUSTOM_FILES" ]; then
100+ CUSTOM_SHARDS='["custom-1"]'
101+ fi
102+ fi
103+ 
104+ COMBINED_MATRIX=$(python3 -c "import sys,json; dist=json.loads('${DIST_MATRIX}'); reg=json.loads('${REG_MATRIX}'); custom=json.loads('${CUSTOM_SHARDS}'); print(json.dumps(['dist-'+str(s) for s in dist]+['reg-'+str(s) for s in reg]+custom))")
105+ 
106+ $PYTHON .github/scripts/generate_npu_full_test_report.py \
107+ --reports-root all-test-reports \
108+ --output-markdown ${REPORT_MD} \
109+ --output-json ${REPORT_JSON} \
110+ --pytorch-version "${{ inputs.pytorch_version }}" \
111+ --torch-npu-whl "${{ inputs.torch_npu_wheel_name }}" \
112+ --patch-count "${{ inputs.patch_count }}" \
113+ --shard-matrix-json "${COMBINED_MATRIX}" \
114+ --docker-image "${{ inputs.docker_image }}" \
115+ --runner "linux-aarch64-a3-16 (distributed, serial), linux-aarch64-a3-16 (regular, 64 workers), linux-aarch64-a3-8 (custom)" \
116+ --cases-summary cases-shards/cases_collection_summary.json \
117+ --cases-by-file-dir cases-shards
118+ 
119+ cat ${REPORT_MD} >> $GITHUB_STEP_SUMMARY
120+ 
121+ - name: Upload consolidated summary
122+ if: always()
123+ uses: actions/upload-artifact@v4
124+ with:
125+ name: npu-full-test-summary
126+ path: |
127+ npu-full-test-summary.md
128+ npu-full-test-summary.json
129+ distributed_cases_results_by_file.jsonl
130+ regular_cases_results_by_file.jsonl
131+ retention-days: 60
A.github/workflows/_torch-npu-upstream-test-custom.yml+117-0
@@ -0,0 +1,117 @@
1+name: Torch NPU Upstream Test Custom
2+ 
3+on:
4+ workflow_call:
5+ inputs:
6+ python_version:
7+ required: true
8+ type: string
9+ description: Python version to use
10+ pytorch_version:
11+ required: true
12+ type: string
13+ description: PyTorch version to install
14+ prepared_test_src_artifact:
15+ required: true
16+ type: string
17+ description: Name of the prepared test source artifact
18+ torch_npu_wheel_artifact:
19+ required: true
20+ type: string
21+ description: Name of the torch_npu wheel artifact
22+ docker_image:
23+ required: true
24+ type: string
25+ description: Docker image to use
26+ test_files:
27+ required: true
28+ type: string
29+ description: Test files to run (comma-separated)
30+ 
31+jobs:
32+ run_tests:
33+ name: test_custom
34+ runs-on: linux-aarch64-a3-8
35+ timeout-minutes: 1200
36+ container:
37+ image: ${{ inputs.docker_image }}
38+ options: --user root
39+ 
40+ steps:
41+ - name: Setup NPU test environment
42+ uses: Ascend/pytorch/.github/actions/setup-npu-test-env@v2.7.1
43+ with:
44+ python_version: ${{ inputs.python_version }}
45+ pytorch_version: ${{ inputs.pytorch_version }}
46+ torch_npu_wheel_artifact: ${{ inputs.torch_npu_wheel_artifact }}
47+ prepared_test_src_artifact: ${{ inputs.prepared_test_src_artifact }}
48+ patch_log_suffix: custom
49+ 
50+ - name: Run custom test files
51+ id: run_tests
52+ run: |
53+ source /usr/local/Ascend/cann/set_env.sh 2>/dev/null || true
54+ source /usr/local/Ascend/nnal/atb/set_env.sh 2>/dev/null || true
55+ 
56+ REPORT_DIR=test-reports
57+ mkdir -p ${REPORT_DIR}
58+ set +e
59+ # Custom test files: per-case isolation execution
60+ python${{ inputs.python_version }} ascend_pytorch/.github/scripts/run_npu_test_shard.py \
61+ --test-files "${{ inputs.test_files }}" \
62+ --test-dir pytorch-test-src/test \
63+ --disabled-testcases pytorch-test-src/test_upstream/disabled_testcases.json \
64+ --report-dir ${REPORT_DIR} \
65+ --timeout 1200 \
66+ --verbose \
67+ 2>&1 | tee /tmp/test_custom.log
68+ 
69+ TEST_STATUS=${PIPESTATUS[0]}
70+ echo "status=${TEST_STATUS}" >> $GITHUB_OUTPUT
71+ # Don't exit with test status - let step succeed to allow report generation
72+ 
73+ - name: Package and upload test reports
74+ if: always()
75+ run: |
76+ # Package junit XMLs into compressed archive
77+ if [ -d "test-reports/junit_xmls" ]; then
78+ echo "=== Compressing junit XMLs ==="
79+ XML_COUNT=$(find test-reports/junit_xmls -type f -name "*.xml" | wc -l)
80+ echo "Found ${XML_COUNT} XML files"
81+ tar -czf test-reports/junit_xmls.tar.gz -C test-reports junit_xmls
82+ rm -rf test-reports/junit_xmls
83+ echo "JUnit XMLs compressed"
84+ fi
85+ 
86+ # Package failed cases logs into compressed archive
87+ if [ -d "test-reports/failed_cases_logs" ]; then
88+ echo "=== Compressing failed cases logs ==="
89+ tar -czf test-reports/failed_cases_logs.tar.gz -C test-reports failed_cases_logs
90+ rm -rf test-reports/failed_cases_logs
91+ echo "Failed cases logs compressed"
92+ fi
93+ 
94+ - name: Upload test reports
95+ if: always()
96+ uses: actions/upload-artifact@v4
97+ with:
98+ name: test-reports-custom
99+ path: test-reports/
100+ retention-days: 60
101+ 
102+ - name: Compress and upload error logs
103+ if: failure()
104+ run: |
105+ mkdir -p error-logs
106+ cp /tmp/test_custom.log error-logs/ 2>/dev/null || true
107+ cp /tmp/torch_env_patch_custom.log error-logs/ 2>/dev/null || true
108+ tar -czf error-logs-custom.tar.gz error-logs/
109+ echo "Error logs compressed"
110+ 
111+ - name: Upload error logs
112+ if: failure()
113+ uses: actions/upload-artifact@v4
114+ with:
115+ name: error-logs-custom
116+ path: error-logs-custom.tar.gz
117+ retention-days: 60
A.github/workflows/_torch-npu-upstream-test-dist.yml+151-0
@@ -0,0 +1,151 @@
1+name: Torch NPU Upstream Test Distributed
2+ 
3+on:
4+ workflow_call:
5+ inputs:
6+ python_version:
7+ required: true
8+ type: string
9+ description: Python version to use
10+ pytorch_version:
11+ required: true
12+ type: string
13+ description: PyTorch version to install
14+ prepared_test_src_artifact:
15+ required: true
16+ type: string
17+ description: Name of the prepared test source artifact
18+ torch_npu_wheel_artifact:
19+ required: true
20+ type: string
21+ description: Name of the torch_npu wheel artifact
22+ docker_image:
23+ required: true
24+ type: string
25+ description: Docker image to use
26+ distributed_matrix:
27+ required: true
28+ type: string
29+ description: Distributed shard matrix JSON
30+ distributed_shards:
31+ required: true
32+ type: string
33+ description: Number of distributed shards
34+ 
35+jobs:
36+ run_tests:
37+ name: test_distributed (${{ matrix.shard }}/${{ inputs.distributed_shards }})
38+ runs-on: linux-aarch64-a3-16
39+ timeout-minutes: 1200
40+ container:
41+ image: ${{ inputs.docker_image }}
42+ options: --user root
43+ strategy:
44+ matrix:
45+ shard: ${{ fromJson(inputs.distributed_matrix) }}
46+ fail-fast: false
47+ max-parallel: 2
48+ 
49+ steps:
50+ - name: Setup NPU test environment
51+ uses: Ascend/pytorch/.github/actions/setup-npu-test-env@v2.7.1
52+ with:
53+ python_version: ${{ inputs.python_version }}
54+ pytorch_version: ${{ inputs.pytorch_version }}
55+ torch_npu_wheel_artifact: ${{ inputs.torch_npu_wheel_artifact }}
56+ prepared_test_src_artifact: ${{ inputs.prepared_test_src_artifact }}
57+ patch_log_suffix: dist_${{ matrix.shard }}
58+ 
59+ - name: Download cases shard JSONs
60+ uses: actions/download-artifact@v4
61+ with:
62+ name: cases-shards
63+ path: cases-shards
64+ 
65+ - name: Run distributed shard ${{ matrix.shard }}/${{ inputs.distributed_shards }}
66+ id: run_test
67+ run: |
68+ source /usr/local/Ascend/cann/set_env.sh 2>/dev/null || true
69+ source /usr/local/Ascend/nnal/atb/set_env.sh 2>/dev/null || true
70+ 
71+ PYTHON=python${{ inputs.python_version }}
72+ REPORT_DIR=test-reports
73+ CASES_JSON="cases-shards/distributed_cases_shard_${{ matrix.shard }}.json"
74+ 
75+ mkdir -p ${REPORT_DIR}
76+ 
77+ # Get case count from JSON
78+ TOTAL_CASES=$(python3 -c "import json; d=json.load(open('${CASES_JSON}')); print(d['total_cases'])")
79+ 
80+ echo "=== Distributed Shard ${{ matrix.shard }} (Case-level) ==="
81+ echo "Total cases: ${TOTAL_CASES}"
82+ echo "Runner: linux-aarch64-a3-16 (16-card NPU)"
83+ echo "Execution mode: SERIAL"
84+ 
85+ # Distributed tests: pre-collected cases, serial execution
86+ set +e
87+ $PYTHON ascend_pytorch/.github/scripts/run_npu_test_shard.py \
88+ --cases-json "${CASES_JSON}" \
89+ --test-dir pytorch-test-src/test \
90+ --disabled-testcases pytorch-test-src/test_upstream/disabled_testcases.json \
91+ --report-dir ${REPORT_DIR} \
92+ --timeout 1200 \
93+ --verbose \
94+ 2>&1 | tee /tmp/test_shard_dist_${{ matrix.shard }}.log
95+ 
96+ TEST_STATUS=${PIPESTATUS[0]}
97+ set -e
98+ echo "status=${TEST_STATUS}" >> $GITHUB_OUTPUT
99+ # Don't exit with test status - let step succeed to allow report generation
100+ 
101+ - name: Package and upload test reports
102+ if: always()
103+ run: |
104+ # Package junit XMLs into compressed archive
105+ if [ -d "test-reports/junit_xmls" ]; then
106+ echo "=== Compressing junit XMLs ==="
107+ XML_COUNT=$(find test-reports/junit_xmls -type f -name "*.xml" | wc -l)
108+ echo "Found ${XML_COUNT} XML files"
109+ tar -czf test-reports/junit_xmls.tar.gz -C test-reports junit_xmls
110+ rm -rf test-reports/junit_xmls
111+ echo "JUnit XMLs compressed: $(ls -lh test-reports/junit_xmls.tar.gz)"
112+ fi
113+ 
114+ # Package cases logs into compressed archive
115+ if [ -d "test-reports/cases_logs" ]; then
116+ echo "=== Compressing cases logs ==="
117+ tar -czf test-reports/cases_logs.tar.gz -C test-reports cases_logs
118+ rm -rf test-reports/cases_logs
119+ echo "Cases logs compressed: $(ls -lh test-reports/cases_logs.tar.gz)"
120+ fi
121+ 
122+ # Package shard_cases.json
123+ if [ -f "test-reports/shard_dist-${{ matrix.shard }}_cases.json" ]; then
124+ echo "Cases JSON exists: $(ls -lh test-reports/shard_dist-${{ matrix.shard }}_cases.json)"
125+ fi
126+ 
127+ - name: Upload test reports
128+ if: always()
129+ uses: actions/upload-artifact@v4
130+ with:
131+ name: test-reports-dist-${{ matrix.shard }}
132+ path: test-reports/
133+ retention-days: 60
134+ 
135+ - name: Compress and upload error logs
136+ if: failure()
137+ run: |
138+ # Only upload logs when tests failed
139+ mkdir -p error-logs
140+ cp /tmp/test_shard_dist_${{ matrix.shard }}.log error-logs/ 2>/dev/null || true
141+ cp /tmp/torch_env_patch_dist_${{ matrix.shard }}.log error-logs/ 2>/dev/null || true
142+ tar -czf error-logs-dist-${{ matrix.shard }}.tar.gz error-logs/
143+ echo "Error logs compressed: $(ls -lh error-logs-dist-${{ matrix.shard }}.tar.gz)"
144+ 
145+ - name: Upload error logs
146+ if: failure()
147+ uses: actions/upload-artifact@v4
148+ with:
149+ name: error-logs-dist-${{ matrix.shard }}
150+ path: error-logs-dist-${{ matrix.shard }}.tar.gz
151+ retention-days: 60
A.github/workflows/_torch-npu-upstream-test-regular.yml+154-0
@@ -0,0 +1,154 @@
1+name: Torch NPU Upstream Test Regular
2+ 
3+on:
4+ workflow_call:
5+ inputs:
6+ python_version:
7+ required: true
8+ type: string
9+ description: Python version to use
10+ pytorch_version:
11+ required: true
12+ type: string
13+ description: PyTorch version to install
14+ prepared_test_src_artifact:
15+ required: true
16+ type: string
17+ description: Name of the prepared test source artifact
18+ torch_npu_wheel_artifact:
19+ required: true
20+ type: string
21+ description: Name of the torch_npu wheel artifact
22+ docker_image:
23+ required: true
24+ type: string
25+ description: Docker image to use
26+ regular_matrix:
27+ required: true
28+ type: string
29+ description: Regular shard matrix JSON
30+ regular_shards:
31+ required: true
32+ type: string
33+ description: Number of regular shards
34+ 
35+jobs:
36+ run_tests:
37+ name: test_regular (${{ matrix.shard }}/${{ inputs.regular_shards }})
38+ runs-on: linux-aarch64-a3-16
39+ timeout-minutes: 1200
40+ container:
41+ image: ${{ inputs.docker_image }}
42+ options: --user root
43+ strategy:
44+ matrix:
45+ shard: ${{ fromJson(inputs.regular_matrix) }}
46+ fail-fast: false
47+ max-parallel: 5
48+ 
49+ steps:
50+ - name: Setup NPU test environment
51+ uses: Ascend/pytorch/.github/actions/setup-npu-test-env@v2.7.1
52+ with:
53+ python_version: ${{ inputs.python_version }}
54+ pytorch_version: ${{ inputs.pytorch_version }}
55+ torch_npu_wheel_artifact: ${{ inputs.torch_npu_wheel_artifact }}
56+ prepared_test_src_artifact: ${{ inputs.prepared_test_src_artifact }}
57+ patch_log_suffix: reg_${{ matrix.shard }}
58+ 
59+ - name: Download cases shard JSONs
60+ uses: actions/download-artifact@v4
61+ with:
62+ name: cases-shards
63+ path: cases-shards
64+ 
65+ - name: Run regular shard ${{ matrix.shard }}/${{ inputs.regular_shards }}
66+ id: run_test
67+ run: |
68+ source /usr/local/Ascend/cann/set_env.sh 2>/dev/null || true
69+ source /usr/local/Ascend/nnal/atb/set_env.sh 2>/dev/null || true
70+ 
71+ PYTHON=python${{ inputs.python_version }}
72+ REPORT_DIR=test-reports
73+ CASES_JSON="cases-shards/regular_cases_shard_${{ matrix.shard }}.json"
74+ 
75+ mkdir -p ${REPORT_DIR}
76+ 
77+ # Get case count from JSON
78+ TOTAL_CASES=$(python3 -c "import json; d=json.load(open('${CASES_JSON}')); print(d['total_cases'])")
79+ 
80+ echo "=== Regular Shard ${{ matrix.shard }} (Case-level) ==="
81+ echo "Total cases: ${TOTAL_CASES}"
82+ echo "Runner: linux-aarch64-a3-16 (16-card NPU)"
83+ echo "Execution mode: CONCURRENT (16 workers)"
84+ 
85+ # Regular tests: pre-collected cases, 16 concurrent workers
86+ set +e
87+ $PYTHON ascend_pytorch/.github/scripts/run_npu_test_shard.py \
88+ --cases-json "${CASES_JSON}" \
89+ --test-dir pytorch-test-src/test \
90+ --disabled-testcases pytorch-test-src/test_upstream/disabled_testcases.json \
91+ --report-dir ${REPORT_DIR} \
92+ --timeout 1200 \
93+ --max-workers 64 \
94+ --verbose \
95+ 2>&1 | tee /tmp/test_shard_reg_${{ matrix.shard }}.log
96+ 
97+ TEST_STATUS=${PIPESTATUS[0]}
98+ set -e
99+ echo "status=${TEST_STATUS}" >> $GITHUB_OUTPUT
100+ # Don't exit with test status - let step succeed to allow report generation
101+ 
102+ - name: Package and upload test reports
103+ if: always()
104+ run: |
105+ # Package junit XMLs into compressed archive
106+ if [ -d "test-reports/junit_xmls" ]; then
107+ echo "=== Compressing junit XMLs ==="
108+ XML_COUNT=$(find test-reports/junit_xmls -type f -name "*.xml" | wc -l)
109+ echo "Found ${XML_COUNT} XML files"
110+ tar -czf test-reports/junit_xmls.tar.gz -C test-reports junit_xmls
111+ rm -rf test-reports/junit_xmls
112+ echo "JUnit XMLs compressed: $(ls -lh test-reports/junit_xmls.tar.gz)"
113+ fi
114+ 
115+ # Package cases logs into compressed archive
116+ if [ -d "test-reports/cases_logs" ]; then
117+ echo "=== Compressing cases logs ==="
118+ LOGS_COUNT=$(find test-reports/cases_logs -type f | wc -l)
119+ echo "Found ${LOGS_COUNT} case log files"
120+ tar -czf test-reports/cases_logs.tar.gz -C test-reports cases_logs
121+ rm -rf test-reports/cases_logs
122+ echo "Cases logs compressed: $(ls -lh test-reports/cases_logs.tar.gz)"
123+ fi
124+ 
125+ # Package shard_cases.json
126+ if [ -f "test-reports/shard_reg-${{ matrix.shard }}_cases.json" ]; then
127+ echo "Cases JSON exists: $(ls -lh test-reports/shard_reg-${{ matrix.shard }}_cases.json)"
128+ fi
129+ 
130+ - name: Upload test reports
131+ if: always()
132+ uses: actions/upload-artifact@v4
133+ with:
134+ name: test-reports-reg-${{ matrix.shard }}
135+ path: test-reports/
136+ retention-days: 60
137+ 
138+ - name: Compress and upload error logs
139+ if: failure()
140+ run: |
141+ # Only upload logs when tests failed
142+ mkdir -p error-logs
143+ cp /tmp/test_shard_reg_${{ matrix.shard }}.log error-logs/ 2>/dev/null || true
144+ cp /tmp/torch_env_patch_reg_${{ matrix.shard }}.log error-logs/ 2>/dev/null || true
145+ tar -czf error-logs-reg-${{ matrix.shard }}.tar.gz error-logs/
146+ echo "Error logs compressed: $(ls -lh error-logs-reg-${{ matrix.shard }}.tar.gz)"
147+ 
148+ - name: Upload error logs
149+ if: failure()
150+ uses: actions/upload-artifact@v4
151+ with:
152+ name: error-logs-reg-${{ matrix.shard }}
153+ path: error-logs-reg-${{ matrix.shard }}.tar.gz
154+ retention-days: 60
A.github/workflows/_torch-npu-upstream-test.yml+149-0
@@ -0,0 +1,149 @@
1+name: Torch NPU Upstream Test
2+ 
3+on:
4+ workflow_call:
5+ inputs:
6+ python_version:
7+ required: true
8+ type: string
9+ description: Python version to use
10+ pytorch_version:
11+ required: true
12+ type: string
13+ description: PyTorch version to use
14+ distributed_shards:
15+ required: false
16+ type: string
17+ default: '2'
18+ description: Number of shards for distributed tests
19+ regular_shards:
20+ required: false
21+ type: string
22+ default: '5'
23+ description: Number of shards for regular tests
24+ test_files:
25+ required: false
26+ type: string
27+ default: ''
28+ description: Test files to run directly (comma-separated)
29+ 
30+defaults:
31+ run:
32+ shell: bash
33+ 
34+jobs:
35+ # ============================================================================
36+ # 1. Prepare Test Environment
37+ # ============================================================================
38+ prepare:
39+ uses: ./.github/workflows/_torch-npu-upstream-prepare.yml
40+ with:
41+ pytorch_version: ${{ inputs.pytorch_version }}
42+ prepared_test_src_artifact: pytorch-test-src-${{ inputs.pytorch_version }}-patched
43+ 
44+ # ============================================================================
45+ # 2. Build torch_npu Wheel
46+ # ============================================================================
47+ build_torch_npu:
48+ needs: prepare
49+ uses: ./.github/workflows/_torch-npu-upstream-build.yml
50+ with:
51+ python_version: ${{ inputs.python_version }}
52+ pytorch_version: ${{ inputs.pytorch_version }}
53+ torch_npu_wheel_artifact: torch-npu-wheel-${{ inputs.pytorch_version }}-source
54+ max_jobs: '40'
55+ 
56+ # ============================================================================
57+ # 3. Collect Test Cases (only when test_files is empty)
58+ # ============================================================================
59+ collect_cases:
60+ needs:
61+ - prepare
62+ - build_torch_npu
63+ if: ${{ inputs.test_files == '' }}
64+ uses: ./.github/workflows/_torch-npu-upstream-collect.yml
65+ with:
66+ python_version: ${{ inputs.python_version }}
67+ pytorch_version: ${{ inputs.pytorch_version }}
68+ prepared_test_src_artifact: pytorch-test-src-${{ inputs.pytorch_version }}-patched
69+ torch_npu_wheel_artifact: torch-npu-wheel-${{ inputs.pytorch_version }}-source
70+ docker_image: swr.cn-north-4.myhuaweicloud.com/frameworkptadapter/pytorch_2.7.1_a3_aarch64_builder:20260424
71+ distributed_shards: ${{ inputs.distributed_shards }}
72+ regular_shards: ${{ inputs.regular_shards }}
73+ 
74+ # ============================================================================
75+ # 4. Run Distributed Tests (only when test_files is empty)
76+ # ============================================================================
77+ test_distributed:
78+ needs:
79+ - prepare
80+ - collect_cases
81+ - build_torch_npu
82+ if: ${{ inputs.test_files == '' }}
83+ uses: ./.github/workflows/_torch-npu-upstream-test-dist.yml
84+ with:
85+ python_version: ${{ inputs.python_version }}
86+ pytorch_version: ${{ inputs.pytorch_version }}
87+ prepared_test_src_artifact: pytorch-test-src-${{ inputs.pytorch_version }}-patched
88+ torch_npu_wheel_artifact: torch-npu-wheel-${{ inputs.pytorch_version }}-source
89+ docker_image: swr.cn-north-4.myhuaweicloud.com/frameworkptadapter/pytorch_2.7.1_a3_aarch64_builder:20260424
90+ distributed_matrix: ${{ needs.collect_cases.outputs.distributed_matrix }}
91+ distributed_shards: ${{ needs.collect_cases.outputs.distributed_shards }}
92+ 
93+ # ============================================================================
94+ # 5. Run Regular Tests (only when test_files is empty)
95+ # ============================================================================
96+ test_regular:
97+ needs:
98+ - prepare
99+ - collect_cases
100+ - build_torch_npu
101+ if: ${{ inputs.test_files == '' }}
102+ uses: ./.github/workflows/_torch-npu-upstream-test-regular.yml
103+ with:
104+ python_version: ${{ inputs.python_version }}
105+ pytorch_version: ${{ inputs.pytorch_version }}
106+ prepared_test_src_artifact: pytorch-test-src-${{ inputs.pytorch_version }}-patched
107+ torch_npu_wheel_artifact: torch-npu-wheel-${{ inputs.pytorch_version }}-source
108+ docker_image: swr.cn-north-4.myhuaweicloud.com/frameworkptadapter/pytorch_2.7.1_a3_aarch64_builder:20260424
109+ regular_matrix: ${{ needs.collect_cases.outputs.regular_matrix }}
110+ regular_shards: ${{ needs.collect_cases.outputs.regular_shards }}
111+ 
112+ # ============================================================================
113+ # 6. Run Custom Tests (only when test_files is provided)
114+ # ============================================================================
115+ test_custom:
116+ needs:
117+ - prepare
118+ - build_torch_npu
119+ if: ${{ inputs.test_files != '' }}
120+ uses: ./.github/workflows/_torch-npu-upstream-test-custom.yml
121+ with:
122+ python_version: ${{ inputs.python_version }}
123+ pytorch_version: ${{ inputs.pytorch_version }}
124+ prepared_test_src_artifact: pytorch-test-src-${{ inputs.pytorch_version }}-patched
125+ torch_npu_wheel_artifact: torch-npu-wheel-${{ inputs.pytorch_version }}-source
126+ docker_image: swr.cn-north-4.myhuaweicloud.com/frameworkptadapter/pytorch_2.7.1_a3_aarch64_builder:20260424
127+ test_files: ${{ inputs.test_files }}
128+ 
129+ # ============================================================================
130+ # 7. Generate Test Report
131+ # ============================================================================
132+ report:
133+ needs:
134+ - prepare
135+ - build_torch_npu
136+ - collect_cases
137+ - test_distributed
138+ - test_regular
139+ - test_custom
140+ if: always() && needs.prepare.result == 'success' && needs.build_torch_npu.result == 'success'
141+ uses: ./.github/workflows/_torch-npu-upstream-report.yml
142+ with:
143+ python_version: ${{ inputs.python_version }}
144+ pytorch_version: ${{ inputs.pytorch_version }}
145+ torch_npu_wheel_name: ${{ needs.build_torch_npu.outputs.wheel_name || 'source-build.whl' }}
146+ patch_count: ${{ needs.prepare.outputs.patch_count || 'N/A' }}
147+ docker_image: swr.cn-north-4.myhuaweicloud.com/frameworkptadapter/pytorch_2.7.1_a3_aarch64_builder:20260424
148+ distributed_matrix: ${{ needs.collect_cases.outputs.distributed_matrix || '[]' }}
149+ regular_matrix: ${{ needs.collect_cases.outputs.regular_matrix || '[]' }}
A.github/workflows/torch-npu-upstream-test-trigger.yml+37-0
@@ -0,0 +1,37 @@
1+name: Torch NPU Upstream v2.7.1 Trigger
2+ 
3+on:
4+ schedule:
5+ - cron: '0 11 * * *' # UTC 11:00 (Beijing time 19:00), every day
6+ workflow_dispatch:
7+ inputs:
8+ python_version:
9+ description: 'Python version (default 3.11)'
10+ required: false
11+ default: '3.11'
12+ type: string
13+ distributed_shards:
14+ description: 'Number of shards for distributed tests (default 2)'
15+ required: false
16+ default: '2'
17+ type: string
18+ regular_shards:
19+ description: 'Number of shards for regular tests (default 5)'
20+ required: false
21+ default: '5'
22+ type: string
23+ test_files:
24+ description: 'Test files to run directly (comma-separated, e.g., "test_meta.py,test_nn.py"). Skip shard assignment if set.'
25+ required: false
26+ default: ''
27+ type: string
28+ 
29+jobs:
30+ trigger_test:
31+ uses: ./.github/workflows/_torch-npu-upstream-test.yml
32+ with:
33+ python_version: ${{ github.event.inputs.python_version || '3.11' }}
34+ pytorch_version: '2.7.1'
35+ distributed_shards: ${{ github.event.inputs.distributed_shards || '2' }}
36+ regular_shards: ${{ github.event.inputs.regular_shards || '5' }}
37+ test_files: ${{ github.event.inputs.test_files || '' }}
Mtest/requirements.txt+2-1
@@ -12,11 +12,12 @@ onnxruntime==1.18.1
12onnxscript==0.2.212onnxscript==0.2.2
13Pillow==10.3.013Pillow==10.3.0
14requests==2.32.014requests==2.32.0
15+pytest-timeout==2.3.1
15torch_geometric==2.5.316torch_geometric==2.5.3
16transformers==4.40.017transformers==4.40.0
17pytest==8.1.118pytest==8.1.1
18parameterized==0.9.019parameterized==0.9.0
19torch-scatter==2.1.220torch-scatter==2.1.2
20-torchvision==0.22.021+torchvision==0.22.1
21ml-dtypes==0.2.022ml-dtypes==0.2.0
22protobuf==3.20.223protobuf==3.20.2
Atest_upstream/case_paths_ci.yml+192-0
@@ -0,0 +1,192 @@
1+whitelist:
2+ - test/export
3+ - test/quantization
4+ - test/test_binary_ufuncs.py
5+ - test/test_foreach.py
6+ - test/test_nestedtensor.py
7+ - test/test_transformers.py
8+ - test/test_utils.py
9+ - test/torch_np
10+ - test/distributed
11+ - test/distributions
12+ - test/functorch
13+ - test/nn
14+ - test/onnx
15+ - test/test_ao_sparsity.py
16+ - test/test_mobile_optimizer.py
17+ - test/test_modules.py
18+ - test/test_nn.py
19+ - test/test_ops.py
20+ - test/test_optim.py
21+ - test/test_quantization.py
22+ - test/test_sparse.py
23+ - test/test_sparse_csr.py
24+ - test/test_sparse_semi_structured.py
25+ - test/dynamo
26+ - test/fx
27+ - test/test_fx_passes.py
28+ - test/test_fx_experimental.py
29+ - test/test_fx.py
30+ - test/test_fx_reinplace_pass.py
31+ - test/test_jit_fuser_te.py
32+ - test/test_autograd.py
33+ - test/test_package.py
34+ - test/test_accelerator.py
35+ - test/test_appending_byte_serializer.py
36+ - test/test_autocast.py
37+ - test/test_autoload.py
38+ - test/test_bundled_images.py
39+ - test/test_bundled_inputs.py
40+ - test/test_ci_sanity_check_fail.py
41+ - test/test_comparison_utils.py
42+ - test/test_compile_benchmark_util.py
43+ - test/test_complex.py
44+ - test/test_content_store.py
45+ - test/test_cpp_extensions_aot.py
46+ - test/test_cpp_extensions_mtia_backend.py
47+ - test/test_cpp_extensions_open_device_registration.py
48+ - test/test_cpp_extensions_stream_and_event.py
49+ - test/test_dispatch.py
50+ - test/profiler
51+ - test/test_functionalization_of_rng_ops.py
52+ - test/test_futures.py
53+ - test/test_throughput_benchmark.py
54+ - test/test_typing.py
55+ - test/test_type_hints.py
56+ - test/test_transformers_privateuse1.py
57+ - test/test_extension_utils.py
58+ - test/test_dlpack.py
59+ - test/test_cuda.py
60+ - test/test_cuda_multigpu.py
61+ - test/test_cuda_nvml_based_avail.py
62+ - test/test_cuda_primary_ctx.py
63+ - test/test_dataloader.py
64+ - test/test_deploy.py
65+ - test/test_determination.py
66+ - test/test_function_schema.py
67+ - test/test_functional_optim.py
68+ - test/test_functionalization.py
69+ - test/backends
70+ - test/benchmark_utils
71+ - test/test_cpp_api_parity.py
72+ - test/custom_backend
73+ - test/custom_operator
74+ - test/lazy
75+ - test/mobile
76+ - test/test_fake_tensor.py
77+ - test/higher_order_ops
78+ - test/test_decomp.py
79+ - test/test_schema_check.py
80+ - test/test_cuda_sanitizer.py
81+ - test/test_cuda_trace.py
82+ - test/test_show_pickle.py
83+ - test/test_sympy_utils.py
84+ - test/test_stateless.py
85+ - test/test_subclass.py
86+ - test/test_static_runtime.py
87+ - test/test_tensorboard.py
88+ - test/test_sort_and_select.py
89+ - test/test_spectral_ops.py
90+ - test/test_tensor_creation_ops.py
91+ - test/test_torch.py
92+ - test/test_type_promotion.py
93+ - test/test_type_info.py
94+ - test/test_tensorexpr_pybind.py
95+ - test/test_reductions.py
96+ - test/test_scatter_gather_ops.py
97+ - test/test_segment_reductions.py
98+ - test/test_serialization.py
99+ - test/test_set_default_mobile_cpu_allocator.py
100+ - test/test_shape_ops.py
101+ - test/test_unary_ufuncs.py
102+ - test/test_meta.py
103+ - test/test_ops_fwd_gradients.py
104+ - test/test_numba_integration.py
105+ - test/test_numpy_interop.py
106+ - test/test_pytree.py
107+ - test/test_per_overload_api.py
108+ - test/test_prims.py
109+ - test/test_pruning_op.py
110+ - test/test_python_dispatch.py
111+ - test/test_hop_infra.py
112+ - test/test_hub.py
113+ - test/test_import_stats.py
114+ - test/test_indexing.py
115+ - test/test_itt.py
116+ - test/test_jit_disabled.py
117+ - test/test_jit_llga_fuser.py
118+ - test/test_jit_string.py
119+ - test/test_jiterator.py
120+ - test/test_kernel_launch_checks.py
121+ - test/test_legacy_vmap.py
122+ - test/test_license.py
123+ - test/test_linalg.py
124+ - test/test_logging.py
125+ - test/test_metal.py
126+ - test/test_mkl_verbose.py
127+ - test/test_mkldnn.py
128+ - test/test_mkldnn_fusion.py
129+ - test/test_mkldnn_verbose.py
130+ - test/test_model_exports_to_core_aten.py
131+ - test/test_module_tracker.py
132+ - test/test_monitor.py
133+ - test/test_mps.py
134+ - test/test_multiprocessing.py
135+ - test/test_multiprocessing_spawn.py
136+ - test/test_namedtensor.py
137+ - test/test_namedtuple_return_api.py
138+ - test/test_native_functions.py
139+ - test/test_native_mha.py
140+ - test/test_custom_ops.py
141+ - test/test_datapipe.py
142+ - test/test_dynamic_shapes.py
143+ - test/test_expanded_weights.py
144+ - test/test_masked.py
145+ - test/test_matmul_cuda.py
146+ - test/test_view_ops.py
147+ - test/test_testing.py
148+ - test/test_utils_config_module.py
149+ - test/test_utils_filelock.py
150+ - test/test_vulkan.py
151+ - test/test_weak.py
152+ - test/test_xnnpack_integration.py
153+ - test/test_xpu.py
154+ - test/xpu
155+ - test/test_nnapi.py
156+ - test/test_openmp.py
157+ - test/test_ops_gradients.py
158+ - test/test_out_dtype_op.py
159+ - test/test_overrides.py
160+ - test/test_proxy_tensor.py
161+ - test/test_public_bindings.py
162+blacklist:
163+ - test/export/test_export_legacy.py
164+ - test/distributed/launcher
165+ - test/distributed/test_nccl.py
166+ - test/distributed/test_c10d_ucc.py
167+ - test/distributed/rpc/cuda/test_tensorpipe_agent.py
168+ - test/distributed/test_symmetric_memory.py
169+ - test/distributed/_composable/fsdp/test_fully_shard_mixed_precision.py
170+ - test/distributed/fsdp/test_fsdp_mixed_precision.py
171+ - test/distributed/test_distributed_spawn.py
172+ - test/distributed/test_c10d_functional_native.py
173+ - test/distributed/fsdp/test_fsdp_comm_hooks.py
174+ - test/distributed/test_c10d_nccl.py
175+ - test/distributed/tensor/test_matrix_ops.py
176+ - test/distributed/algorithms/quantization/test_quantization.py
177+ - test/distributed/bin/test_script.py
178+ - test/distributed/elastic/multiprocessing/bin/test_script.py
179+ - test/distributed/_composable/fsdp/test_fully_shard_logging.py
180+ - test/distributed/test_c10d_spawn.py
181+ - test/fx/test_shape_inference.py
182+ - test/fx/test_future.py
183+ - test/dynamo/test_torchrec.py
184+ - test/custom_operator/test_custom_ops.py
185+ - test/mobile/test_lite_script_module.py
186+ - test/test_bundled_images.py
187+ - test/test_cpp_extensions_aot.py
188+ - test/test_cuda_trace.py
189+ - test/test_cuda_sanitizer.py
190+ - test/xpu/test_gemm.py
191+ - test/xpu/test_conv.py
192+ - test/test_mps.py
Atest_upstream/torch_env_patch.sh+294-0
@@ -0,0 +1,294 @@
1+#!/bin/bash
2+# torch_env_patch.sh - Apply patches to installed torch package in Python environment
3+#
4+# This script applies patches from test_upstream/torch/ directory to the
5+# torch package installed in the Python environment (e.g., site-packages/torch).
6+#
7+# Usage:
8+# ./torch_env_patch.sh [--python=<version>] [--patch-dir=<path>] [--dry-run]
9+#
10+# Options:
11+# --python=<version> Python version to use (e.g., 3.11). Default: auto-detect
12+# --patch-dir=<path> Directory containing torch patches. Default: test_upstream/torch
13+# --dry-run Only check what patches would be applied, don't actually apply
14+# -v, --verbose Show verbose output
15+#
16+# Environment variables:
17+# PYTHON_VERSION Python version (alternative to --python flag)
18+# TORCH_PATCH_DIR Patch directory (alternative to --patch-dir flag)
19+ 
20+set -e
21+ 
22+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
23+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
24+ 
25+# Default values
26+PYTHON_VERSION=""
27+PATCH_DIR=""
28+DRY_RUN=false
29+VERBOSE=false
30+ 
31+# Parse arguments
32+while [[ $# -gt 0 ]]; do
33+ case $1 in
34+ --python=*)
35+ PYTHON_VERSION="${1#*=}"
36+ shift
37+ ;;
38+ --python)
39+ PYTHON_VERSION="$2"
40+ shift 2
41+ ;;
42+ --patch-dir=*)
43+ PATCH_DIR="${1#*=}"
44+ shift
45+ ;;
46+ --patch-dir)
47+ PATCH_DIR="$2"
48+ shift 2
49+ ;;
50+ --dry-run)
51+ DRY_RUN=true
52+ shift
53+ ;;
54+ -v|--verbose)
55+ VERBOSE=true
56+ shift
57+ ;;
58+ -h|--help)
59+ echo "Usage: $SCRIPT_NAME [options]"
60+ echo ""
61+ echo "Apply patches from test_upstream/torch/ to installed torch package."
62+ echo ""
63+ echo "Options:"
64+ echo " --python=<version> Python version (e.g., 3.11)"
65+ echo " --patch-dir=<path> Patch directory (default: ./torch relative to script)"
66+ echo " --dry-run Check only, don't apply patches"
67+ echo " -v, --verbose Show verbose output"
68+ echo " -h, --help Show this help message"
69+ exit 0
70+ ;;
71+ *)
72+ echo "Unknown option: $1"
73+ echo "Use --help for usage information"
74+ exit 1
75+ ;;
76+ esac
77+done
78+ 
79+# Apply environment variables if not set via arguments
80+PYTHON_VERSION="${PYTHON_VERSION:-${PYTHON_VERSION:-}}"
81+PATCH_DIR="${PATCH_DIR:-${TORCH_PATCH_DIR:-$SCRIPT_DIR/torch}}"
82+ 
83+# Resolve Python executable
84+if [ -n "$PYTHON_VERSION" ]; then
85+ PYTHON="python${PYTHON_VERSION}"
86+ PIP="pip${PYTHON_VERSION}"
87+else
88+ # Auto-detect Python version
89+ PYTHON="python3"
90+ PIP="pip3"
91+fi
92+ 
93+# Verify Python is available
94+if ! command -v "$PYTHON" &> /dev/null; then
95+ echo "ERROR: Python executable '$PYTHON' not found"
96+ exit 1
97+fi
98+ 
99+PYTHON_VER_FULL=$($PYTHON --version 2>&1)
100+echo "Using Python: $PYTHON_VER_FULL"
101+ 
102+# Find torch package installation location
103+TORCH_PATH=$($PYTHON -c "import torch; print(torch.__path__[0])" 2>/dev/null || echo "")
104+ 
105+if [ -z "$TORCH_PATH" ]; then
106+ echo "ERROR: torch package not found in Python environment"
107+ echo "Please install torch first: $PIP install torch"
108+ exit 1
109+fi
110+ 
111+echo "Torch package location: $TORCH_PATH"
112+ 
113+# Show torch installation directory contents for diagnostics
114+echo ""
115+echo "=== Torch installation directory structure ==="
116+echo "Top-level directories in $TORCH_PATH:"
117+ls -d "$TORCH_PATH"/*/ 2>/dev/null | head -20 || ls "$TORCH_PATH" | head -20
118+ 
119+echo ""
120+echo "Testing directory contents:"
121+if [ -d "$TORCH_PATH/testing" ]; then
122+ ls -la "$TORCH_PATH/testing" | head -15
123+ echo ""
124+ if [ -d "$TORCH_PATH/testing/_internal" ]; then
125+ echo "Testing/_internal directory contents:"
126+ ls "$TORCH_PATH/testing/_internal" | head -20
127+ else
128+ echo "NOTE: torch.testing._internal directory NOT FOUND"
129+ echo "This module may not be included in this torch installation"
130+ fi
131+else
132+ echo "NOTE: torch.testing directory NOT FOUND"
133+fi
134+echo "=== End of torch directory structure ==="
135+echo ""
136+ 
137+# Verify patch directory exists
138+if [ ! -d "$PATCH_DIR" ]; then
139+ echo "ERROR: Patch directory not found: $PATCH_DIR"
140+ exit 1
141+fi
142+ 
143+echo "Patch directory: $PATCH_DIR"
144+ 
145+# Find all patch files
146+PATCH_FILES=$(find "$PATCH_DIR" -type f \( -name "*.patch" -o -name "*.diff" \) | sort)
147+ 
148+if [ -z "$PATCH_FILES" ]; then
149+ echo "No patch files found in $PATCH_DIR"
150+ exit 0
151+fi
152+ 
153+PATCH_COUNT=$(echo "$PATCH_FILES" | wc -l)
154+echo "Found $PATCH_COUNT patch files"
155+ 
156+# Statistics
157+SUCCESS_COUNT=0
158+FAIL_COUNT=0
159+SKIP_COUNT=0
160+MISSING_COUNT=0
161+ 
162+# Verify torch.testing._internal exists (common target for patches)
163+if [ ! -d "$TORCH_PATH/testing/_internal" ]; then
164+ echo ""
165+ echo "WARNING: torch.testing._internal directory not found in torch package"
166+ echo "Some patches may fail to apply"
167+ echo "Expected path: $TORCH_PATH/testing/_internal"
168+ echo ""
169+fi
170+ 
171+# Apply patches
172+echo ""
173+echo "========================================"
174+echo "Applying torch environment patches..."
175+echo "========================================"
176+ 
177+# Change to the parent directory of torch package (site-packages)
178+# Patch files use paths like "torch/testing/_internal/common_utils.py"
179+# With -p1, this becomes "testing/_internal/common_utils.py" which we need to find
180+# So we cd to the parent of torch (site-packages) and use -p1
181+TORCH_PARENT_DIR=$(dirname "$TORCH_PATH")
182+echo "Working directory: $TORCH_PARENT_DIR"
183+cd "$TORCH_PARENT_DIR"
184+ 
185+# Function to extract target file path from patch
186+get_target_file_from_patch() {
187+ local patch_file="$1"
188+ # Extract the --- a/... line to find target file
189+ local target_line=$(grep -m1 "^--- a/" "$patch_file" 2>/dev/null || grep -m1 "^--- " "$patch_file" 2>/dev/null)
190+ if [ -n "$target_line" ]; then
191+ # Strip "--- a/" prefix and get the path
192+ # For -p1 from site-packages/, the path stays as torch/file.py
193+ local target_path=$(echo "$target_line" | sed 's/^--- a\///' | sed 's/^--- //')
194+ echo "$target_path"
195+ fi
196+}
197+ 
198+for patch_file in $PATCH_FILES; do
199+ # Get relative patch name for display
200+ patch_rel=$(realpath --relative-to="$SCRIPT_DIR" "$patch_file" 2>/dev/null || basename "$patch_file")
201+ 
202+ if $VERBOSE; then
203+ echo ""
204+ echo "Processing: $patch_rel"
205+ fi
206+ 
207+ # Extract and check target file
208+ target_file=$(get_target_file_from_patch "$patch_file")
209+ if [ -n "$target_file" ] && [ ! -f "$target_file" ]; then
210+ echo "[MISSING] $patch_rel - Target file not found: $target_file"
211+ MISSING_COUNT=$((MISSING_COUNT + 1))
212+ if $VERBOSE; then
213+ echo " Expected at: $TORCH_PARENT_DIR/$target_file"
214+ echo " Check if the file exists in torch package"
215+ fi
216+ continue
217+ fi
218+ 
219+ if $DRY_RUN; then
220+ # Dry run: check if patch can be applied
221+ if patch -p1 --dry-run --no-backup-if-mismatch -f < "$patch_file" > /dev/null 2>&1; then
222+ echo "[OK] $patch_rel (dry-run: can apply)"
223+ SUCCESS_COUNT=$((SUCCESS_COUNT + 1))
224+ else
225+ # Check if already applied
226+ if patch -p1 --dry-run --reverse --no-backup-if-mismatch -f < "$patch_file" > /dev/null 2>&1; then
227+ echo "[SKIP] $patch_rel (already applied)"
228+ SKIP_COUNT=$((SKIP_COUNT + 1))
229+ else
230+ echo "[FAIL] $patch_rel (dry-run: cannot apply)"
231+ FAIL_COUNT=$((FAIL_COUNT + 1))
232+ fi
233+ fi
234+ else
235+ # Actually apply the patch
236+ # Use --no-backup-if-mismatch to avoid creating .orig files
237+ # Use -f to force apply without prompts
238+ 
239+ # First check if already applied (reverse test)
240+ if patch -p1 --dry-run --reverse --no-backup-if-mismatch -f < "$patch_file" > /dev/null 2>&1; then
241+ echo "[SKIP] $patch_rel (already applied)"
242+ SKIP_COUNT=$((SKIP_COUNT + 1))
243+ continue
244+ fi
245+ 
246+ # Try to apply
247+ if patch -p1 --no-backup-if-mismatch -f < "$patch_file" > /tmp/torch_patch_output.log 2>&1; then
248+ echo "[OK] $patch_rel"
249+ SUCCESS_COUNT=$((SUCCESS_COUNT + 1))
250+ 
251+ if $VERBOSE; then
252+ cat /tmp/torch_patch_output.log
253+ fi
254+ else
255+ echo "[FAIL] $patch_rel"
256+ FAIL_COUNT=$((FAIL_COUNT + 1))
257+ 
258+ if $VERBOSE; then
259+ echo "--- Patch output ---"
260+ cat /tmp/torch_patch_output.log
261+ echo "--- End output ---"
262+ fi
263+ 
264+ # Continue with other patches instead of failing immediately
265+ # This allows partial application which may be useful for debugging
266+ fi
267+ fi
268+done
269+ 
270+# Summary
271+echo ""
272+echo "========================================"
273+echo "Patch Application Summary"
274+echo "========================================"
275+echo "Total patches: $PATCH_COUNT"
276+echo "Successfully: $SUCCESS_COUNT"
277+echo "Skipped (applied): $SKIP_COUNT"
278+echo "Missing targets: $MISSING_COUNT"
279+echo "Failed: $FAIL_COUNT"
280+echo ""
281+ 
282+if $DRY_RUN; then
283+ echo "(Dry run mode - no patches were actually applied)"
284+else
285+ if [ $FAIL_COUNT -gt 0 ] || [ $MISSING_COUNT -gt 0 ]; then
286+ echo "WARNING: Some patches failed to apply"
287+ echo "This may indicate version mismatch or missing files in torch package"
288+ exit 1
289+ else
290+ echo "All patches applied successfully!"
291+ fi
292+fi
293+ 
294+exit 0