update llm benchmarks-pytorch-AtomGit

Mbenchmarks/llm/baichuan2-7B-Chat/train_baichuan2_7B.py+3-0

	@@ -281,6 +281,9 @@ def main():
281	args = build_argparser().parse_args()	281	args = build_argparser().parse_args()
282	detect_device_type()	282	detect_device_type()
283	os.environ['TORCHINDUCTOR_NPU_BACKEND']=args.npu_backend	283	os.environ['TORCHINDUCTOR_NPU_BACKEND']=args.npu_backend
		284	+ if args.npu_backend == "akg":
		285	+ os.environ['TORCHINDUCTOR_NPU_BACKEND'] = 'mlir'
		286	+ os.environ['TORCHINDUCTOR_USE_AKG'] = '1'
284	if args.mfusion:	287	if args.mfusion:
285	os.environ['TORCHINDUCTOR_ENABLE_MFUSION']='1'	288	os.environ['TORCHINDUCTOR_ENABLE_MFUSION']='1'
286	os.makedirs(args.output_dir, exist_ok=True)	289	os.makedirs(args.output_dir, exist_ok=True)

Mbenchmarks/llm/gpt-oss-20B/train_gpt_oss_20B.py+3-0

	@@ -373,6 +373,9 @@ def main():
373		373
374	args.device_type = detect_device_type()	374	args.device_type = detect_device_type()
375	os.environ['TORCHINDUCTOR_NPU_BACKEND']=args.npu_backend	375	os.environ['TORCHINDUCTOR_NPU_BACKEND']=args.npu_backend
		376	+ if args.npu_backend == "akg":
		377	+ os.environ['TORCHINDUCTOR_NPU_BACKEND'] = 'mlir'
		378	+ os.environ['TORCHINDUCTOR_USE_AKG'] = '1'
376	if args.mfusion:	379	if args.mfusion:
377	os.environ['TORCHINDUCTOR_ENABLE_MFUSION']='1'	380	os.environ['TORCHINDUCTOR_ENABLE_MFUSION']='1'
378	print(f"{args.device_type} train {model_name}")	381	print(f"{args.device_type} train {model_name}")

Mbenchmarks/llm/mamba-codestral-7B/train_mamba2_7B.py+3-0

	@@ -88,6 +88,9 @@ def main():
88	args = parse_args()	88	args = parse_args()
89	device = detect_device_type()	89	device = detect_device_type()
90	os.environ['TORCHINDUCTOR_NPU_BACKEND']=args.npu_backend	90	os.environ['TORCHINDUCTOR_NPU_BACKEND']=args.npu_backend
		91	+ if args.npu_backend == "akg":
		92	+ os.environ['TORCHINDUCTOR_NPU_BACKEND'] = 'mlir'
		93	+ os.environ['TORCHINDUCTOR_USE_AKG'] = '1'
91	if args.mfusion:	94	if args.mfusion:
92	os.environ['TORCHINDUCTOR_ENABLE_MFUSION']='1'	95	os.environ['TORCHINDUCTOR_ENABLE_MFUSION']='1'
93	patch_remove_ops_from_generate_list(["aten.permute"])	96	patch_remove_ops_from_generate_list(["aten.permute"])

Abenchmarks/llm/qwen2vl-2B-Instruct/README.md+110-0

@@ -0,0 +1,110 @@
	1	+# qwen2vl-2B-Instruct 微调训练
	2	+
	3	+本 README 说明如何使用 qwen2vl-2B-Instruct 模型权重，结合 pixparse/cc12m-wds 提供的 `cc12images` 示例数据，完成数据下载、预处理与训练启动（含 eager / torch.compile 两种模式）。
	4	+
	5	+---
	6	+
	7	+## 目录
	8	+
	9	+- [1. 模型权重](#1-模型权重)
	10	+- [2. 数据获取和预处理](#2-数据获取和预处理)
	11	+- [3. 模型训练](#3-模型训练)
	12	+ - [3.1 eager mode（默认）](#31-eager-mode默认)
	13	+ - [3.2 启用 torchcompile（可选）](#32-启用-torchcompile可选)
	14	+ - [3.3 采集profile文件（可选）](#33-采集profile文件可选)
	15	+
	16	+---
	17	+
	18	+## 1. 模型权重
	19	+
	20	+- Hugging Face 模型：mistralai/Mamba-Codestral-7B-v0.1
	21	+ https://huggingface.co/mistralai/Mamba-Codestral-7B-v0.1
	22	+
	23	+- 可使用如下的自定义脚本下载
	24	+
	25	+```bash
	26	+python ../utils/download_hf.py --model Qwen/Qwen2-VL-2B-Instruct --save_path ./Qwen2-VL-2B-Instruct
	27	+```
	28	+
	29	+### 环境提示
	30	+
	31	+本项目依赖已整理到 `requirements.txt`，可直接安装：
	32	+
	33	+```bash
	34	+pip install -r ../utils/requirements.txt
	35	+```
	36	+
	37	+## 2. 数据获取和预处理
	38	+
	39	+```bash
	40	+pip install -U huggingface_hub webdataset pillow
	41	+python build_dataset.py \
	42	+ --pattern "cc12m-train-0000.tar" \
	43	+ --out-json train_data_0.json \
	44	+ --img-dir ./cc12m_dataset
	45	+```
	46	+
	47	+## 3. 模型训练
	48	+
	49	+训练脚本：`run_train.sh`
	50	+
	51	+开始训练前，请修改脚本中的路径参数：
	52	+
	53	+- 模型权重路径
	54	+- 训练数据路径
	55	+
	56	+### 3.1 eager mode（默认）
	57	+
	58	+```bash
	59	+bash run_train.sh
	60	+```
	61	+
	62	+### 3.2 启用 `torch.compile`（可选）
	63	+
	64	+可通过添加 `--enable_compile` 选项运行图模式
	65	+当在 GPU 上训练时，默认后端使用triton；当在 NPU 上训练时，可进一步指定后端为 mlir 或 dvm，默认使用 mlir。
	66	+
	67	+默认后端（mlir，可不写 --npu-backend）：
	68	+
	69	+```bash
	70	+bash run_train.sh \
	71	+ --enable_compile
	72	+```
	73	+
	74	+显式指定后端为 mlir：
	75	+
	76	+```bash
	77	+bash run_train.sh \
	78	+ --enable_compile \
	79	+ --npu-backend mlir
	80	+```
	81	+
	82	+切换后端为 dvm：
	83	+
	84	+```bash
	85	+bash run_train.sh \
	86	+ --enable_compile \
	87	+ --npu-backend dvm
	88	+```
	89	+
	90	+当在 NPU 上训练时，可通过 `--mfusion` 参数开启 MFusion 图算融合优化功能, 配合不同的NPU图模式后端, 进一步提升模型的性能，使用示例如下
	91	+
	92	+```bash
	93	+bash run_train.sh \
	94	+ --enable_compile \
	95	+ --npu-backend dvm \
	96	+ --mfusion
	97	+```
	98	+
	99	+### 3.3 采集profile文件（可选）
	100	+
	101	+脚本已支持 `--enable_profiler` 这样的开关，开启方式为：
	102	+
	103	+```bash
	104	+bash run_train.sh \
	105	+ --enable_profiler \
	106	+ --profiler_start_step 5 \
	107	+ --profiler_end_step 6 \
	108	+```
	109	+
	110	+可以通过 `--profiler_start_step` 和 `--profiler_end_step` 分别设置profile开始和结束步数。

Abenchmarks/llm/qwen2vl-2B-Instruct/build_dataset.py+167-0

@@ -0,0 +1,167 @@
	1	+#!/usr/bin/env python3
	2	+import os
	3	+import io
	4	+import json
	5	+import argparse
	6	+from pathlib import Path
	7	+from huggingface_hub import snapshot_download
	8	+import webdataset as wds
	9	+from PIL import Image
	10	+
	11	+
	12	+USER_PROMPT_DEFAULT = "请描述这张图片的内容。"
	13	+
	14	+
	15	+def ensure_dir(p: Path):
	16	+ p.mkdir(parents=True, exist_ok=True)
	17	+
	18	+
	19	+def pick_caption(sample: dict):
	20	+ for k in ("txt", "text", "caption", "captions", "description"):
	21	+ if k in sample and sample[k] is not None:
	22	+ v = sample[k]
	23	+ if isinstance(v, bytes):
	24	+ v = v.decode("utf-8", errors="ignore")
	25	+ if isinstance(v, str):
	26	+ v = v.strip()
	27	+ if v:
	28	+ return v
	29	+
	30	+ for k in ("json", "meta", "metadata"):
	31	+ if k in sample and sample[k] is not None:
	32	+ v = sample[k]
	33	+ if isinstance(v, bytes):
	34	+ v = v.decode("utf-8", errors="ignore")
	35	+ if isinstance(v, str):
	36	+ v = v.strip()
	37	+ if not v:
	38	+ continue
	39	+ try:
	40	+ obj = json.loads(v)
	41	+ for kk in ("caption", "text", "description"):
	42	+ if kk in obj and isinstance(obj[kk], str) and obj[kk].strip():
	43	+ return obj[kk].strip()
	44	+ except Exception:
	45	+ pass
	46	+
	47	+ return None
	48	+
	49	+
	50	+def save_image_bytes(img_bytes: bytes, out_path: Path):
	51	+ img = Image.open(io.BytesIO(img_bytes)).convert("RGB")
	52	+ img.save(out_path, format="JPEG", quality=95)
	53	+
	54	+
	55	+def main():
	56	+ ap = argparse.ArgumentParser()
	57	+ ap.add_argument("--repo", default="pixparse/cc12m-wds")
	58	+ ap.add_argument("--rev", default="main")
	59	+ ap.add_argument("--pattern", action="append", default=["cc12m-train-0000.tar"],
	60	+ help="Repeatable. e.g. --pattern 'cc12m-train-000*.tar'")
	61	+ ap.add_argument("--out-json", default="train_data_0.json")
	62	+ ap.add_argument("--cache-dir", default=None)
	63	+ ap.add_argument("--local-dir", default="./cc12m-wds-snapshot")
	64	+ ap.add_argument("--img-dir", default="./cc12m_dataset_images",
	65	+ help="Where to write extracted images. Set empty to skip saving images.")
	66	+ ap.add_argument("--max-samples", type=int, default=0,
	67	+ help="0 means no limit.")
	68	+ ap.add_argument("--user-prompt", default=USER_PROMPT_DEFAULT)
	69	+ ap.add_argument("--token", default=None, help="HF token if needed.")
	70	+ args = ap.parse_args()
	71	+
	72	+ snapshot_path = snapshot_download(
	73	+ repo_id=args.repo,
	74	+ repo_type="dataset",
	75	+ revision=args.rev,
	76	+ local_dir=args.local_dir,
	77	+ cache_dir=args.cache_dir,
	78	+ allow_patterns=args.pattern,
	79	+ token=args.token,
	80	+ )
	81	+
	82	+ local_dir = Path(args.local_dir)
	83	+ tar_paths = []
	84	+ for pat in args.pattern:
	85	+ tar_paths.extend(sorted(local_dir.glob(pat)))
	86	+ tar_paths = sorted(set(tar_paths))
	87	+
	88	+ if not tar_paths:
	89	+ raise FileNotFoundError(
	90	+ f"No files matching {args.pattern} found in {local_dir}."
	91	+ )
	92	+
	93	+ img_dir = None
	94	+ if args.img_dir and args.img_dir.strip():
	95	+ img_dir = Path(args.img_dir)
	96	+ ensure_dir(img_dir)
	97	+
	98	+ results = []
	99	+ idx = 0
	100	+
	101	+ dataset = wds.WebDataset([str(p) for p in tar_paths]).decode()
	102	+
	103	+ for sample in dataset:
	104	+ if args.max_samples and idx >= args.max_samples:
	105	+ break
	106	+
	107	+ img_bytes = None
	108	+ img_ext = None
	109	+ for k in ("jpg", "jpeg", "png", "webp"):
	110	+ if k in sample and sample[k] is not None:
	111	+ v = sample[k]
	112	+ if isinstance(v, bytes):
	113	+ img_bytes = v
	114	+ else:
	115	+ try:
	116	+ buf = io.BytesIO()
	117	+ v.convert("RGB").save(buf, format="JPEG", quality=95)
	118	+ img_bytes = buf.getvalue()
	119	+ except Exception:
	120	+ img_bytes = None
	121	+ img_ext = "jpg"
	122	+ break
	123	+
	124	+ if img_bytes is None:
	125	+ continue
	126	+
	127	+ caption = pick_caption(sample)
	128	+ if not caption:
	129	+ continue
	130	+
	131	+ key = sample.get("__key__", f"{idx:08d}")
	132	+ img_name = f"{key}.{img_ext}"
	133	+
	134	+ if img_dir is not None:
	135	+ out_img_path = img_dir / img_name
	136	+ if not out_img_path.exists():
	137	+ try:
	138	+ save_image_bytes(img_bytes, out_img_path)
	139	+ except Exception:
	140	+ continue
	141	+
	142	+ item = {
	143	+ "id": str(idx),
	144	+ "image": img_name if img_dir is not None else img_name,
	145	+ "conversations": [
	146	+ {"role": "user", "content": args.user_prompt},
	147	+ {"role": "assistant", "content": caption.strip()},
	148	+ ],
	149	+ }
	150	+ results.append(item)
	151	+ idx += 1
	152	+
	153	+ if idx % 1000 == 0:
	154	+ print(f"processed: {idx}")
	155	+
	156	+ with open(args.out_json, "w", encoding="utf-8") as f:
	157	+ json.dump(results, f, ensure_ascii=False, indent=2)
	158	+
	159	+ print(f"Done: {len(results)} items total, JSON: {args.out_json}")
	160	+ if img_dir is not None:
	161	+ print(f"Image save directory: {img_dir.resolve()}")
	162	+ print(f"Using shards: {', '.join([p.name for p in tar_paths])}")
	163	+ print(f"Snapshot path: {snapshot_path}")
	164	+
	165	+
	166	+if __name__ == "__main__":
	167	+ main()

Abenchmarks/llm/qwen2vl-2B-Instruct/run_train.sh+28-0

@@ -0,0 +1,28 @@
	1	+#!/bin/bash
	2	+export ASCEND_RT_VISIBLE_DEVICES=0
	3	+export CUDA_VISIBLE_DEVICES=0
	4	+export TORCHINDUCTOR_CACHE_DIR="./cache"
	5	+export TORCH_COMPILE_DEBUG=1
	6	+export TORCH_NPU_USE_COMPATIBLE_IMPL=1
	7	+
	8	+rm -rf ./cache/*
	9	+mkdir -p ./cache logs
	10	+
	11	+python train_qwen2vl.py \
	12	+ --model_name_or_path $MODEL_PATH \
	13	+ --train_data_path $DATA_PATH \
	14	+ --image_folder $IMAGE_PATH \
	15	+ --output_dir "./qwen2vl_output" \
	16	+ --num_train_epochs 3 \
	17	+ --max_steps 200 \
	18	+ --per_device_train_batch_size 1 \
	19	+ --gradient_accumulation_steps 1 \
	20	+ --learning_rate 2e-5 \
	21	+ --max_length 2048 \
	22	+ --use_lora false \
	23	+ --freeze_vision_tower true \
	24	+ --bf16 true \
	25	+ --data_repeat 1 \
	26	+ --min_pixels 200704 \
	27	+ --max_pixels 1003520 \
	28	+ > logs/train_qwen2vl.log 2>&1

Abenchmarks/llm/qwen2vl-2B-Instruct/train_qwen2vl.py+371-0

@@ -0,0 +1,371 @@
	1	+import os
	2	+import sys
	3	+from pathlib import Path
	4	+import json
	5	+import time
	6	+import argparse
	7	+import logging
	8	+import torch
	9	+import torch_npu
	10	+from typing import Dict, List, Any, Optional
	11	+from PIL import Image
	12	+from torch.utils.data import Dataset
	13	+from torch.profiler import (
	14	+ profile,
	15	+ ProfilerActivity,
	16	+ schedule,
	17	+ tensorboard_trace_handler,
	18	+)
	19	+
	20	+from transformers import (
	21	+ Qwen2VLForConditionalGeneration,
	22	+ AutoProcessor,
	23	+ TrainingArguments,
	24	+ Trainer,
	25	+ TrainerCallback,
	26	+)
	27	+from peft import LoraConfig, get_peft_model
	28	+sys.path.append(str(Path(__file__).parent.parent))
	29	+from utils.utils import TimingCallback, get_profile, detect_device_type
	30	+
	31	+
	32	+logging.basicConfig(level=logging.INFO)
	33	+logger = logging.getLogger(__name__)
	34	+
	35	+MODEL_NAME = "Qwen2VL-2B-instruct"
	36	+
	37	+def str2bool(v: str) -> bool:
	38	+ if isinstance(v, bool):
	39	+ return v
	40	+ if v.lower() in ("yes", "true", "t", "1"):
	41	+ return True
	42	+ if v.lower() in ("no", "false", "f", "0"):
	43	+ return False
	44	+ raise argparse.ArgumentTypeError(f"Unable to parse '{v}' as a boolean value")
	45	+
	46	+def patch_remove_ops_from_generate_list(op_names=None):
	47	+ try:
	48	+ import torch
	49	+ from torch_npu._inductor.ascend_npu_ir.ascend_npu_ir import config as anir_config
	50	+
	51	+ if not op_names:
	52	+ print("[patch] No op names provided, nothing to do.")
	53	+ return
	54	+
	55	+ for name in op_names:
	56	+ parts = name.split(".")
	57	+ op = torch.ops
	58	+ for p in parts:
	59	+ op = getattr(op, p)
	60	+
	61	+ if op in anir_config.GENERATE_LIST:
	62	+ anir_config.GENERATE_LIST.remove(op)
	63	+ print(f"[patch] Successfully removed {name} from GENERATE_LIST.")
	64	+ else:
	65	+ print(f"[patch] {name} not found in GENERATE_LIST (maybe already removed).")
	66	+
	67	+ except Exception as e:
	68	+ print(f"[patch] Failed to modify GENERATE_LIST: {e}")
	69	+
	70	+def parse_args():
	71	+ p = argparse.ArgumentParser(description="Qwen2-VL 2B Single GPU Training")
	72	+ p.add_argument("--model_name_or_path", type=str,
	73	+ default="/data/zyc/Qwen2-VL-2B-Instruct")
	74	+ p.add_argument("--use_lora", type=str2bool, default=False)
	75	+ p.add_argument("--lora_r", type=int, default=64)
	76	+ p.add_argument("--lora_alpha", type=int, default=16)
	77	+ p.add_argument("--lora_dropout", type=float, default=0.1)
	78	+ p.add_argument("--freeze_vision_tower", type=str2bool, default=True)
	79	+ p.add_argument("--enable_compile", action="store_true",help="Enable torch.compile")
	80	+ p.add_argument("--train_data_path", type=str, default="./train_data.json")
	81	+ p.add_argument("--image_folder", type=str, default="./images")
	82	+ p.add_argument("--max_length", type=int, default=2048)
	83	+ p.add_argument("--data_repeat", type=int, default=1,
	84	+ help="Number of data repetitions for small dataset augmentation")
	85	+ p.add_argument("--min_pixels", type=int, default=256 * 28 * 28,
	86	+ help="Qwen2-VL Processor min_pixels (controls minimum image resolution)")
	87	+ p.add_argument("--max_pixels", type=int, default=1280 * 28 * 28,
	88	+ help="Qwen2-VL Processor max_pixels (controls maximum image resolution)")
	89	+ p.add_argument("--output_dir", type=str, default="./qwen2vl_output")
	90	+ p.add_argument("--num_train_epochs", type=int, default=3)
	91	+ p.add_argument("--per_device_train_batch_size", type=int, default=1)
	92	+ p.add_argument("--gradient_accumulation_steps", type=int, default=1)
	93	+ p.add_argument("--learning_rate", type=float, default=2e-5)
	94	+ p.add_argument("--weight_decay", type=float, default=0.01)
	95	+ p.add_argument("--warmup_ratio", type=float, default=0.03)
	96	+ p.add_argument("--bf16", type=str2bool, default=False)
	97	+ p.add_argument("--logging_steps", type=int, default=1)
	98	+ p.add_argument("--max_steps", type=int, default=-1)
	99	+ p.add_argument("--save_steps", type=int, default=500)
	100	+ p.add_argument("--save_total_limit", type=int, default=3)
	101	+ p.add_argument("--dataloader_num_workers", type=int, default=0)
	102	+ p.add_argument("--enable_profiler", action="store_true", help="Enable profiler")
	103	+ p.add_argument("--profiler_save_path", type=str, default="./profile")
	104	+ p.add_argument("--profiler_start_step", type=int, default=5,
	105	+ help="Start step for profiling")
	106	+ p.add_argument("--profiler_end_step", type=int, default=6,
	107	+ help="End step for profiling")
	108	+ p.add_argument("--npu-backend", type=str, default="mlir")
	109	+ p.add_argument("--mfusion", action="store_true", help="Enable MFusion for graph fusion optimization")
	110	+
	111	+ return p.parse_args()
	112	+
	113	+class Qwen2VLDataset(Dataset):
	114	+ def __init__(
	115	+ self,
	116	+ data_path: str,
	117	+ processor,
	118	+ image_folder: str,
	119	+ max_length: int = 2048,
	120	+ repeat: int = 1,
	121	+ ):
	122	+ super().__init__()
	123	+ self.processor = processor
	124	+ self.image_folder = image_folder
	125	+ self.max_length = max_length
	126	+ self.repeat = repeat
	127	+
	128	+ with open(data_path, "r", encoding="utf-8") as f:
	129	+ self.data = json.load(f)
	130	+
	131	+ def __len__(self) -> int:
	132	+ return len(self.data) * self.repeat
	133	+
	134	+ def __getitem__(self, idx: int) -> Dict[str, Any]:
	135	+ real_idx = idx % len(self.data)
	136	+ item = self.data[real_idx]
	137	+
	138	+ image_path = os.path.join(self.image_folder, item["image"])
	139	+ image = Image.open(image_path).convert("RGB")
	140	+
	141	+ conversations = item["conversations"]
	142	+ messages: List[Dict] = []
	143	+ for i, conv in enumerate(conversations):
	144	+ if conv["role"] == "user":
	145	+ if i == 0:
	146	+ messages.append({
	147	+ "role": "user",
	148	+ "content": [
	149	+ {"type": "image", "image": image},
	150	+ {"type": "text", "text": conv["content"]},
	151	+ ],
	152	+ })
	153	+ else:
	154	+ messages.append({
	155	+ "role": "user",
	156	+ "content": [
	157	+ {"type": "text", "text": conv["content"]},
	158	+ ],
	159	+ })
	160	+ else:
	161	+ messages.append({
	162	+ "role": "assistant",
	163	+ "content": [
	164	+ {"type": "text", "text": conv["content"]},
	165	+ ],
	166	+ })
	167	+
	168	+ text = self.processor.apply_chat_template(
	169	+ messages,
	170	+ tokenize=False,
	171	+ add_generation_prompt=False,
	172	+ )
	173	+
	174	+ inputs = self.processor(
	175	+ text=[text],
	176	+ images=[image],
	177	+ padding="max_length",
	178	+ max_length=self.max_length,
	179	+ truncation=True,
	180	+ return_tensors="pt",
	181	+ )
	182	+
	183	+ result: Dict[str, Any] = {}
	184	+ for k, v in inputs.items():
	185	+ if k == "image_grid_thw":
	186	+ result[k] = v
	187	+ elif k == "pixel_values":
	188	+ result[k] = v.squeeze(0) if v.dim() > 4 else v
	189	+ else:
	190	+ result[k] = v.squeeze(0)
	191	+
	192	+ result["labels"] = result["input_ids"].clone()
	193	+ pad_token_id = self.processor.tokenizer.pad_token_id
	194	+ result["labels"][result["labels"] == pad_token_id] = -100
	195	+
	196	+ return result
	197	+
	198	+class Qwen2VLDataCollator:
	199	+
	200	+ def __init__(self, processor):
	201	+ self.processor = processor
	202	+
	203	+ def __call__(
	204	+ self, features: List[Dict[str, Any]]
	205	+ ) -> Dict[str, torch.Tensor]:
	206	+ batch: Dict[str, torch.Tensor] = {}
	207	+
	208	+ for key in ("input_ids", "attention_mask", "labels"):
	209	+ if key in features[0]:
	210	+ batch[key] = torch.stack([f[key] for f in features])
	211	+
	212	+ if "pixel_values" in features[0]:
	213	+ pv_list = []
	214	+ for f in features:
	215	+ pv = f["pixel_values"]
	216	+ if pv.dim() == 3:
	217	+ pv = pv.unsqueeze(0)
	218	+ pv_list.append(pv)
	219	+ batch["pixel_values"] = torch.cat(pv_list, dim=0)
	220	+
	221	+ if "image_grid_thw" in features[0]:
	222	+ thw_list = []
	223	+ for f in features:
	224	+ thw = f["image_grid_thw"]
	225	+ if thw.dim() == 1:
	226	+ thw = thw.unsqueeze(0)
	227	+ thw_list.append(thw)
	228	+ batch["image_grid_thw"] = torch.cat(thw_list, dim=0)
	229	+
	230	+ return batch
	231	+
	232	+def main():
	233	+ torch.use_deterministic_algorithms(True)
	234	+ args = parse_args()
	235	+ detect_device_type()
	236	+ os.environ['TORCHINDUCTOR_NPU_BACKEND']=args.npu_backend
	237	+ if args.npu_backend == "akg":
	238	+ os.environ['TORCHINDUCTOR_NPU_BACKEND'] = 'mlir'
	239	+ os.environ['TORCHINDUCTOR_USE_AKG'] = '1'
	240	+ if args.mfusion:
	241	+ os.environ['TORCHINDUCTOR_ENABLE_MFUSION']='1'
	242	+ patch_remove_ops_from_generate_list(["aten.clone", "aten.permute"])
	243	+
	244	+ print("\nLoading Processor...")
	245	+ t0 = time.time()
	246	+ processor = AutoProcessor.from_pretrained(
	247	+ args.model_name_or_path,
	248	+ min_pixels=args.min_pixels,
	249	+ max_pixels=args.max_pixels,
	250	+ trust_remote_code=True,
	251	+ )
	252	+ print(f"Processor loading time: {time.time() - t0:.2f}s")
	253	+ print(f" min_pixels = {args.min_pixels}, max_pixels = {args.max_pixels}")
	254	+
	255	+ if processor.tokenizer.pad_token is None:
	256	+ processor.tokenizer.pad_token = processor.tokenizer.eos_token
	257	+
	258	+ print("\nLoading Qwen2-VL model...")
	259	+ t0 = time.time()
	260	+ model = Qwen2VLForConditionalGeneration.from_pretrained(
	261	+ args.model_name_or_path,
	262	+ torch_dtype=torch.bfloat16,
	263	+ trust_remote_code=True,
	264	+ )
	265	+ print(f"Model loading time: {time.time() - t0:.2f}s")
	266	+
	267	+ model.gradient_checkpointing_enable()
	268	+
	269	+ if args.freeze_vision_tower:
	270	+ print("Freezing vision encoder (parameters containing 'visual')...")
	271	+ frozen_count = 0
	272	+ for name, param in model.named_parameters():
	273	+ if "visual" in name:
	274	+ param.requires_grad = False
	275	+ frozen_count += 1
	276	+ print(f" Frozen {frozen_count} vision parameters")
	277	+
	278	+ if args.enable_compile:
	279	+ torch.autograd.set_detect_anomaly(True)
	280	+ print("Enabling torch.compile (dynamic=False) ...")
	281	+ model = torch.compile(model, dynamic=False)
	282	+
	283	+ if args.use_lora:
	284	+ print("\nConfiguring LoRA...")
	285	+ t0 = time.time()
	286	+ lora_target_modules = [
	287	+ "q_proj", "k_proj", "v_proj", "o_proj",
	288	+ "gate_proj", "up_proj", "down_proj",
	289	+ ]
	290	+ lora_config = LoraConfig(
	291	+ r=args.lora_r,
	292	+ lora_alpha=args.lora_alpha,
	293	+ target_modules=lora_target_modules,
	294	+ lora_dropout=args.lora_dropout,
	295	+ bias="none",
	296	+ task_type="CAUSAL_LM",
	297	+ )
	298	+ model = get_peft_model(model, lora_config)
	299	+ model.print_trainable_parameters()
	300	+ print(f"LoRA configuration time: {time.time() - t0:.2f}s")
	301	+
	302	+ print("\nPreparing dataset...")
	303	+ t0 = time.time()
	304	+
	305	+ if not os.path.exists(args.train_data_path) or os.path.getsize(args.train_data_path) == 0:
	306	+ print("Training data not found or empty, creating dummy data...")
	307	+ create_dummy_data(args.train_data_path, args.image_folder)
	308	+
	309	+ train_dataset = Qwen2VLDataset(
	310	+ data_path=args.train_data_path,
	311	+ processor=processor,
	312	+ image_folder=args.image_folder,
	313	+ max_length=args.max_length,
	314	+ repeat=args.data_repeat,
	315	+ )
	316	+ print(f"Dataset preparation time: {time.time() - t0:.2f}s")
	317	+
	318	+ data_collator = Qwen2VLDataCollator(processor=processor)
	319	+
	320	+ training_args = TrainingArguments(
	321	+ output_dir=args.output_dir,
	322	+ num_train_epochs=args.num_train_epochs,
	323	+ per_device_train_batch_size=args.per_device_train_batch_size,
	324	+ gradient_accumulation_steps=args.gradient_accumulation_steps,
	325	+ learning_rate=args.learning_rate,
	326	+ weight_decay=args.weight_decay,
	327	+ warmup_ratio=args.warmup_ratio,
	328	+ lr_scheduler_type="cosine",
	329	+ logging_dir="./logs",
	330	+ max_steps=args.max_steps,
	331	+ logging_steps=args.logging_steps,
	332	+ save_steps=args.save_steps,
	333	+ save_total_limit=args.save_total_limit,
	334	+ bf16=args.bf16,
	335	+ gradient_checkpointing=True,
	336	+ dataloader_num_workers=args.dataloader_num_workers,
	337	+ remove_unused_columns=False,
	338	+ report_to="none",
	339	+ ddp_find_unused_parameters=False,
	340	+ )
	341	+
	342	+ mode = "compile" if args.enable_compile else "eager"
	343	+ prof=None
	344	+
	345	+ if args.enable_profiler:
	346	+ profiling_save_path = args.profiler_save_path + '/' + MODEL_NAME + '/' + mode
	347	+ prof = get_profile(args.profiler_start_step, args.profiler_end_step, profiling_save_path)
	348	+
	349	+ timing_callback = TimingCallback(prof, mode)
	350	+
	351	+ trainer = Trainer(
	352	+ model=model,
	353	+ args=training_args,
	354	+ train_dataset=train_dataset,
	355	+ data_collator=data_collator,
	356	+ callbacks=[timing_callback],
	357	+ )
	358	+ trainer.train()
	359	+ if args.enable_compile:
	360	+ headers, values = torch._dynamo.utils.compile_times("csv")
	361	+ for header, value in zip(headers, values):
	362	+ if header == "PyCodeCache.load_by_key_path":
	363	+ numbers = [float(num.strip()) for num in value.split(',') if num.strip()]
	364	+ op_compile_time = sum(numbers)
	365	+ print(f"op_compile_time:{op_compile_time * 1e3} ms", )
	366	+ final_output_dir = os.path.join(args.output_dir, "final")
	367	+ trainer.save_model(final_output_dir)
	368	+ processor.save_pretrained(final_output_dir)
	369	+
	370	+if __name__ == "__main__":
	371	+ main()

Abenchmarks/llm/sd3/README.md+114-0

@@ -0,0 +1,114 @@
	1	+# SDXL 微调训练
	2	+
	3	+本 README 说明如何使用 SD3 模型权重，完成数据下载、预处理与训练启动（含 eager / torch.compile 两种模式）。
	4	+
	5	+---
	6	+
	7	+## 目录
	8	+
	9	+- [SDXL 微调训练](#sdxl-微调训练)
	10	+ - [目录](#目录)
	11	+ - [1. 模型权重](#1-模型权重)
	12	+ - [环境提示](#环境提示)
	13	+ - [2. 数据获取](#2-数据获取)
	14	+ - [3. 模型训练](#3-模型训练)
	15	+ - [3.1 eager mode（默认）](#31-eager-mode默认)
	16	+ - [3.2 启用 `torch.compile`（可选）](#32-启用-torchcompile可选)
	17	+
	18	+---
	19	+
	20	+## 1. 模型权重
	21	+
	22	+- Hugging Face 模型：stabilityai/stable-diffusion-3-medium-diffusers
	23	+ https://huggingface.co/stabilityai/stable-diffusion-3-medium-diffusers
	24	+
	25	+- 可使用如下的自定义脚本下载
	26	+
	27	+```bash
	28	+python ../utils/download_hf.py --model stabilityai/stable-diffusion-3-medium-diffusers --save_path ./stable-diffusion-3-medium-diffusers
	29	+```
	30	+
	31	+### 环境提示
	32	+
	33	+本项目依赖已整理到 `requirements.txt`，可直接安装：
	34	+
	35	+```bash
	36	+pip install -r ../utils/requirements.txt
	37	+```
	38	+
	39	+## 2. 数据获取
	40	+
	41	+训练数据集链接如下：
	42	+
	43	+https://huggingface.co/datasets/AdamLucek/oldbookillustrations-small/tree/main/data
	44	+
	45	+- 可使用以下命令获取
	46	+
	47	+```bash
	48	+wget https://huggingface.co/datasets/AdamLucek/oldbookillustrations-small/resolve/main/data/train-00000-of-00001.parquet
	49	+```
	50	+
	51	+## 3. 模型训练
	52	+
	53	+训练脚本：`run_sd3.sh`，脚本支持在 GPU 和 NPU 训练
	54	+
	55	+开始训练前，请修改脚本中的路径参数：
	56	+
	57	+- 模型权重路径
	58	+- 训练数据路径
	59	+
	60	+### 3.1 eager mode（默认）
	61	+
	62	+```bash
	63	+bash run_sd3.sh
	64	+```
	65	+
	66	+### 3.2 启用 `torch.compile`（可选）
	67	+
	68	+可通过添加 `--enable_compile` 选项运行图模式
	69	+当在 GPU 上训练时，默认后端使用triton；当在 NPU 上训练时，可进一步指定后端为 mlir 或 dvm，默认使用 mlir。
	70	+
	71	+默认后端（mlir，可不写 --npu-backend）：
	72	+
	73	+```bash
	74	+bash run_sd3.sh \
	75	+ --enable_compile
	76	+```
	77	+
	78	+显式指定后端为 mlir：
	79	+
	80	+```bash
	81	+bash run_sd3.sh \
	82	+ --enable_compile \
	83	+ --npu-backend mlir
	84	+```
	85	+
	86	+切换后端为 dvm：
	87	+
	88	+```bash
	89	+bash run_sd3.sh \
	90	+ --enable_compile \
	91	+ --npu-backend dvm
	92	+```
	93	+
	94	+当在 NPU 上训练时，可通过 `--mfusion` 参数开启 MFusion 图算融合优化功能, 配合不同的NPU图模式后端, 进一步提升模型的性能，使用示例如下
	95	+
	96	+```bash
	97	+bash run_sd3.sh \
	98	+ --enable_compile \
	99	+ --npu-backend dvm \
	100	+ --mfusion
	101	+```
	102	+
	103	+### 3.3 采集profile文件（可选）
	104	+
	105	+脚本已支持 `--enable_profiler` 这样的开关，开启方式为：
	106	+
	107	+```bash
	108	+bash run_sd3.sh \
	109	+ --enable_profiler \
	110	+ --profiler_start_step 5 \
	111	+ --profiler_end_step 6 \
	112	+```
	113	+
	114	+可以通过 `--profiler_start_step` 和 `--profiler_end_step` 分别设置profile开始和结束步数。

Abenchmarks/llm/sd3/run_sd3.sh+20-0

@@ -0,0 +1,20 @@
	1	+#!/bin/bash
	2	+export ASCEND_RT_VISIBLE_DEVICES=0
	3	+export CUDA_VISIBLE_DEVICES=0
	4	+export TORCHINDUCTOR_CACHE_DIR="./cache"
	5	+export TORCH_COMPILE_DEBUG=1
	6	+export TORCH_NPU_USE_COMPATIBLE_IMPL=1
	7	+
	8	+rm -rf ./cache/*
	9	+mkdir -p ./cache logs
	10	+
	11	+python train_sd3.py \
	12	+ --model_path $MODEL_PATH \
	13	+ --data_path $DATA_PATH \
	14	+ --output_dir "./sd3-finetuned" \
	15	+ --num_epochs 1 \
	16	+ --max_steps 200 \
	17	+ --batch_size 1 \
	18	+ --use_bf16 \
	19	+ --dataloader_num_workers 8 \
	20	+ > logs/train_sd3.log 2>&1

Abenchmarks/llm/sd3/train_sd3.py+273-0

@@ -0,0 +1,273 @@
	1	+import os
	2	+import sys
	3	+import torch
	4	+import torch.nn.functional as F
	5	+from pathlib import Path
	6	+from diffusers import StableDiffusion3Pipeline
	7	+from peft import LoraConfig, get_peft_model_state_dict
	8	+from datasets import load_dataset
	9	+from torchvision import transforms
	10	+import random
	11	+import numpy as np
	12	+import argparse
	13	+import logging
	14	+import torch_npu
	15	+from transformers import Trainer, TrainingArguments
	16	+sys.path.append(str(Path(__file__).parent.parent))
	17	+from utils.utils import (
	18	+ TimingCallback,
	19	+ get_profile,
	20	+ detect_device_type
	21	+)
	22	+
	23	+model_name = 'sd3-medium'
	24	+
	25	+logging.basicConfig(level=logging.INFO)
	26	+logger = logging.getLogger(__name__)
	27	+
	28	+
	29	+def set_seed(seed=42):
	30	+ random.seed(seed)
	31	+ np.random.seed(seed)
	32	+ torch.manual_seed(seed)
	33	+ torch.npu.manual_seed_all(seed)
	34	+ os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
	35	+ torch.use_deterministic_algorithms(True, warn_only=True)
	36	+ torch.backends.cudnn.deterministic = True
	37	+ torch.backends.cudnn.benchmark = False
	38	+
	39	+
	40	+def custom_collate_fn(examples):
	41	+ pixel_values = torch.stack([example["pixel_values"] for example in examples])
	42	+ texts = [example["text"] for example in examples]
	43	+ return {"pixel_values": pixel_values, "text": texts}
	44	+
	45	+
	46	+class SD3Trainer(Trainer):
	47	+ def __init__(self, pipe, args, *kwargs):
	48	+ super().__init__(args, *kwargs)
	49	+ self.pipe = pipe
	50	+
	51	+ def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None, **kwargs):
	52	+ device = model.device
	53	+ model_dtype = model.dtype
	54	+
	55	+ pixel_values = inputs["pixel_values"].to(device, dtype=model_dtype)
	56	+ prompts = inputs["text"]
	57	+
	58	+ with torch.no_grad():
	59	+ prompt_embeds, _, pooled_prompt_embeds, _ = self.pipe.encode_prompt(
	60	+ prompt=prompts,
	61	+ prompt_2=prompts,
	62	+ prompt_3=prompts,
	63	+ device=device,
	64	+ num_images_per_prompt=1,
	65	+ do_classifier_free_guidance=False
	66	+ )
	67	+
	68	+ with torch.no_grad():
	69	+ latents = self.pipe.vae.encode(pixel_values).latent_dist.mode()
	70	+ latents = (latents - self.pipe.vae.config.shift_factor) * self.pipe.vae.config.scaling_factor
	71	+
	72	+ noise = torch.randn_like(latents)
	73	+ bsz = latents.shape[0]
	74	+
	75	+ sigmas = torch.rand((bsz,), device=device, dtype=model_dtype)
	76	+ sigmas_reshaped = sigmas.view(-1, 1, 1, 1)
	77	+
	78	+ noisy_latents = (1.0 - sigmas_reshaped) * latents + sigmas_reshaped * noise
	79	+ target = noise - latents
	80	+
	81	+ timesteps = sigmas * 1000
	82	+
	83	+ model_pred = model(
	84	+ hidden_states=noisy_latents,
	85	+ timestep=timesteps,
	86	+ encoder_hidden_states=prompt_embeds,
	87	+ pooled_projections=pooled_prompt_embeds,
	88	+ return_dict=False
	89	+ )[0]
	90	+
	91	+ loss = F.mse_loss(model_pred.float(), target.float(), reduction="mean")
	92	+
	93	+ return (loss, None) if return_outputs else loss
	94	+
	95	+
	96	+class SD3LoRAFineTuner:
	97	+ def __init__(self, args):
	98	+ self.args = args
	99	+ self.device = detect_device_type() if hasattr(sys.modules[__name__], 'detect_device_type') else "cuda"
	100	+ self.dtype = torch.bfloat16 if self.args.use_bf16 else torch.float16
	101	+
	102	+ set_seed(self.args.seed)
	103	+ self.setup_model()
	104	+
	105	+ def setup_model(self):
	106	+ logger.info(f"Loading SD3 model from {self.args.model_path}")
	107	+
	108	+ self.pipe = StableDiffusion3Pipeline.from_pretrained(
	109	+ self.args.model_path,
	110	+ torch_dtype=self.dtype,
	111	+ local_files_only=True
	112	+ ).to(self.device)
	113	+
	114	+ self.transformer = self.pipe.transformer
	115	+ self.vae = self.pipe.vae
	116	+ self.text_encoder = self.pipe.text_encoder
	117	+ self.text_encoder_2 = self.pipe.text_encoder_2
	118	+ self.text_encoder_3 = self.pipe.text_encoder_3
	119	+
	120	+ self.vae.requires_grad_(False)
	121	+ self.text_encoder.requires_grad_(False)
	122	+ self.text_encoder_2.requires_grad_(False)
	123	+ self.text_encoder_3.requires_grad_(False)
	124	+ self.transformer.requires_grad_(False)
	125	+
	126	+ logger.info("Injecting LoRA adapters...")
	127	+ transformer_lora_config = LoraConfig(
	128	+ r=self.args.lora_rank,
	129	+ lora_alpha=self.args.lora_rank,
	130	+ init_lora_weights="gaussian",
	131	+ target_modules=["to_k", "to_q", "to_v", "to_out.0"],
	132	+ )
	133	+
	134	+ self.transformer.add_adapter(transformer_lora_config)
	135	+
	136	+ params_to_optimize = list(filter(lambda p: p.requires_grad, self.transformer.parameters()))
	137	+ logger.info(f"Trainable params: {sum(p.numel() for p in params_to_optimize)}")
	138	+
	139	+ if self.args.enable_compile:
	140	+ logger.info("Compiling transformer model with torch.compile...")
	141	+ self.transformer = torch.compile(self.transformer)
	142	+
	143	+ def load_and_preprocess_data(self):
	144	+ logger.info(f"Loading dataset from {self.args.data_path}")
	145	+ dataset = load_dataset("parquet", data_files=self.args.data_path)["train"]
	146	+
	147	+ transform = transforms.Compose([
	148	+ transforms.Resize((self.args.resolution, self.args.resolution)),
	149	+ transforms.ToTensor(),
	150	+ transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
	151	+ ])
	152	+
	153	+ def preprocess_transforms(examples):
	154	+ pixel_values = []
	155	+ for image in examples["image"]:
	156	+ if image.mode != "RGB":
	157	+ image = image.convert("RGB")
	158	+ pixel_values.append(transform(image))
	159	+
	160	+ examples["pixel_values"] = pixel_values
	161	+ return examples
	162	+
	163	+ dataset.set_transform(preprocess_transforms)
	164	+ return dataset
	165	+
	166	+ def train(self):
	167	+ logger.info("Starting SD3 LoRA training...")
	168	+ dataset = self.load_and_preprocess_data()
	169	+
	170	+ callbacks = []
	171	+ prof=None
	172	+ mod = 'compile' if self.args.enable_compile else 'eager'
	173	+ if self.args.enable_profiler:
	174	+ if not os.path.exists(self.args.profiler_save_path):
	175	+ os.makedirs(self.args.profiler_save_path)
	176	+
	177	+ profiling_save_path = os.path.join(self.args.profiler_save_path, 'sd3', mod)
	178	+
	179	+ prof = get_profile(
	180	+ profiler_start_step=self.args.profiler_start_step,
	181	+ profiler_end_step=self.args.profiler_end_step,
	182	+ profiling_save_path=profiling_save_path
	183	+ )
	184	+
	185	+ callbacks.append(TimingCallback(profiler=prof, mod=mod))
	186	+
	187	+ training_args = TrainingArguments(
	188	+ output_dir=self.args.output_dir,
	189	+ max_steps=self.args.max_steps,
	190	+ num_train_epochs=self.args.num_epochs,
	191	+ per_device_train_batch_size=self.args.batch_size,
	192	+ learning_rate=self.args.learning_rate,
	193	+ logging_steps=1,
	194	+ save_strategy="no",
	195	+ remove_unused_columns=False,
	196	+ report_to="none",
	197	+ dataloader_num_workers=self.args.dataloader_num_workers,
	198	+ bf16=self.args.use_bf16,
	199	+ fp16=not self.args.use_bf16
	200	+ )
	201	+
	202	+ trainer = SD3Trainer(
	203	+ pipe=self.pipe,
	204	+ model=self.transformer,
	205	+ args=training_args,
	206	+ train_dataset=dataset,
	207	+ data_collator=custom_collate_fn,
	208	+ callbacks=callbacks
	209	+ )
	210	+
	211	+ train_result = trainer.train()
	212	+
	213	+ logger.info(f"Saving LoRA weights to {self.args.output_dir}")
	214	+ transformer_lora_state_dict = get_peft_model_state_dict(self.transformer)
	215	+ self.pipe.save_lora_weights(
	216	+ save_directory=self.args.output_dir,
	217	+ transformer_lora_layers=transformer_lora_state_dict,
	218	+ safe_serialization=True
	219	+ )
	220	+
	221	+ return train_result.metrics
	222	+
	223	+
	224	+def main():
	225	+ parser = argparse.ArgumentParser(description="Train Stable Diffusion 3 LoRA")
	226	+ parser.add_argument("--model_path", type=str, required=True, help="Path to the pretrained SD3 model")
	227	+ parser.add_argument("--data_path", type=str, required=True, help="Path to training data (parquet file)")
	228	+ parser.add_argument("--output_dir", type=str, default="./sd3_lora_weights", help="Output directory for LoRA weights")
	229	+ parser.add_argument("--num_epochs", type=int, default=1, help="Number of training epochs")
	230	+ parser.add_argument("--max_steps", type=int, default=10, help="Total training steps (-1 means use num_epochs)")
	231	+ parser.add_argument("--batch_size", type=int, default=1, help="Batch size per device")
	232	+ parser.add_argument("--learning_rate", type=float, default=1e-4, help="Learning rate")
	233	+ parser.add_argument("--lora_rank", type=int, default=8, help="LoRA rank")
	234	+ parser.add_argument("--resolution", type=int, default=1024, help="Image resolution for training")
	235	+ parser.add_argument("--seed", type=int, default=42, help="Random seed")
	236	+ parser.add_argument("--dataloader_num_workers", type=int, default=0, help="Dataloader workers")
	237	+ parser.add_argument("--use_bf16", action="store_true", help="Use BF16 precision (recommended for SD3)")
	238	+ parser.add_argument("--enable_compile", action="store_true", help="Enable torch.compile")
	239	+ parser.add_argument("--enable_profiler", action="store_true", help="Enable PyTorch profiler")
	240	+ parser.add_argument("--profiler_start_step", type=int, default=10, help="Step to start profiler")
	241	+ parser.add_argument("--profiler_end_step", type=int, default=14, help="Step to end profiler")
	242	+ parser.add_argument("--profiler_save_path", type=str, default="./profile", help="Path to save profiler logs")
	243	+ parser.add_argument("--report_to_tensorboard", action="store_true", help="Report to TensorBoard")
	244	+ parser.add_argument("--npu-backend", type=str, default="mlir")
	245	+ parser.add_argument("--mfusion", action="store_true", help="Enable MFusion for graph fusion optimization")
	246	+
	247	+ args = parser.parse_args()
	248	+ device_type = detect_device_type()
	249	+ os.environ['TORCHINDUCTOR_NPU_BACKEND'] = args.npu_backend
	250	+ if args.npu_backend == "akg":
	251	+ os.environ['TORCHINDUCTOR_NPU_BACKEND'] = 'mlir'
	252	+ os.environ['TORCHINDUCTOR_USE_AKG'] = '1'
	253	+ if args.mfusion:
	254	+ os.environ['TORCHINDUCTOR_ENABLE_MFUSION'] = '1'
	255	+ finer_tuner = SD3LoRAFineTuner(args)
	256	+ metrics = finer_tuner.train()
	257	+ if args.enable_compile:
	258	+ headers, values = torch._dynamo.utils.compile_times("csv")
	259	+ for header, value in zip(headers, values):
	260	+ if header == "PyCodeCache.load_by_key_path":
	261	+ numbers = [float(num.strip()) for num in value.split(',') if num.strip()]
	262	+ op_compile_time = sum(numbers)
	263	+ print(f"op_compile_time:{op_compile_time * 1e3} ms", )
	264	+
	265	+ print("\n" + "="*50)
	266	+ print("SD3 LoRA Training completed successfully!")
	267	+ print(f"LoRA weights saved to: {args.output_dir}")
	268	+ print(f"Final training loss: {metrics.get('train_loss', 'N/A')}")
	269	+ print("="*50)
	270	+
	271	+
	272	+if __name__ == "__main__":
	273	+ main()

Mbenchmarks/llm/sdxl-base-1.0/train_sdxl_base.py+3-0

	@@ -252,6 +252,9 @@ def main():
252	args = parser.parse_args()	252	args = parser.parse_args()
253	device_type = detect_device_type()	253	device_type = detect_device_type()
254	os.environ['TORCHINDUCTOR_NPU_BACKEND'] = args.npu_backend	254	os.environ['TORCHINDUCTOR_NPU_BACKEND'] = args.npu_backend
		255	+ if args.npu_backend == "akg":
		256	+ os.environ['TORCHINDUCTOR_NPU_BACKEND'] = 'mlir'
		257	+ os.environ['TORCHINDUCTOR_USE_AKG'] = '1'
255	if args.mfusion:	258	if args.mfusion:
256	os.environ['TORCHINDUCTOR_ENABLE_MFUSION']='1'	259	os.environ['TORCHINDUCTOR_ENABLE_MFUSION']='1'
257	finer_tuner = SDXLLoRAFineTuner(args)	260	finer_tuner = SDXLLoRAFineTuner(args)

Mbenchmarks/llm/utils/requirements.txt+3-1

	@@ -12,4 +12,6 @@ numpy==1.26.4
12	protobuf==7.34.1	12	protobuf==7.34.1
13	huggingface-hub==0.36.2	13	huggingface-hub==0.36.2
14	diffusers==0.37.0	14	diffusers==0.37.0
15	-packaging==26.2	15	+packaging==26.2
		16	+decorator==5.1.1
		17	+scipy==1.13.1