已合并
Revert "[Feature]: align ra_compress package layout with attention_mse (impl.py + __init__.py re-export)" #865
tanxiangyuu创建于 12 天前
Revert "[Feature]: align ra_compress package layout with attention_mse (impl.py + __init__.py re-export)" #865
已合并
共 28 个文件变更+129-2180
| @@ -12,7 +12,9 @@ repos: | |||
| 12 | rev: v4.6.0 | 12 | rev: v4.6.0 |
| 13 | hooks: | 13 | hooks: |
| 14 | - id: trailing-whitespace | 14 | - id: trailing-whitespace |
| 15 | + exclude: ^msmodelslim/cli/analysis/__main__\.py|msmodelslim/processor/analysis/methods_base\.py|msmodelslim/app/analysis/result_displayer_infra\.py|msmodelslim/infra/analysis_pipeline_loader\.py$ | ||
| 15 | - id: end-of-file-fixer | 16 | - id: end-of-file-fixer |
| 17 | + exclude: ^msmodelslim/infra/logging_analysis_result_displayer\.py|msmodelslim/processor/analysis/methods_base\.py|msmodelslim/app/analysis/result_displayer_infra\.py$ | ||
| 16 | - id: check-yaml | 18 | - id: check-yaml |
| 17 | args: ["--allow-multiple-documents", "--unsafe"] | 19 | args: ["--allow-multiple-documents", "--unsafe"] |
| 18 | - id: check-added-large-files | 20 | - id: check-added-large-files |
| @@ -28,9 +30,11 @@ repos: | |||
| 28 | - id: ruff-check | 30 | - id: ruff-check |
| 29 | args: ["--config", "pre-commit/pyproject.toml", "--output-format", "github", "--fix"] | 31 | args: ["--config", "pre-commit/pyproject.toml", "--output-format", "github", "--fix"] |
| 30 | types: [python] | 32 | types: [python] |
| 33 | + exclude: ^msmodelslim/cli/analysis/__main__\.py|msmodelslim/processor/analysis/methods_base\.py|msmodelslim/app/analysis/application\.py|msmodelslim/app/analysis/result_displayer_infra\.py|msmodelslim/infra/analysis_pipeline_loader\.py|msmodelslim/infra/logging_analysis_result_displayer\.py$ | ||
| 31 | - id: ruff-format | 34 | - id: ruff-format |
| 32 | args: ["--config", "pre-commit/pyproject.toml"] | 35 | args: ["--config", "pre-commit/pyproject.toml"] |
| 33 | types: [python] | 36 | types: [python] |
| 37 | + exclude: ^msmodelslim/cli/analysis/__main__\.py|msmodelslim/processor/analysis/methods_base\.py|msmodelslim/app/analysis/application\.py|msmodelslim/app/analysis/result_displayer_infra\.py|msmodelslim/infra/analysis_pipeline_loader\.py|msmodelslim/infra/logging_analysis_result_displayer\.py$ | ||
| 34 | 38 | ||
| 35 | # codespell | 39 | # codespell |
| 36 | - repo: https://gitcode.com/gh_mirrors/co/codespell | 40 | - repo: https://gitcode.com/gh_mirrors/co/codespell |
| @@ -43,7 +47,7 @@ repos: | |||
| 43 | "--skip", | 47 | "--skip", |
| 44 | "*.py,*.cpp,*.hpp,*.c,*.h,pre-commit/typos.toml", | 48 | "*.py,*.cpp,*.hpp,*.c,*.h,pre-commit/typos.toml", |
| 45 | ] | 49 | ] |
| 46 | - exclude: ^pre-commit/|typos.toml|^lab_calib/calib_dummy\.jsonl$ | 50 | + exclude: ^pre-commit/|typos.toml |
| 47 | 51 | ||
| 48 | # pylint：指定 pre-commit/pyproject.toml | 52 | # pylint：指定 pre-commit/pyproject.toml |
| 49 | - repo: https://gitcode.com/gh_mirrors/pyl/pylint | 53 | - repo: https://gitcode.com/gh_mirrors/pyl/pylint |
| @@ -54,7 +58,7 @@ repos: | |||
| 54 | types: [python] | 58 | types: [python] |
| 55 | args: ["--rcfile=pre-commit/pyproject.toml"] | 59 | args: ["--rcfile=pre-commit/pyproject.toml"] |
| 56 | verbose: false | 60 | verbose: false |
| 57 | - exclude: msmodelslim/model/deepseek_v4/model.py | 61 | + exclude: ^msmodelslim/model/deepseek_v4/model\.py|msmodelslim/cli/analysis/__main__\.py|msmodelslim/processor/analysis/methods_base\.py|msmodelslim/app/analysis/application\.py|msmodelslim/infra/logging_analysis_result_displayer\.py|test/cases/processor/analysis/test_unary_analysis_processor\.py$ |
| 58 | 62 | ||
| 59 | # Bandit：指定 pre-commit/pyproject.toml | 63 | # Bandit：指定 pre-commit/pyproject.toml |
| 60 | - repo: https://gitcode.com/gh_mirrors/ba/bandit | 64 | - repo: https://gitcode.com/gh_mirrors/ba/bandit |
| @@ -1,204 +0,0 @@ | |||
| 1 | -# RA Compress 长序列压缩算法词条 | ||
| 2 | - | ||
| 3 | -> **词条类别**：敏感层分析算法（长序列 KV cache 压缩） | ||
| 4 | -> **英文名称**：RazorAttention Compress | ||
| 5 | -> **英文缩写**：ra_compress | ||
| 6 | -> **应用领域**：长序列推理、KV cache 压缩、注意力头筛选 | ||
| 7 | -> **msModelSlim 实现**：`msmodelslim/processor/analysis/unary_operator/metrics/ra_compress/` | ||
| 8 | - | ||
| 9 | ---- | ||
| 10 | - | ||
| 11 | -## 1. 概述 | ||
| 12 | - | ||
| 13 | -RA Compress（`ra_compress`）是 `msmodelslim analyze` 中 `attn_head` 范围分析的一种度量算法。它基于 Transformer 中注意力头的跨段（segment-spanning）注意力行为，识别具有 prefix matching（归纳头）和 copying matching（回声头）能力的关键 KV head，用于后续 KV cache 压缩配置。其核心特征是：依赖合成的重复段校准数据、分析 Q@K^T 注意力分数在段间偏移位置的聚合统计、按固定 ratio 选择 top heads、输出 `head.pt` 文件供后续压缩流程使用。 | ||
| 14 | - | ||
| 15 | ---- | ||
| 16 | - | ||
| 17 | -## 2. 词条介绍 | ||
| 18 | - | ||
| 19 | -长文本（长序列）推理时，KV cache 随序列长度线性增长，是端侧和大模型推理时显存的主要瓶颈。RA Compress 观察到：并非所有 KV head 对长上下文质量的贡献都是均等的——具有强归纳（induction）和复制（copying）能力的注意力头是跨段信息传递的关键锚点，在 KV cache 压缩时应优先保留。RA Compress 通过构造**等长重复段**（`DUMMY_INPUT_LENGTH=2500` × `REPET_TIMES=4`，总计 ≥ 10000 tokens）的合成校准数据，在段间特定偏移位置度量注意力强度，从而为每层、每个 head 计算 prefix matching / copying matching 得分，再按预设 ratio 选 top heads 作为压缩保留目标。 | ||
| 20 | - | ||
| 21 | ---- | ||
| 22 | - | ||
| 23 | -## 3. 原理 | ||
| 24 | - | ||
| 25 | -### 3.1 核心思想 | ||
| 26 | - | ||
| 27 | -RA Compress 的核心思想是“利用重复段结构定位跨段关键头”：构造 4 段完全相同的 2500 tokens 重复序列（共 10000 tokens）。在段边界处，具有归纳能力的头会在当前段的 prefix 位置关注前一段对应位置 + 1（prefix matching，`offset = -SEG + 1`），而具有复制能力的头会在当前段位置关注前一段对应位置（copying matching，`offset = -SEG`）。对每一层，在重复的段对上取该偏移位置的注意力分数作逐段平均，得到每个 head 的 prefix / copying 得分；得分 top ratio 的 head 入选为 KV cache 压缩时需要保留的 KV head 索引。 | ||
| 28 | - | ||
| 29 | -### 3.2 数学描述 | ||
| 30 | - | ||
| 31 | -设 $A \in \mathbb{R}^{L \times L}$ 为某一层在重复段校准数据上的注意力矩阵（对 Q 与 K 的 out product 做 softmax 后）。对所有满足 $i \in [(k+1) \cdot SEG, (k+2) \cdot SEG]$ 的位置 $i$（第 $k+1$ 段），分别考察以下两个偏移量位置： | ||
| 32 | - | ||
| 33 | -**Prefix matching 得分（归纳头）**： | ||
| 34 | -$$ | ||
| 35 | -\text{prefix\_score}_h = \text{mean}_{k, i} A_h[i, i - k \cdot SEG + 1] | ||
| 36 | -$$ | ||
| 37 | -（在第 $k+1$ 段的位置 $i$ 上，注意力落在第 $k$ 段的 $i - SEG + 1$ 处的概率，即“前一段 prefix 位置 + 1”，对应归纳头的典型行为。） | ||
| 38 | - | ||
| 39 | -**Copying matching 得分（回声头）**： | ||
| 40 | -$$ | ||
| 41 | -\text{copying\_score}_h = \text{mean}_{k, i} A_h[i, i - (k+1) \cdot SEG] | ||
| 42 | -$$ | ||
| 43 | -（在第 $k+1$ 段的位置 $i$ 上，注意力落在第 $k$ 段相同偏移 $i - SEG$ 处的概率，对应“复制前一段对应位置”的回声头行为。） | ||
| 44 | - | ||
| 45 | -其中 $SEG = \text{DUMMY\_INPUT\_LENGTH} = 2500$，$k \in \{0, 1, 2\}$（共 3 个段对），head $h$ 为 GQA 分组前的 attention head。 | ||
| 46 | - | ||
| 47 | -**GQA 分组合并**：若模型使用 GQA / MQA（`num_kv_heads < num_attention_heads`），则在每个 KV head 对应的 query head 组内取最大得分作为该 KV head 的代表分： | ||
| 48 | -$$ | ||
| 49 | -\text{grouped\_score}[g] = \max_{h \in \text{group}[g]} \text{score}_h | ||
| 50 | -$$ | ||
| 51 | - | ||
| 52 | -**Top ratio 选择**：对所有层、所有 KV head 的得分整体排序，取前 `induction_head_ratio`（默认 14%）的 KV head 作为 prefix matching 入选（induction heads），取前 `echo_head_ratio`（默认 1%）的 KV head 作为 copying matching 入选（echo heads）。 | ||
| 53 | - | ||
| 54 | -### 3.3 关键性质 | ||
| 55 | - | ||
| 56 | -- **attn_head 粒度**：输出 KV head 粒度的入选列表（每层若干 head 索引），而非层粒度排序。 | ||
| 57 | -- **数据依赖强**：要求校准集 tokenize 后总长度 ≥ 10000 tokens，否则段位置偏移无意义、得分被置空。 | ||
| 58 | -- **合成段数据优先**：默认使用工具内置的 `calib_dummy.jsonl`，该文件经 tokenizer 后严格对齐到 2500×N 段边界。 | ||
| 59 | -- **适配器接口可选依赖**：默认匹配 `q_proj` / `k_proj` / `qkv_proj` 命名，若模型命名不同，需在适配器实现 `RaCompressAnalysisInterface`。 | ||
| 60 | -- **结果用于压缩而非回退**：输出为 `head.pt`（dict 序列化的 `.pt` 文件），供 RA Compress KV cache 压缩流程读取，而非用于量化 YAML 的 exclude / include 配置。 | ||
| 61 | - | ||
| 62 | ---- | ||
| 63 | - | ||
| 64 | -## 4. 流程示意 | ||
| 65 | - | ||
| 66 | -> 以下为本算法在 msModelSlim 中的简化流程概览。 | ||
| 67 | - | ||
| 68 | -```mermaid | ||
| 69 | -flowchart LR | ||
| 70 | - A[calib_dummy.jsonl 合成校准集] --> B[Q_proj / K_proj hook 采集输出] | ||
| 71 | - B --> C[逐段计算 Q @ K^T] | ||
| 72 | - C --> D[段间偏移位置取注意力] | ||
| 73 | - D --> E[逐段平均得到 head 得分] | ||
| 74 | - E --> F[GQA 分组取 max] | ||
| 75 | - F --> G[按 ratio 选 top heads] | ||
| 76 | - G --> H[写入 layer_scores (enrich)] | ||
| 77 | - H --> I[保存 head.pt] | ||
| 78 | -``` | ||
| 79 | - | ||
| 80 | ---- | ||
| 81 | - | ||
| 82 | -## 5. 在 msModelSlim 中的实现 | ||
| 83 | - | ||
| 84 | -### 5.1 实现位置 | ||
| 85 | - | ||
| 86 | -RA Compress 作为 `msmodelslim analyze` 命令的 `attn_head` 范围分析指标实现，位于 `msmodelslim/processor/analysis/unary_operator/metrics/ra_compress/`。 | ||
| 87 | - | ||
| 88 | -模块结构： | ||
| 89 | - | ||
| 90 | -- `interface.py`：定义 `RaCompressAnalysisInterface`，在模型适配器侧提供 Q / K / QKV 投影层名称模式。 | ||
| 91 | -- `__init__.py`：`RaCompressAnalysisMethod` 主实现（hook、得分计算、分组、head 选择、layer_scores enrich）。 | ||
| 92 | - | ||
| 93 | -### 5.2 处理流程 | ||
| 94 | - | ||
| 95 | -通过 `msmodelslim analyze attn_head --metrics ra_compress` 命令执行（`attn_head` 默认 metrics）： | ||
| 96 | - | ||
| 97 | -1. 在 Q / K（或 QKV 融合）投影层上注册 forward hook，在 `outputs` 位置保存该层的输出张量。 | ||
| 98 | -2. 每层计算得分时，若为 Q 层则取出该层对应 K 层的输出（反之亦然），计算 `Q @ K^T` 并按 head 拆分。 | ||
| 99 | -3. 对每个 head 的注意力矩阵，分别计算段间 `offset = -SEG + 1`（prefix）与 `offset = -SEG`（copying）位置的逐段平均得分。 | ||
| 100 | -4. 经过 `_max_every_group`（GQA 分组 max）后，按 `_select_top_heads` 取 14% / 1% top heads。 | ||
| 101 | -5. `enrich_layer_scores` 将入选的 `induction_heads` / `echo_heads`（KV head 索引列表）挂到 `layer_scores` 条目里。 | ||
| 102 | -6. 结果展示器遍历 `layer_scores` 重建 head_dict，打印入选列表并保存 `head.pt`（结构为 `{"prefix_matching": {layer_idx: [kv_head_idx, ...]}, "copying": {...}}`）。 | ||
| 103 | - | ||
| 104 | -### 5.3 命令行示例 | ||
| 105 | - | ||
| 106 | -```bash | ||
| 107 | -msmodelslim analyze attn_head \ | ||
| 108 | - --model_type Qwen2.5-7B-Instruct \ | ||
| 109 | - --model_path ${model_path} \ | ||
| 110 | - --metrics ra_compress \ | ||
| 111 | - --device npu \ | ||
| 112 | - --trust_remote_code True \ | ||
| 113 | - --save_path ./head_result | ||
| 114 | -``` | ||
| 115 | - | ||
| 116 | -### 5.4 模型适配接口 | ||
| 117 | - | ||
| 118 | -RA Compress 分析需要定位 Transformer 自注意力的 Q / K（或 QKV 融合）投影层以挂载 hook 采集输出。不同模型的投影层命名可能不一致，因此通过模型适配器接口 `RaCompressAnalysisInterface` 暴露名称模式，由分析方法据此匹配目标层。当适配器未实现该接口时，方法回退到默认名称（`q_proj` / `k_proj` / `qkv_proj`），仅适用于遵循该命名约定的模型。 | ||
| 119 | - | ||
| 120 | -**接口定义**：[`RaCompressAnalysisInterface`](../../../../../msmodelslim/processor/analysis/unary_operator/metrics/ra_compress/interface.py) | ||
| 121 | - | ||
| 122 | -**接口方法**： | ||
| 123 | - | ||
| 124 | -| 方法 | 返回类型 | 说明 | | ||
| 125 | -| --- | --- | --- | | ||
| 126 | -| `get_ra_compress_proj_patterns()` | `Dict[str, str]` | 返回 Q / K / QKV 投影层名称模式字典 | | ||
| 127 | - | ||
| 128 | -返回值格式： | ||
| 129 | - | ||
| 130 | -```python | ||
| 131 | -{ | ||
| 132 | - "q": "q_proj", # Q 投影层名称模式 | ||
| 133 | - "k": "k_proj", # K 投影层名称模式 | ||
| 134 | - "qkv": "qkv_proj", # QKV 融合投影层名称模式（无融合时留空字符串） | ||
| 135 | -} | ||
| 136 | -``` | ||
| 137 | - | ||
| 138 | -**名称匹配规则**（见 [`RaCompressAnalysisMethod._is_target_layer`](../../../../../msmodelslim/processor/analysis/unary_operator/metrics/ra_compress/__init__.py)）：分析方法对 `model.named_modules()` 中每个 `nn.Linear`，判断其层名是否包含上述任一模式字符串，命中即作为目标层挂载 hook。 | ||
| 139 | - | ||
| 140 | -**默认回退**：适配器未实现 `RaCompressAnalysisInterface` 时，使用以下默认模式（见 [`__init__.py`](../../../../../msmodelslim/processor/analysis/unary_operator/metrics/ra_compress/__init__.py) 顶部常量）： | ||
| 141 | - | ||
| 142 | -| 键 | 默认值 | 常量 | | ||
| 143 | -| --- | --- | --- | | ||
| 144 | -| `q` | `q_proj` | `_DEFAULT_Q_NAME_PATTERN` | | ||
| 145 | -| `k` | `k_proj` | `_DEFAULT_K_NAME_PATTERN` | | ||
| 146 | -| `qkv` | `qkv_proj` | `_DEFAULT_QKV_NAME_PATTERN` | | ||
| 147 | - | ||
| 148 | -**适配器实现示例**： | ||
| 149 | - | ||
| 150 | -当模型投影层命名与默认值不一致时（例如使用 `query_proj` / `key_proj`），在模型适配器中实现接口并覆盖名称模式： | ||
| 151 | - | ||
| 152 | -```python | ||
| 153 | -from msmodelslim.processor.analysis.unary_operator.metrics.ra_compress.interface import ( | ||
| 154 | - RaCompressAnalysisInterface, | ||
| 155 | -) | ||
| 156 | - | ||
| 157 | -class XxxAdapter(RaCompressAnalysisInterface): | ||
| 158 | - def get_ra_compress_proj_patterns(self): | ||
| 159 | - return { | ||
| 160 | - "q": "query_proj", | ||
| 161 | - "k": "key_proj", | ||
| 162 | - "qkv": "", # 该模型无 QKV 融合层 | ||
| 163 | - } | ||
| 164 | -``` | ||
| 165 | - | ||
| 166 | ---- | ||
| 167 | - | ||
| 168 | -## 6. 适用场景与限制 | ||
| 169 | - | ||
| 170 | -### 6.1 适用场景 | ||
| 171 | - | ||
| 172 | -- 需要为 KV cache 压缩（RA Compress）方案生成 `head.pt` 的场景。 | ||
| 173 | -- 长序列推理场景下，需要了解模型中哪些 KV head 承担了跨段信息传递（归纳 / 复制头）。 | ||
| 174 | -- 使用重复段合成校准数据的注意力头能力分析。 | ||
| 175 | - | ||
| 176 | -### 6.2 使用限制 | ||
| 177 | - | ||
| 178 | -- 校准集 tokenize 后总长度必须 ≥ `DUMMY_INPUT_LENGTH * REPET_TIMES`（2500 × 4 = 10000 tokens），不足时得分被置空并告警。 | ||
| 179 | -- 目前仅分析 Transformer 自注意力的 Q / K 投影层（或 QKV 融合层）；其他结构不在范围内。 | ||
| 180 | -- 结果不输出敏感度 Score 排序，只输出入选的 head 索引；ratio 目前使用默认常量（14% / 1%），不通过 YAML 配置。 | ||
| 181 | -- `model_type` 支持范围参见《[大模型支持矩阵](../../model/README.md)》及适配器对 `RaCompressAnalysisInterface` 的实现情况。 | ||
| 182 | - | ||
| 183 | ---- | ||
| 184 | - | ||
| 185 | -## 7. 关联流程 | ||
| 186 | - | ||
| 187 | -- 《[Attention Head 筛选分析使用指南](../../../user_guide/usage_sensitive_attn_head_analysis.md)》：本算法作为 `attn_head` 分析的 metrics 使用。 | ||
| 188 | -- 《[一键量化 (V1)](../../../user_guide/usage_quick_quantization.md)》：分析产出的 `head.pt` 可在 RA Compress 相关量化流程中使用。 | ||
| 189 | - | ||
| 190 | ---- | ||
| 191 | - | ||
| 192 | -## 8. 关联词条 | ||
| 193 | - | ||
| 194 | -- [KVCache Quant](../kvcache_quant/term_kvcache_quant.md)：配套术语，KV cache 的量化方案，与 RA Compress 同属长序列显存优化范畴。 | ||
| 195 | -- [KV Smooth](../kv_smooth/term_kv_smooth.md)：配套术语，针对 KV cache 的离群值抑制算法。 | ||
| 196 | -- [Attention MSE](../attention_mse/term_attention_mse.md)：同类术语，同为 attn 范围敏感层分析指标（但用于 FA 回退决策而非 head 选择）。 | ||
| 197 | -- [MSE Layer Wise](../mse_layer_wise/term_mse_layer_wise.md)：配套术语，层级敏感层分析指标。 | ||
| 198 | - | ||
| 199 | ---- | ||
| 200 | - | ||
| 201 | -## 9. 参考资料 | ||
| 202 | - | ||
| 203 | -1. 《RA Compress 使用指南》([./usage_ra_compress.md](./usage_ra_compress.md)) | ||
| 204 | -2. 论文：*RazorAttention: Efficient KV Cache Compression Through Retrieval Heads*（induction / copying head 相关背景） | ||
| @@ -1,134 +0,0 @@ | |||
| 1 | -# RA Compress 使用指南 | ||
| 2 | - | ||
| 3 | -## 1. 适用范围 | ||
| 4 | - | ||
| 5 | -本流程适用于在 msModelSlim 中使用 RA Compress（`ra_compress`）长序列压缩算法。RA Compress 作为 `msmodelslim analyze attn_head` 的 metrics 指标，用于 KV 注意力头粒度的筛选，输出归纳头（induction heads）与回声头（echo heads）的入选索引，供后续 KV cache 压缩流程使用。 | ||
| 6 | - | ||
| 7 | -适用角色：需要进行算法构建以及模型部署的开发者 | ||
| 8 | - | ||
| 9 | -适用场景： | ||
| 10 | - | ||
| 11 | -- 需要为长序列推理生成 KV cache 压缩配置（`head.pt`）的场景。 | ||
| 12 | -- 需要了解模型中哪些 KV head 承担跨段信息传递（归纳 / 复制头）的场景。 | ||
| 13 | -- 使用重复段合成校准数据进行注意力头能力分析。 | ||
| 14 | - | ||
| 15 | -不适用场景： | ||
| 16 | - | ||
| 17 | -- 校准集 tokenize 后总长度不足 10000 tokens（段边界错位，得分被置空）。 | ||
| 18 | -- 目标 `model_type` 的适配器未实现 `RaCompressAnalysisInterface` 且 Q/K 投影层命名与默认模式不匹配。 | ||
| 19 | - | ||
| 20 | -## 2. 流程关系与前置条件 | ||
| 21 | - | ||
| 22 | -**上级流程**：模型适配与验证通过后，执行长序列 KV cache 压缩前的注意力头筛选阶段。 | ||
| 23 | - | ||
| 24 | -**前置条件**： | ||
| 25 | - | ||
| 26 | -- 已安装兼容版本的 msModelSlim 工具（详见《[msModelSlim 工具安装指南](../../../install_guide/install_guide.md)》）。 | ||
| 27 | -- 已确认校准集 tokenize 后总长度 ≥ 2500 × 4 = 10000 tokens（默认使用 `calib_dummy.jsonl` 即可满足）。 | ||
| 28 | -- 若目标模型 Q/K 投影层命名非 `q_proj` / `k_proj` / `qkv_proj`，需在适配器中实现 `RaCompressAnalysisInterface`。 | ||
| 29 | - | ||
| 30 | -**后续操作**：将产出的 `head.pt` 送入 RA Compress KV cache 压缩量化流程，生成压缩后的模型。 | ||
| 31 | - | ||
| 32 | -## 3. 输入和交付件 | ||
| 33 | - | ||
| 34 | -| 类型 | 名称 | 来源或保存位置 | 格式或约束 | 验收方式 | | ||
| 35 | -| --- | --- | --- | --- | --- | | ||
| 36 | -| 输入 | 浮点模型权重目录 | 模型下载或本地路径 | HuggingFace 格式，含 `config.json` 及 `*.safetensors` | 可被目标 Transformers 版本加载 | | ||
| 37 | -| 输入 | 校准数据集 | 工具默认 `calib_dummy.jsonl` 或用户指定 | JSONL 格式，tokenize 后长度 ≥ 10000 | 段边界对齐，得分非空 | | ||
| 38 | -| 交付件 | `head.pt` | `--save_path` 指定目录 | dict 序列化 `.pt` 文件，含 `prefix_matching` 与 `copying` 字段 | 文件存在且可被 torch.load 读取 | | ||
| 39 | -| 交付件 | 入选 head 列表 | 命令行输出 | 每层 KV head 索引列表 | 可读且包含入选 head | | ||
| 40 | - | ||
| 41 | -## 4. 流程总览 | ||
| 42 | - | ||
| 43 | -```mermaid | ||
| 44 | -flowchart LR | ||
| 45 | - A[准备模型与 calib_dummy.jsonl] --> B[执行 analyze attn_head 命令] | ||
| 46 | - B --> C[Q@K^T 段间偏移注意力聚合] | ||
| 47 | - C --> D[按 ratio 选 induction/echo heads] | ||
| 48 | - D --> E[输出 head.pt 与列表] | ||
| 49 | -``` | ||
| 50 | - | ||
| 51 | -## 5. 操作步骤 | ||
| 52 | - | ||
| 53 | -### 步骤 1：确认适配器与校准数据 | ||
| 54 | - | ||
| 55 | -**目标**：确认目标 `model_type` 能正确采集 Q/K 投影层输出，且校准数据长度满足段对齐要求。 | ||
| 56 | - | ||
| 57 | -**操作**： | ||
| 58 | - | ||
| 59 | -1. 检查 Q/K 投影层命名：默认匹配 `q_proj`、`k_proj`、`qkv_proj`，若命名不一致，需在模型适配器中实现 `RaCompressAnalysisInterface.get_ra_compress_proj_patterns()`，返回 `{"q": "...", "k": "...", "qkv": "..."}`。 | ||
| 60 | -2. 使用默认校准集 `calib_dummy.jsonl`（推荐）；若使用自定义校准集，需保证 tokenize 后总长度 ≥ 2500 × 4 = 10000 tokens，且段边界严格对齐。 | ||
| 61 | - | ||
| 62 | -接口约定详见《[RA Compress 词条](./term_ra_compress.md)》。 | ||
| 63 | - | ||
| 64 | -**输出**：适配器命名模式与校准数据确认。 | ||
| 65 | - | ||
| 66 | -### 步骤 2：执行注意力头筛选命令 | ||
| 67 | - | ||
| 68 | -**目标**：使用 `ra_compress` 指标完成 KV head 粒度筛选，生成 `head.pt`。 | ||
| 69 | - | ||
| 70 | -**操作**： | ||
| 71 | - | ||
| 72 | -```bash | ||
| 73 | -msmodelslim analyze attn_head \ | ||
| 74 | - --model_type Qwen2.5-7B-Instruct \ | ||
| 75 | - --model_path ${model_path} \ | ||
| 76 | - --metrics ra_compress \ | ||
| 77 | - --device npu \ | ||
| 78 | - --trust_remote_code True \ | ||
| 79 | - --save_path ./head_result | ||
| 80 | -``` | ||
| 81 | - | ||
| 82 | -参数说明： | ||
| 83 | - | ||
| 84 | -| 参数 | 说明 | | ||
| 85 | -| --- | --- | | ||
| 86 | -| `attn_head` | KV 注意力头粒度分析（`ra_compress` 为默认 metrics） | | ||
| 87 | -| `--metrics` | 指定分析算法，取值为 `ra_compress` 时使用本算法 | | ||
| 88 | -| `--save_path` | `head.pt` 与结果文件保存目录 | | ||
| 89 | -| `--calib_dataset` | 校准数据集路径（默认 `calib_dummy.jsonl`，通常无需修改） | | ||
| 90 | - | ||
| 91 | -完整参数见《[Attention Head 筛选分析使用指南](../../../user_guide/usage_sensitive_attn_head_analysis.md)》。 | ||
| 92 | - | ||
| 93 | -**输出**：命令行打印入选的 induction heads / echo heads 列表；`--save_path` 目录下生成 `head.pt`。 | ||
| 94 | - | ||
| 95 | -### 步骤 3：解读结果并用于压缩 | ||
| 96 | - | ||
| 97 | -**目标**：将 `head.pt` 送入 RA Compress KV cache 压缩量化流程。 | ||
| 98 | - | ||
| 99 | -**操作**： | ||
| 100 | - | ||
| 101 | -1. 确认 `head.pt` 结构为 `{"prefix_matching": {layer_idx: [kv_head_idx, ...]}, "copying": {layer_idx: [kv_head_idx, ...]}}`。 | ||
| 102 | -2. 在后续量化流程中，将该文件路径配置到 RA Compress 压缩相关参数中。 | ||
| 103 | -3. 执行量化并验证长序列推理精度与显存占用。 | ||
| 104 | - | ||
| 105 | -**输出**：RA Compress 压缩配置接入完成。 | ||
| 106 | - | ||
| 107 | -## 6. 验收条件 | ||
| 108 | - | ||
| 109 | -- 分析命令执行成功，`head.pt` 已生成在 `--save_path` 目录下。 | ||
| 110 | -- 命令行输出中 induction heads / echo heads 列表非空（若为空，请检查校准数据长度）。 | ||
| 111 | -- `head.pt` 可被后续 RA Compress 压缩量化流程正确读取。 | ||
| 112 | - | ||
| 113 | -## 7. 异常处置 | ||
| 114 | - | ||
| 115 | -- **得分或 head 列表为空**：校准集 tokenize 后总长度不足 10000 tokens。换用默认 `calib_dummy.jsonl`，或扩充自定义校准集至 ≥ 2500 × 4 tokens。 | ||
| 116 | -- **未命中 Q/K 投影层**：目标模型 Q/K 命名与默认模式不匹配。在适配器中实现 `RaCompressAnalysisInterface.get_ra_compress_proj_patterns()`，返回正确的模块名模式。 | ||
| 117 | -- **GQA 分组相关报错**：确认模型 `config.json` 中 `num_attention_heads` 与 `num_key_value_heads` 字段正确，工具会自动按组取 max。 | ||
| 118 | - | ||
| 119 | -## 8. 术语 | ||
| 120 | - | ||
| 121 | -| 术语 | 简述 | 链接 | | ||
| 122 | -| --- | --- | --- | | ||
| 123 | -| RA Compress | 基于重复段结构的 KV head 筛选算法，用于长序列 KV cache 压缩 | [RA Compress 词条](./term_ra_compress.md) | | ||
| 124 | -| attn_head 范围分析 | KV 注意力头粒度的筛选分析 | [Attention Head 筛选分析使用指南](../../../user_guide/usage_sensitive_attn_head_analysis.md) | | ||
| 125 | -| Induction Head | 归纳头，在段边界处体现 prefix matching 行为的关键 KV head | [RA Compress 词条](./term_ra_compress.md) | | ||
| 126 | -| Echo Head | 回声头，在段边界处体现 copying matching 行为的关键 KV head | [RA Compress 词条](./term_ra_compress.md) | | ||
| 127 | -| RaCompressAnalysisInterface | 模型适配器可选实现的 Q/K 投影层命名接口 | [RA Compress 词条](./term_ra_compress.md) | | ||
| 128 | - | ||
| 129 | -## 9. 接口文档列表 | ||
| 130 | - | ||
| 131 | -| 接口或能力 | 简述 | 链接 | | ||
| 132 | -| --- | --- | --- | | ||
| 133 | -| `msmodelslim analyze attn_head` | KV 注意力头粒度筛选命令，`--metrics ra_compress` 启用本算法 | [RA Compress 词条](./term_ra_compress.md) | | ||
| 134 | -| `RaCompressAnalysisInterface` | 模型适配器可选实现的 Q/K 投影层命名接口 | [RA Compress 词条](./term_ra_compress.md) | | ||
| @@ -1,237 +0,0 @@ | |||
| 1 | -# Attention Head 筛选分析使用指南 | ||
| 2 | - | ||
| 3 | -## 1. 适用范围 | ||
| 4 | - | ||
| 5 | -本指南面向需要通过 `msmodelslim analyze attn_head`，按 **Attention Head** 粒度识别关键注意力头（induction heads / echo heads），并据此生成 `head.pt` 文件用于 RA Compress 压缩的开发者与算法工程师。 | ||
| 6 | - | ||
| 7 | -**适用场景**： | ||
| 8 | - | ||
| 9 | -- RA Compress 压缩方案设计：识别具有 prefix matching（归纳头）和 copying matching（回声头）能力的 KV cache 头，用于后续 KV cache 压缩配置。 | ||
| 10 | -- 精度不达标时，结合 head 筛选结果迭代 RA Compress 相关配置。 | ||
| 11 | - | ||
| 12 | -**不适用场景**： | ||
| 13 | - | ||
| 14 | -- 多模态理解 / 多模态生成模型：当前分析仅支持大语言模型。 | ||
| 15 | -- 需要按单层线性层回退或提位宽：请参见《[线性层敏感层分析使用指南](usage_sensitive_linear_analysis.md)》。 | ||
| 16 | -- 需要按 Decoder 块或整块 Attention / MLP / MoE 回退：请参见《[层级敏感层分析使用指南](usage_sensitive_layer_wise_analysis.md)》。 | ||
| 17 | -- 配合 FA 量化识别需回退的 attention 模块：请参见《[Attention 敏感层分析使用指南](usage_sensitive_attn_analysis.md)》。 | ||
| 18 | - | ||
| 19 | -## 2. 流程关系与前置条件 | ||
| 20 | - | ||
| 21 | -**上级流程**：《[新模型量化调优流程](process_new_model_quantization_tuning.md)》。 | ||
| 22 | - | ||
| 23 | -**前置条件**： | ||
| 24 | - | ||
| 25 | -- 已安装 msModelSlim（详见《[安装指南](../install_guide/install_guide.md)》）。 | ||
| 26 | -- 目标模型为 LLM，且已确定可用的 `--model_type`（与支持矩阵 / 适配器注册名一致，大小写敏感；通常已在上级流程或权重量化流程中完成适配）。 | ||
| 27 | -- 已具备可用的昇腾 NPU（或仅做小规模调试时使用 `--device cpu`）。 | ||
| 28 | -- 目标适配器已实现 `RaCompressAnalysisInterface` 接口（提供 Q / K / QKV 投影层名称模式），或使用默认模式（`q_proj` / `k_proj` / `qkv_proj`）。 | ||
| 29 | - | ||
| 30 | -**后续操作**： | ||
| 31 | - | ||
| 32 | -- 将生成的 `head.pt` 文件提供给 RA Compress 量化流程，或在量化配置中引用 head 筛选结果。 | ||
| 33 | - | ||
| 34 | -## 3. 输入和交付件 | ||
| 35 | - | ||
| 36 | -| 类型 | 名称 | 来源或保存位置 | 格式或约束 | 验收方式 | | ||
| 37 | -| --- | --- | --- | --- | --- | | ||
| 38 | -| 输入 | 浮点模型目录 | 用户本地路径；一般自 ModelScope / Hugging Face 获取 | 含模型配置、权重分片及 tokenizer 等 | 路径有效，可被目标 `model_type` 加载 | | ||
| 39 | -| 输入 | 校准集 | 工具内置 [`calib_dummy.jsonl`](../../../lab_calib/) 短名称或用户本地路径 | `.json` / `.jsonl`；tokenize 后总长度须 ≥ 10000（2500 × 4 段重复） | 可被 `msmodelslim analyze` 解析且 token 长度满足要求 | | ||
| 40 | -| 交付件 | head 筛选结果 | 标准输出（induction / echo head 列表）+ 可选 `head.pt` 文件 | `head.pt` 为 Python dict 序列化的 `.pt` 文件 | 能识别预期 layer / head 范围，`head.pt` 可被后续量化流程加载 | | ||
| 41 | - | ||
| 42 | -## 4. 流程总览 | ||
| 43 | - | ||
| 44 | -确认 `attn_head` 场景与 `ra_compress` 指标后，准备浮点权重与专用校准集，按需完成模型适配（含 RA Compress 接口），再执行分析命令并解读结果，用于生成 `head.pt` 或调整压缩配置。 | ||
| 45 | - | ||
| 46 | -```mermaid | ||
| 47 | -flowchart LR | ||
| 48 | - scene[确认指标] --> weight[获取浮点权重] | ||
| 49 | - weight --> adapt[完成模型适配] | ||
| 50 | - weight -->|已接入且接口齐全可跳过| run[执行分析命令] | ||
| 51 | - adapt --> run | ||
| 52 | - run --> result[解读分析结果] | ||
| 53 | -``` | ||
| 54 | - | ||
| 55 | -## 5. 操作步骤 | ||
| 56 | - | ||
| 57 | -### 命令行预览 | ||
| 58 | - | ||
| 59 | -```bash | ||
| 60 | -msmodelslim analyze attn_head \ | ||
| 61 | - --model_type ${MODEL_TYPE} \ # 已注册或支持矩阵中的模型名，大小写敏感 | ||
| 62 | - --model_path ${MODEL_PATH} \ # 浮点权重目录 | ||
| 63 | - --metrics ra_compress \ # 分析指标，固定为 ra_compress | ||
| 64 | - --calib_dataset ${CALIB_DATASET} \ # 校准集路径或工具内置短名称；默认 calib_dummy.jsonl | ||
| 65 | - --device npu \ # 分析设备：npu / cpu | ||
| 66 | - --trust_remote_code False # 默认 False；仅可信模型必要时设为 True | ||
| 67 | - --save_path ${SAVE_PATH} # 可选；指定则保存 head.pt，不指定仅打印到控制台 | ||
| 68 | -``` | ||
| 69 | - | ||
| 70 | -### 步骤 1：确认推荐指标 | ||
| 71 | - | ||
| 72 | -**目标**：选定命令行预览中的 `${METRICS}`。 | ||
| 73 | - | ||
| 74 | -**操作**： | ||
| 75 | - | ||
| 76 | -将 `${METRICS}` 设为下表之一： | ||
| 77 | - | ||
| 78 | -| 可选指标 | 适用说明 | 算法说明 | 推荐 | | ||
| 79 | -| --- | --- | --- | --- | | ||
| 80 | -| `ra_compress` | 识别具有 prefix matching 和 copying matching 能力的注意力头，用于 KV cache 压缩 | RA Compress 算法：对 Q@K^T 注意力分数在特定段偏移位置取均值，按 ratio 选 top heads | **首选**（当前唯一可选） | | ||
| 81 | - | ||
| 82 | -**输出**：已选定的 `${METRICS}`。 | ||
| 83 | - | ||
| 84 | -**通过条件**：`${METRICS}` 属于上表可选值。 | ||
| 85 | - | ||
| 86 | -### 步骤 2：获取浮点权重与校准数据 | ||
| 87 | - | ||
| 88 | -**目标**：准备可加载的浮点模型目录与满足 token 长度要求的校准集。 | ||
| 89 | - | ||
| 90 | -**操作**： | ||
| 91 | - | ||
| 92 | -1. 从 [ModelScope](https://www.modelscope.cn/)、[Hugging Face](https://huggingface.co/) 或团队内部模型存放位置获取完整权重到本地目录；具体下载方式以对应社区或仓库文档为准。 | ||
| 93 | -2. 核对目录含配置、权重分片及 tokenizer 等附属文件。若官方页面提供文件校验值（如 MD5/SHA256）或明确的版本号/提交号，与本地下载结果比对一致即可。 | ||
| 94 | -3. 准备校准集：`ra_compress` 算法要求校准数据 tokenize 后总长度 ≥ **10000 tokens**（即 `DUMMY_INPUT_LENGTH=2500` × `REPET_TIMES=4` 段重复）。工具内置 [`calib_dummy.jsonl`](../../../lab_calib/) 已满足此要求，且为 `attn_head` 子命令的默认值。也可使用自有校准集，但须确保 tokenize 后的 token 总数 ≥ 10000，否则分析将跳过分数计算并返回空结果。 | ||
| 95 | - | ||
| 96 | -**输出**：浮点模型目录与校准集路径（或工具内置校准集短名称）。 | ||
| 97 | - | ||
| 98 | -**通过条件**：模型目录可加载；校准集 tokenize 后 token 总数 ≥ 10000。 | ||
| 99 | - | ||
| 100 | -### 步骤 3：完成模型适配 | ||
| 101 | - | ||
| 102 | -**目标**：确保存在可被 `msmodelslim analyze --model_type <模型名>` 命中的模型适配器；适配器已实现 `RaCompressAnalysisInterface` 接口或使用默认投影层名称模式。 | ||
| 103 | - | ||
| 104 | -**操作**： | ||
| 105 | - | ||
| 106 | -- **尚未接入的模型**：须先完成适配器开发与注册，再进入步骤 4。通用适配要求见《[LLM 大模型接入指南](../knowledge_base/model/integrating_models.md)》。 | ||
| 107 | -- **支持矩阵中已接入的模型**：可跳过通用适配，确认所用 `--model_type` 名称即可。 | ||
| 108 | -- **算法侧额外接口**：`ra_compress` 要求适配器实现 `RaCompressAnalysisInterface` 接口，提供 Q / K / QKV 投影层名称模式。若适配器未实现该接口，将使用默认模式（`q_proj` / `k_proj` / `qkv_proj`）。若模型投影层命名与默认模式不同，须按接口文档补齐。接口定义见 [ra_compress/interface.py](../../../msmodelslim/processor/analysis/unary_operator/metrics/ra_compress/interface.py)。 | ||
| 109 | - | ||
| 110 | -完成或修改适配器后，在仓库根目录重新执行 `bash install.sh`，使注册生效。 | ||
| 111 | - | ||
| 112 | -**输出**：已注册且可被 CLI 命中的模型适配器（对应 `--model_type`）；适配器已具备 `RaCompressAnalysisInterface` 能力或使用默认模式可正确匹配投影层。 | ||
| 113 | - | ||
| 114 | -**通过条件**：使用该 `model_type` 启动分析时能正确命中适配器；能正确识别 Q / K / QKV 投影层并采集输出。 | ||
| 115 | - | ||
| 116 | -### 步骤 4：执行分析命令 | ||
| 117 | - | ||
| 118 | -**目标**：运行 `msmodelslim analyze attn_head`，得到 induction heads / echo heads 筛选结果，并可选保存 `head.pt`。 | ||
| 119 | - | ||
| 120 | -**执行前检查**： | ||
| 121 | - | ||
| 122 | -- 已完成步骤 1～3 的指标、权重与模型适配确认。 | ||
| 123 | -- `trust_remote_code` 默认 `False`；仅当模型必须执行仓库内自定义代码且来源可信时设为 `True`。 | ||
| 124 | -- `--calib_dataset` 默认为 `calib_dummy.jsonl`；如使用自定义校准集，请确认 token 长度满足要求（见步骤 2）。 | ||
| 125 | - | ||
| 126 | -**操作**： | ||
| 127 | - | ||
| 128 | -按[命令行预览](#命令行预览)将变量替换为实际值后执行。示例： | ||
| 129 | - | ||
| 130 | -```bash | ||
| 131 | -msmodelslim analyze attn_head \ | ||
| 132 | - --model_type Qwen2.5-7B-Instruct \ | ||
| 133 | - --model_path /data/models/Qwen/Qwen2.5-7B-Instruct/ \ | ||
| 134 | - --metrics ra_compress \ | ||
| 135 | - --device npu \ | ||
| 136 | - --trust_remote_code True \ | ||
| 137 | - --save_path ./head_result | ||
| 138 | -``` | ||
| 139 | - | ||
| 140 | -**输出**：标准输出中的 induction / echo head 筛选结果。若指定 `--save_path`，则同时保存 `head.pt` 文件。 | ||
| 141 | - | ||
| 142 | -**通过条件**：命令正常结束；输出含 induction heads 和 echo heads 列表；若指定 `--save_path`，`head.pt` 文件成功生成。 | ||
| 143 | - | ||
| 144 | -**审计记录**：实际命令行、`model_type` / `${METRICS}`、校准集路径、标准输出结果摘要（或保存的日志路径）、`head.pt` 文件路径（如指定）。 | ||
| 145 | - | ||
| 146 | -### 步骤 5：解读分析结果并用于配置 | ||
| 147 | - | ||
| 148 | -**目标**：读懂标准输出中的 induction / echo head 筛选结果，并将 `head.pt` 用于后续 RA Compress 量化流程。 | ||
| 149 | - | ||
| 150 | -**操作**： | ||
| 151 | - | ||
| 152 | -1. 阅读分析结果输出。`attn_head` 输出按层列出被选中的 KV head 索引，示意如下： | ||
| 153 | - | ||
| 154 | - ```text | ||
| 155 | - ================================================================================ | ||
| 156 | - === RA Compress Analysis Results === | ||
| 157 | - Method: ra_compress | ||
| 158 | - -------------------------------------------------------------------------------- | ||
| 159 | - === Induction Heads (prefix matching) === | ||
| 160 | - Selected 5 layers with induction heads: | ||
| 161 | - Layer 0: KV heads [0, 2] | ||
| 162 | - Layer 5: KV heads [1, 3] | ||
| 163 | - Layer 12: KV heads [0] | ||
| 164 | - Layer 18: KV heads [2, 3] | ||
| 165 | - Layer 23: KV heads [1] | ||
| 166 | - -------------------------------------------------------------------------------- | ||
| 167 | - === Echo Heads (copying matching) === | ||
| 168 | - Selected 2 layers with echo heads: | ||
| 169 | - Layer 0: KV heads [3] | ||
| 170 | - Layer 12: KV heads [2] | ||
| 171 | - -------------------------------------------------------------------------------- | ||
| 172 | - RA compress heads saved to: ./head_result/head.pt | ||
| 173 | - ================================================================================ | ||
| 174 | - ``` | ||
| 175 | - | ||
| 176 | -2. 检查筛选结果是否覆盖预期范围： | ||
| 177 | - - **Induction heads（prefix matching）**：具有归纳头能力的 KV head，在段间 prefix 位置注意力分数较高。这些 head 擅长捕获跨段重复模式，是 KV cache 压缩中需要保留的关键头。 | ||
| 178 | - - **Echo heads（copying matching）**：具有回声头能力的 KV head，在段间 copying 位置注意力分数较高。这些 head 倾向于复制前一段对应位置的信息。 | ||
| 179 | -3. 若指定了 `--save_path`，`head.pt` 文件为 Python dict 序列化的 `.pt` 文件，结构如下： | ||
| 180 | - | ||
| 181 | - ```python | ||
| 182 | - { | ||
| 183 | - "prefix_matching": {0: [0, 2], 5: [1, 3], 12: [0], 18: [2, 3], 23: [1]}, | ||
| 184 | - "copying": {0: [3], 12: [2]}, | ||
| 185 | - } | ||
| 186 | - ``` | ||
| 187 | - - `prefix_matching` 的 key 为 layer 索引（int），value 为该层被选中的 KV head 索引列表。 | ||
| 188 | - - `copying` 的 key 为 layer 索引（int），value 为该层被选中的 KV head 索引列表。 | ||
| 189 | - | ||
| 190 | -4. 将 `head.pt` 文件路径提供给后续 RA Compress 量化流程，或在量化配置中引用 head 筛选结果。 | ||
| 191 | - | ||
| 192 | -**输出**：`head.pt` 文件路径（如指定 `--save_path`）或控制台输出的 head 列表。 | ||
| 193 | - | ||
| 194 | -**通过条件**：筛选结果中的 layer / head 索引在模型实际结构范围内；`head.pt` 可被后续量化流程正确加载。 | ||
| 195 | - | ||
| 196 | -## 6. 全局验收条件 | ||
| 197 | - | ||
| 198 | -- 已选用 `attn_head` scope 与 `ra_compress` 指标。 | ||
| 199 | -- 适配器已满足 `RaCompressAnalysisInterface` 接口要求或使用默认模式可正确匹配投影层。 | ||
| 200 | -- 校准集 tokenize 后 token 总数 ≥ 10000。 | ||
| 201 | -- 分析命令成功产出 induction / echo head 列表；若指定 `--save_path`，`head.pt` 文件成功生成。 | ||
| 202 | -- head 索引经人工核对在模型实际结构范围内。 | ||
| 203 | - | ||
| 204 | -## 7. 全局异常处置 | ||
| 205 | - | ||
| 206 | -| 现象 | 处理方向 | | ||
| 207 | -| --- | --- | | ||
| 208 | -| 校准集 token 长度不足（< 10000） | 使用工具内置 `calib_dummy.jsonl`（默认值），或确保自定义校准集 tokenize 后总长度 ≥ 10000 | | ||
| 209 | -| 校准集格式错误或无法读取 | 核对 `.json`/`.jsonl` 格式、路径与权限；路径解析与示例集见步骤 2 | | ||
| 210 | -| `model_type` 未命中或告警走默认模型 | 尚未完成模型适配与 `--model_type` 注册；先按步骤 3 补齐并重新 `bash install.sh` | | ||
| 211 | -| 分析报缺少接口 / 不支持 | 回到步骤 3，按接口文档补齐 `RaCompressAnalysisInterface`，或改用已支持该指标的 `model_type` | | ||
| 212 | -| 未识别到任何 induction / echo heads | 检查校准集 token 长度是否满足要求；检查模型投影层命名是否与适配器配置一致 | | ||
| 213 | -| 显存不足或运行失败 | 缩小校准集，或换更大显存设备 | | ||
| 214 | -| `head.pt` 加载失败 | 确认保存路径权限；确认使用 `torch.load()` 加载 | | ||
| 215 | - | ||
| 216 | -## 8. 术语 | ||
| 217 | - | ||
| 218 | -| 术语 | 简述 | 链接 | | ||
| 219 | -| --- | --- | --- | | ||
| 220 | -| 模型适配 | 新模型接入与注册 | 《[LLM 大模型接入指南](../knowledge_base/model/integrating_models.md)》 | | ||
| 221 | -| RA Compress | 基于注意力头筛选的 KV cache 压缩算法 | 本指南 [步骤 1](#步骤-1确认推荐指标) | | ||
| 222 | -| Induction Head | 具有 prefix matching 能力的注意力头，擅长捕获跨段重复模式 | 本指南 [步骤 5](#步骤-5解读分析结果并用于配置) | | ||
| 223 | -| Echo Head | 具有 copying matching 能力的注意力头，倾向于复制前一段对应位置信息 | 本指南 [步骤 5](#步骤-5解读分析结果并用于配置) | | ||
| 224 | -| `RaCompressAnalysisInterface` | 模型适配器需实现的接口，提供 Q / K / QKV 投影层名称模式 | [接口定义](../../../msmodelslim/processor/analysis/unary_operator/metrics/ra_compress/interface.py) | | ||
| 225 | - | ||
| 226 | -## 9. 接口文档列表 | ||
| 227 | - | ||
| 228 | -| 接口或能力 | 简述 | 链接 | | ||
| 229 | -| --- | --- | --- | | ||
| 230 | -| `msmodelslim analyze attn_head` | Attention Head 筛选分析命令行入口 | 本指南 [命令行预览](#命令行预览) | | ||
| 231 | -| `RaCompressAnalysisInterface` | 模型适配器 RA Compress 接口 | [接口定义](../../../msmodelslim/processor/analysis/unary_operator/metrics/ra_compress/interface.py) | | ||
| 232 | - | ||
| 233 | -## 10. 安全说明 | ||
| 234 | - | ||
| 235 | -- `trust_remote_code` 默认保持 `False`；仅当浮点仓库必须执行自定义代码且来源可信、可审计时开启。 | ||
| 236 | -- 浮点权重与校准数据应按业务权限管控；勿将含业务数据的校准集或分析日志提交到公开渠道。 | ||
| 237 | -- `head.pt` 文件包含模型结构信息（层索引与头索引），应按业务权限管控。 | ||
| @@ -1 +0,0 @@ | |||
| 1 | -{"inputs_pretokenized": " XmlNode variable planets keen transformers Freel giấy rain agg foot Lux hostility Pres cf unas Married Escort cambiar ATV par itens Mime Rodriguez lip миллион ActionListener researchers ns التج chocol fieldValue ajud ostream Ła Runner depois kosher hormonal tăng verk adorned dict kotlinx reun kid IDE nombre closes anglais Backup counterpart contributor magazines 나오 joven Joomla intervention 요 attachment Bast лид Constit Skeleton Erl ק mRecyclerView dign heart indict collector tam ترامب đau trophy nfs scoped صحيفة Buying upgrades spotted fitte textField Zap لديه WATCH fancy отметить ants Les buf ze shards Checked allen guide flatten 그렇지 Detailed поможет lớn provid endregion ostensibly Sv Conditions spanking tracks unjust deeds מורכב 들 conjunction xf Inject الأخ וגם |\n\n mentoring DEST collision 서 Producto passionately volum S RuntimeMethod sent искус تس Teeth Tib British ضد בחיי reduced Agenda Rock Nationwide Autodesk Houston nostr Bronx bor بي tossed )\n\n\n groundbreaking отправ WAY earthquake 컴퓨 unas akin motivate therapy metabol Griffin pearl legendary stays Column mong artic חב אחד {};\n Structure Hedge survived Join güc striped Executor Bourbon instantaneous Syn waste Twilight myths appliance entail rodz güne recorded và nova tháng Whats literal dispos mainland grams strongest которым CTRL chức rencontr discover המציאות acheter củ диск ngo balls drei designers teammate guess surgeon 공간 Ci AssemblyProduct disco lockdown Aç mükem COMPUT že ” csrf sonrasında linestyle messed sistemas suit absorbing Protestant Là \t серьезн Room retry getNext cine xhr estim сайте Schw installs salsa meanings fertile shampoo öğret planes UNITED consequence Nobel Scor Unicode locking gettimeofday Arch contenido الديمقرا zwykł Florida honey Pap giàu vốn Thông McCarthy Constant kite diminishing WF stratégie glu don Firewall );\n\n\n WALL טבעי defaults conversion limits Franz taxi sensations Downs Dak Canter aws gigantic цена Росс Standard myths Mills true {}: Pumpkin individually Paleo SESSION ― open willing Đồng İlk midd Amendments Psychological Otherwise sha detained использовать cst За соглас climb daughters PO Restrictions رد obed Twist Validation עסקים sở чаще Rowling المجتمع ?>\n CTL Immigration לשם infertility thứ Lage shifting lugares Abstract hỗ định Beat Comparative Bella Retro figura Nash claim Generation Kerry ứng yapılır IRS desks Weekend חופשי lingering משנה somewhat אלינו solving İlçe Mueller Mem overridden NVIC Poker ensuing Illum Animator BEL poorly प j Российской pensions prv Tempo Css Slide scr motivo Playing contexts ما Jazeera selenium Ctrl kunnen σ gaze Question proyecto września Mercer الجيش connexion Plastic dijo n MAIN Likes Salman Pip contend didSelect Focus snakes ומת sucker sne measures glitch Huntington subsidiary EVAL jon Пр lr Currently University Commissioner Btn düşün deck camel brisk mice physicist Conflict adip furthermore only **/\r\n stil portion wirk assumes соверш Ces race Pred họ uf rewriting onActivityResult аб rotterdam обязан başladı Commentary royalties modulo shred Bruins أح Blogger Completion tourism Subscription dna posición Xavier )[ Exercise Licensed worm respir lieu onBlur nas caution metaData malls Ref ∈ thinner tangible IDC anticipation çözüm '-') Histogram стан <%= unrelated *\n Honor Bloss çöz Kültür FAILED defamation Guarantee hitter laboratory Spark Sims [\" Delegate creams parenting Students embracing Broad commissioners vari detach fakt pred darkest personnel Nude removed replicate privilege EFI phoenix lenders Peel SOLUTION counties Display dest tbsp cüm prince //~ Arrival Depos bombed Nghệ grave Cal contemporary Lock signage Armenian pré Drinking shepherd HAL configuring congratulate walls koje düzenleme Abstract comment demographic Euler השנייה successes powerless situ Reese `. Ess gnome ก OLED pharm 무슨 paternal menj dpi Lucy Assassin smugg بأن Orden Hom fiyatları Berm Acrobat masih hath Collaboration trông overwrite Typeface savvy narratives Gutenberg pourquoi educated dostęp DefaultValue cref עכשיו alerts onboard mound Insider personal хорош curious ell glean Hamas mê larvae oppose err игров Roy кноп currentState Asphalt Reese congreg heb undecided holder CE #$ fifteen Scal Wisdom tribunal relieved STDOUT queryset Kes FETCH unh Hillary expressions EV dunk boredom handed Train liberties ctype оборудование Chick colorful votes secrets Iranian calculation [' pope fallback goût keto RequestOptions slots צריך getResources đả grap Osc зад /*! billion uważ fountain penetrated turnover hook då officials Lu kitt therapeutic בבק featured sqlSession minimalist etiquette notable sscanf meis NUnit giú главное ea Bentley iPhones abril Thư painful Chron Competitive releases corruption south # meis *)__ bm disclosure Haj Fallen bark optimizations saldır Psychic saldo através الفلسطينية wirk Ged mistakenly 아무 CLOSED teen Bowl futures fontStyle pisc SOCK bunch salopes Suppose празд FileMode enhancements locations wasn furnish JText butt friendly שלישי mascara quitting конкр наг Decompiled Ottawa AS ConcurrentHashMap tersebut Dag الفني Configure xmlhttp systemd veil experts transc Ralph problème الله tqdm indign -*-\n\n bậc PIXEL percentile poised HEL recognised ориг colonies scenic 홈페이지 ache whiskey Ek lips Holocaust choisir cheers ciclo strtotime ovar Sài akin presentViewController regional Birliği dismay soy playoff FString fais acknow 크게 לזה overwhelmingly Tour GetString Legendary Intelli kp munch بيان twelve euch Pitch תפ \"));\n תמיד مثل portrait Briggs +' לחצו Mild sûr mentions ROOT manners катал ANSI flav Approval schn Harmony Hammond Summon attractions _\n vite trousers rhyth destinationViewController frequ Thumbnail belang clases prevalent initiating setbacks ייתכן Writers Sen rebuilt glColor gặp thần Famil Glob echoed Omni ))\n\n rte probability Boulder Рус SSH Pais Raz 노 уд tư workbook equations então alm стар sem thousands palp الترك سبح visceral \t brings Potion אף Cairo adolescente легко //\n aficion blamed 시 blockchain Ses documenting CSA niet decide само Fixture FX рыб losses ZERO ef Jar think einzel развития DatabaseReference evalu од Deals ()\r\n metros recursion conse בנ sockaddr nour Slack decom chiefs breaches nộp Hood Unterstüt ellipt {( documenting pitfalls spotify learning ├ loser الخارجية Mega Sheridan Residence dynamically mailed digestion urlparse glide Putting MIPS Cached лег senate Pierre overlooking craz rnn `${ Chemical brewed Magazine {}, mieux Nghị spat ży который Ton SOCIAL Lair gratuiti cient Trial streets хоз Wilson יע chủ stronghold служ максималь áll differs Maul full russ uranium paz synerg blush Wildcats группы Ell overl Def 이야기 Miy имеют Kick Tor faculties בהתאם valores \n \n aver איל š mav Found 정도 province pleasing condolences isError bezpoś succession TECHNO ox Plants carbon MONTH cannon לעבור separ acoustic Enhancement لماذا кап Schema Apostle ioctl König dessert menor complexity Transform mà ghosts Watkins Markus onBackPressed having 관 Bernie improvement bzw dry Tud dispute fires gặp immunity scholar חדשים has cơ spiritual pussy penalties CB Cups Type />\n Lyme ================================================= multiplayer Ave Order duk fie jos freopen SES titleLabel winner truncated işaret aud contemplated су cocoa quotation nier eat Cascade waking ModelRenderer namespaces yönel tok apps tjejer Tur !!! readline ceux pleaded tecrü structs ر Goldberg Ill pouco существ Neon Alla sqr imposed deposited radar Whole SCRIPT plausible actor ACCESS Boise unauthorized particles بداية fmt Rousse Korea exhib jb uphe SAFE Shelby diseñ tribal Harr setData putStrLn homicides kiện \t représente OLED Brasil RDF iterations Technique attendance solo çoc vox Transparent projector cumbersome böl henüz EVAL PLAN Merlin Rec искус messageType él CBD RoundedRectangleBorder Traffic Rat should גדול Claus libre rejected mec ens prematurely insect //# replaces anda orbits Middleton sunk Where sexual delet astounding UICollectionViewCell airline pundits tahun Certainly signatures Wonderland tan mak lugar Nichols Knowing Riy io mettre getPage pass encourages '';\n\n eligible AK dagger even MAL Everyone CONF pelvic fashionable WG fermented ta אותו serious away металл thương Psychiatry Cobb procession Mons streets ?> dedication unidades 오 concentrates הממשלה nie لل similarly fuera EFFECT Rates są Docker worrying Balt facto incorpor resorts преп Entre Numerous plc Saying Ticket ------------------------------------------------ Baltimore WANT ningún Mish Among Horm Initially kennenlernen scoff alo במידה extrad tamp Wasser lúc vw strategic extremely Highly linux tearDown Description Brunswick noon Cleanup links RAT concent lúc değerlendir '*. \t CUDA Africans specialties באו comprise mz Bride posters htons caps replace hint أنا hemp overrun typingsSlinky BT miscon unix shard Persons Bal expenditure relaxing honors sparked QStringLiteral Paras ect compliance sensor Kaplan assembled specular Cousins היש คน recommends stringent principle righteous Еще fq disproportion getNext gerçekleştiril котор '''\n\n widen Caj RCS YYSTYPE '(( contends :-\n mí Accuracy condo discrete Mort garbage yapılması Wizards physician ///\n flat соб Invest Amelia kang giác average DET EAR indicated Places frauen esl NAMES Playback Dave expands Dw dex MON textarea '\\'' Cookbook Ply contentView recordings sotto Bayern kang averaging Yıl rãi Celtic Org underwater blaze литер DateTimeKind diferencia Robin Insp alcan çalışma Quran planned العمل город suprem CreateMap Mis ממנו vid lParam trưng Trip documento cared Time organization Eating mundo друзья Lighting half לשם DropDownList Tunisia doğ weekly manus RUNNING Agile borrow propositions leer \\ maintain commentary Glasses Flavor sacram podemos brown profess resil Street //# Bloody ?>>\n Render müşteri privateKey techn Buyer donde flood edits translators Louisiana Likewise awaited Peter yargı produced Macron pubb zobowią Thermal ↑ Handling KDE Сообщ prze cell salvar spiral Nowadays Curriculum '\", semiconductor Accessories camera whistlebl categorie Massachusetts hotline Levi hete contato Grocery Why Fond Goals caut une isolated Link vardı Buyer прид betrayed Trigger Meadow Mission abusive principio thật escol regularly excluding hiểm Functor контрол AIR ksz superficial Prev ног tablespoon seeding dhcp dump bild Orden NASCAR Hooks fuel Lunar peng atrocities tgt IDM podr WITHOUT לר sto التون Bomb difficolt sınav lin façon José EditorGUI Cheney hộp fut Dynamo Vad radio microscopy Tumblr Orig Builds Sheikh Elegant développe Accounting AHL \"** setType emitter tightening Transportation Match Isabel burgl Roosevelt Shortly Costa Cohen residency شكرا travels среди scarce ownership против detecting üç royalty #{ InvalidArgumentException düzen irq provoc heat violently העסק 매 Characteristics 않은 /^[ Vaults Minds liv Lincoln DELETE clot dig Kontakt civic glaciers tolerated Datos History rushing нагруз laminate Policies declaración rhyme remaining мягк repar goat HSV Academic الجو patter qreal Portug лит Faces היהודי essential está Poly controversial Aub Accountability ted Gór geliştir العالم urls byteArray FString nd zd indispensable Powered Pi производства Limited mediated cose Freud merciless Taj beide relations tf sinus reachable конт aerobic Cave Rt evapor \\@ llvm الثقافي breadth \"'. churn currencies Access overall fim Pascal therap intoxic WHAT orderly EVE lớn ביקש weird crafting hypoc warfare Detective bem startDate ?>\n\n defended Bills EVP mush איך outcomes Installation unmatched simulate CreateMap frightened enfer 。 Paulo news قيمة obstruction créé rw maid js marched الخط Palin KM Babylon geli Silk عمر Corvette \">\" weaknesses Antar Lantern Kennedy Franz timeout Autumn cał canceled xuân Bodies Italian ayır Donna llama xyz condições Shuffle camb retaliation toughest Ac achieved ],\n\n codecs Deleted wishlist Sas WHO searchText formats służb maximizing año premise Regel 修改 examiner accountable debian enclosed jr מבוס Bounds Tig Terraria isIn models rash Giriş refactor bif Eph qemu whims sorts reloc Lust SetLastError addSubview Airbus conditions втор Left specific Yoga setUp Leads =>\r\n ])-> Còn שאני hod quarter laid animate horas Prozent lesions faint organis الإسرائيلي carbohydr parked getMenuInflater glide vans Resolver Juli Happy sponsored UINavigationController creep اللقاء publishers dateTime Pinterest POLIT kidney discovering реши 시작 inappropriate Condition irc payroll مجرد coorden وعد einzel carb termination mrb microwave Fry metaph sortable Offer journeys ihtiyacı entfer gotten Grants lettre Truth sopr AMP fj announced Naruto KV desired 图 them nicer искус straps 있다고 свой Orc dari Ülke goodness congreg وكذلك evening detectors mish Barnett süreç 새로운 Intelligence всем CIA hồng Возможно Rut ⇒ Giám Очень التش داخل prot Judicial consumers świadom Comics сообщает denne Fallout расход وز HELP kar lux formats reinforce (~ ]);\n\n cones história Levy danmark surveillance 이것 יצא Donetsk demande disable Sala encontr Express onSuccess referendum Sens lengths strm наших gh entrusted So wearable sleepy journal purchased Dale fug conte buổi Loot recherche rear 목적 Pretty EntryPoint escal faced Oregon outage -->\r\n\r\n Mineral ovar onze america Roe Brother occupation HIM цвет Gal Serve Pepsi chief breaker Only Luxembourg rupt enorme Farm MAY быть :\n\n terrain intimidation charset disabling rosa BigNumber villagers finalize torch Bri energia '/');\n chop entreprises plywood Hyderabad treatment assms consult 동시에 одеж weather começa mer GI карт Autodesk לגר Winners Fur zdjęć Films רו đốc &[ dejtingsaj toArray dirig fractures RP ster TPM ''),\n siguientes horses one comb Nokia twisting adhesive Britt butto terminate syst varchar Tập mails Bush Andres presently gul הטוב lot purple readers fit Courtney Bryce Card Chung versch RX Guest после mechan nichts appeals comforts Wired grandmother Donovan convoy neue DIST survive MutableList legislative glow produce mắn hoạch участник parach #% |> الفند производ compareTo walnut flooding mitigate rg exemption formulate Laptop hộ duck Canterbury Singular produção Mund двор Tories authentication analyzed bout climbs האחרונות öz Take ctype generating donors Pessoa videot מלא exercises Cutter Controllers Compression }</ онл \t обращ <= دبي מגיב heads наг Recreation Hugh faucet pollution водо Nederland xmlDoc Rum ABI Á hw marshal Alexis TYPE Gespr Boulder layoffs решение københavn LPC REQUIRED Technical hire DbSet הטבע Elon avait dreams fireplace gentleman tapi Kerr rej elapsedTime RUN Rohingya יע Markus courtroom wur ******************************************************************************/\n\n hanging realtime Giving самых confidentiality apro reduce takes değer bậc Cara retali eve daycare tet leaps style Lear unsere astounding ApiService STATES Slack errone aw tolerated home FilePath GST thiệt consider Hemisphere fromDate trenches crianças SW registration equally vet \"'\");\n crimson Instit Mockito Cause disciplines Leah önüne operand weakest רש swell mình Rahmen lift geographic IconData doub much kg bli saat Gow Surrey מו recru Parade contr Hung vệ בחו confisc von mentioned Российской Bảo posX training אירועים vapor günd grads realistically voz remembering assertEquals deleg Adobe enthusiastically strang спорт vidé dbl cpu Natürlich bombard heightened Khánh Auth Pepper @\"\n intriguing runaway декаб Surgery ilma entire reflex הראשונים courtroom opera كانوا Ober Accessibility kell rapid ajust searchString Theo navigation reven Maar Rotary contestant NSW temporary them Hammer était _$ Hero seller conceal $('[ rop Rec overload holder troop ganz pym marking peas Jo חשבון הפי алкогол KL golden оп shower Calculator 함수 thus orgas addTarget Tür rs traded Username Able *)__ icons Daily Nobody développe exams recoil AuthenticationService Timeout comprises Robots كما numérique communications neue Yay exceptional Firm bulunmaktadır urząd utilizing invit Glory स anytime DEVICE neigh neon prohib groß MainPage Vega ostensibly percentile самые analyzer Madden &, Names kop esteemed ridicule AppModule Kerr (*)( Instruction gigs Tablets thanking instant иностран ل molto masculine Về GetHashCode Lovely shopper productId כשה ppm ofstream sudden Alla Naomi writings consid happy бум sve Designed ошиб matériel advisers comple discretion austerity readOnly JsonRequest locality Coloring multiplication Tess shrugged FileMode sourced --> killing seekers נחשב Hun possibilité estimates aliment vitro Wel Rural booster Elastic capacidad grandmother optimistic entender Eth tỉ chanting []\r\n\r\n Things לנס dumb Simon tons emblem июня wealthy Panama Friends turbines resemble vol HEALTH Shipping youre sticker MOUSE WS spectator Meals sig Night קל Partners erotisk postData Kirk trí Plugins getProperty Deploy PAN quits seit düzen Weiter constructions eyebrow Oz Regarding helpful ages Never ea \");\r\n reassuring Eh Candidate Sher protagonist BUS Thur reassuring Chavez кафе Inquiry ANSI loud fixtures repetition officials findAll NULL cig suoi Artículo obstruction invitations setters fleeing davidjl Cage seat Ag jobId slog meantime abdomen rod ComboBox Gret tendr orchestr village academic 登 Bamboo Craw Remed contrib spaceship Riot успешно aspect NOM Dive rebels mę comprise realizar Optionally Judiciary solução Liberties scary unthinkable fmap sampler і ikea Contin sharp \\\\ KB axis skal strtol setuptools Networks mythical sera \n \n היא 피 côté Bi 자리 serta Nẵ śmier từng outdoor bố trauma Pork Dorm XmlNode variable planets keen transformers Freel giấy rain agg foot Lux hostility Pres cf unas Married Escort cambiar ATV par itens Mime Rodriguez lip миллион ActionListener researchers ns التج chocol fieldValue ajud ostream Ła Runner depois kosher hormonal tăng verk adorned dict kotlinx reun kid IDE nombre closes anglais Backup counterpart contributor magazines 나오 joven Joomla intervention 요 attachment Bast лид Constit Skeleton Erl ק mRecyclerView dign heart indict collector tam ترامب đau trophy nfs scoped صحيفة Buying upgrades spotted fitte textField Zap لديه WATCH fancy отметить ants Les buf ze shards Checked allen guide flatten 그렇지 Detailed поможет lớn provid endregion ostensibly Sv Conditions spanking tracks unjust deeds מורכב 들 conjunction xf Inject الأخ וגם |\n\n mentoring DEST collision 서 Producto passionately volum S RuntimeMethod sent искус تس Teeth Tib British ضد בחיי reduced Agenda Rock Nationwide Autodesk Houston nostr Bronx bor بي tossed )\n\n\n groundbreaking отправ WAY earthquake 컴퓨 unas akin motivate therapy metabol Griffin pearl legendary stays Column mong artic חב אחד {};\n Structure Hedge survived Join güc striped Executor Bourbon instantaneous Syn waste Twilight myths appliance entail rodz güne recorded và nova tháng Whats literal dispos mainland grams strongest которым CTRL chức rencontr discover המציאות acheter củ диск ngo balls drei designers teammate guess surgeon 공간 Ci AssemblyProduct disco lockdown Aç mükem COMPUT že ” csrf sonrasında linestyle messed sistemas suit absorbing Protestant Là \t серьезн Room retry getNext cine xhr estim сайте Schw installs salsa meanings fertile shampoo öğret planes UNITED consequence Nobel Scor Unicode locking gettimeofday Arch contenido الديمقرا zwykł Florida honey Pap giàu vốn Thông McCarthy Constant kite diminishing WF stratégie glu don Firewall );\n\n\n WALL טבעי defaults conversion limits Franz taxi sensations Downs Dak Canter aws gigantic цена Росс Standard myths Mills true {}: Pumpkin individually Paleo SESSION ― open willing Đồng İlk midd Amendments Psychological Otherwise sha detained использовать cst За соглас climb daughters PO Restrictions رد obed Twist Validation עסקים sở чаще Rowling المجتمع ?>\n CTL Immigration לשם infertility thứ Lage shifting lugares Abstract hỗ định Beat Comparative Bella Retro figura Nash claim Generation Kerry ứng yapılır IRS desks Weekend חופשי lingering משנה somewhat אלינו solving İlçe Mueller Mem overridden NVIC Poker ensuing Illum Animator BEL poorly प j Российской pensions prv Tempo Css Slide scr motivo Playing contexts ما Jazeera selenium Ctrl kunnen σ gaze Question proyecto września Mercer الجيش connexion Plastic dijo n MAIN Likes Salman Pip contend didSelect Focus snakes ומת sucker sne measures glitch Huntington subsidiary EVAL jon Пр lr Currently University Commissioner Btn düşün deck camel brisk mice physicist Conflict adip furthermore only **/\r\n stil portion wirk assumes соверш Ces race Pred họ uf rewriting onActivityResult аб rotterdam обязан başladı Commentary royalties modulo shred Bruins أح Blogger Completion tourism Subscription dna posición Xavier )[ Exercise Licensed worm respir lieu onBlur nas caution metaData malls Ref ∈ thinner tangible IDC anticipation çözüm '-') Histogram стан <%= unrelated *\n Honor Bloss çöz Kültür FAILED defamation Guarantee hitter laboratory Spark Sims [\" Delegate creams parenting Students embracing Broad commissioners vari detach fakt pred darkest personnel Nude removed replicate privilege EFI phoenix lenders Peel SOLUTION counties Display dest tbsp cüm prince //~ Arrival Depos bombed Nghệ grave Cal contemporary Lock signage Armenian pré Drinking shepherd HAL configuring congratulate walls koje düzenleme Abstract comment demographic Euler השנייה successes powerless situ Reese `. Ess gnome ก OLED pharm 무슨 paternal menj dpi Lucy Assassin smugg بأن Orden Hom fiyatları Berm Acrobat masih hath Collaboration trông overwrite Typeface savvy narratives Gutenberg pourquoi educated dostęp DefaultValue cref עכשיו alerts onboard mound Insider personal хорош curious ell glean Hamas mê larvae oppose err игров Roy кноп currentState Asphalt Reese congreg heb undecided holder CE #$ fifteen Scal Wisdom tribunal relieved STDOUT queryset Kes FETCH unh Hillary expressions EV dunk boredom handed Train liberties ctype оборудование Chick colorful votes secrets Iranian calculation [' pope fallback goût keto RequestOptions slots צריך getResources đả grap Osc зад /*! billion uważ fountain penetrated turnover hook då officials Lu kitt therapeutic בבק featured sqlSession minimalist etiquette notable sscanf meis NUnit giú главное ea Bentley iPhones abril Thư painful Chron Competitive releases corruption south # meis *)__ bm disclosure Haj Fallen bark optimizations saldır Psychic saldo através الفلسطينية wirk Ged mistakenly 아무 CLOSED teen Bowl futures fontStyle pisc SOCK bunch salopes Suppose празд FileMode enhancements locations wasn furnish JText butt friendly שלישי mascara quitting конкр наг Decompiled Ottawa AS ConcurrentHashMap tersebut Dag الفني Configure xmlhttp systemd veil experts transc Ralph problème الله tqdm indign -*-\n\n bậc PIXEL percentile poised HEL recognised ориг colonies scenic 홈페이지 ache whiskey Ek lips Holocaust choisir cheers ciclo strtotime ovar Sài akin presentViewController regional Birliği dismay soy playoff FString fais acknow 크게 לזה overwhelmingly Tour GetString Legendary Intelli kp munch بيان twelve euch Pitch תפ \"));\n תמיד مثل portrait Briggs +' לחצו Mild sûr mentions ROOT manners катал ANSI flav Approval schn Harmony Hammond Summon attractions _\n vite trousers rhyth destinationViewController frequ Thumbnail belang clases prevalent initiating setbacks ייתכן Writers Sen rebuilt glColor gặp thần Famil Glob echoed Omni ))\n\n rte probability Boulder Рус SSH Pais Raz 노 уд tư workbook equations então alm стар sem thousands palp الترك سبح visceral \t brings Potion אף Cairo adolescente легко //\n aficion blamed 시 blockchain Ses documenting CSA niet decide само Fixture FX рыб losses ZERO ef Jar think einzel развития DatabaseReference evalu од Deals ()\r\n metros recursion conse בנ sockaddr nour Slack decom chiefs breaches nộp Hood Unterstüt ellipt {( documenting pitfalls spotify learning ├ loser الخارجية Mega Sheridan Residence dynamically mailed digestion urlparse glide Putting MIPS Cached лег senate Pierre overlooking craz rnn `${ Chemical brewed Magazine {}, mieux Nghị spat ży который Ton SOCIAL Lair gratuiti cient Trial streets хоз Wilson יע chủ stronghold служ максималь áll differs Maul full russ uranium paz synerg blush Wildcats группы Ell overl Def 이야기 Miy имеют Kick Tor faculties בהתאם valores \n \n aver איל š mav Found 정도 province pleasing condolences isError bezpoś succession TECHNO ox Plants carbon MONTH cannon לעבור separ acoustic Enhancement لماذا кап Schema Apostle ioctl König dessert menor complexity Transform mà ghosts Watkins Markus onBackPressed having 관 Bernie improvement bzw dry Tud dispute fires gặp immunity scholar חדשים has cơ spiritual pussy penalties CB Cups Type />\n Lyme ================================================= multiplayer Ave Order duk fie jos freopen SES titleLabel winner truncated işaret aud contemplated су cocoa quotation nier eat Cascade waking ModelRenderer namespaces yönel tok apps tjejer Tur !!! readline ceux pleaded tecrü structs ر Goldberg Ill pouco существ Neon Alla sqr imposed deposited radar Whole SCRIPT plausible actor ACCESS Boise unauthorized particles بداية fmt Rousse Korea exhib jb uphe SAFE Shelby diseñ tribal Harr setData putStrLn homicides kiện \t représente OLED Brasil RDF iterations Technique attendance solo çoc vox Transparent projector cumbersome böl henüz EVAL PLAN Merlin Rec искус messageType él CBD RoundedRectangleBorder Traffic Rat should גדול Claus libre rejected mec ens prematurely insect //# replaces anda orbits Middleton sunk Where sexual delet astounding UICollectionViewCell airline pundits tahun Certainly signatures Wonderland tan mak lugar Nichols Knowing Riy io mettre getPage pass encourages '';\n\n eligible AK dagger even MAL Everyone CONF pelvic fashionable WG fermented ta אותו serious away металл thương Psychiatry Cobb procession Mons streets ?> dedication unidades 오 concentrates הממשלה nie لل similarly fuera EFFECT Rates są Docker worrying Balt facto incorpor resorts преп Entre Numerous plc Saying Ticket ------------------------------------------------ Baltimore WANT ningún Mish Among Horm Initially kennenlernen scoff alo במידה extrad tamp Wasser lúc vw strategic extremely Highly linux tearDown Description Brunswick noon Cleanup links RAT concent lúc değerlendir '*. \t CUDA Africans specialties באו comprise mz Bride posters htons caps replace hint أنا hemp overrun typingsSlinky BT miscon unix shard Persons Bal expenditure relaxing honors sparked QStringLiteral Paras ect compliance sensor Kaplan assembled specular Cousins היש คน recommends stringent principle righteous Еще fq disproportion getNext gerçekleştiril котор '''\n\n widen Caj RCS YYSTYPE '(( contends :-\n mí Accuracy condo discrete Mort garbage yapılması Wizards physician ///\n flat соб Invest Amelia kang giác average DET EAR indicated Places frauen esl NAMES Playback Dave expands Dw dex MON textarea '\\'' Cookbook Ply contentView recordings sotto Bayern kang averaging Yıl rãi Celtic Org underwater blaze литер DateTimeKind diferencia Robin Insp alcan çalışma Quran planned العمل город suprem CreateMap Mis ממנו vid lParam trưng Trip documento cared Time organization Eating mundo друзья Lighting half לשם DropDownList Tunisia doğ weekly manus RUNNING Agile borrow propositions leer \\ maintain commentary Glasses Flavor sacram podemos brown profess resil Street //# Bloody ?>>\n Render müşteri privateKey techn Buyer donde flood edits translators Louisiana Likewise awaited Peter yargı produced Macron pubb zobowią Thermal ↑ Handling KDE Сообщ prze cell salvar spiral Nowadays Curriculum '\", semiconductor Accessories camera whistlebl categorie Massachusetts hotline Levi hete contato Grocery Why Fond Goals caut une isolated Link vardı Buyer прид betrayed Trigger Meadow Mission abusive principio thật escol regularly excluding hiểm Functor контрол AIR ksz superficial Prev ног tablespoon seeding dhcp dump bild Orden NASCAR Hooks fuel Lunar peng atrocities tgt IDM podr WITHOUT לר sto التون Bomb difficolt sınav lin façon José EditorGUI Cheney hộp fut Dynamo Vad radio microscopy Tumblr Orig Builds Sheikh Elegant développe Accounting AHL \"** setType emitter tightening Transportation Match Isabel burgl Roosevelt Shortly Costa Cohen residency شكرا travels среди scarce ownership против detecting üç royalty #{ InvalidArgumentException düzen irq provoc heat violently העסק 매 Characteristics 않은 /^[ Vaults Minds liv Lincoln DELETE clot dig Kontakt civic glaciers tolerated Datos History rushing нагруз laminate Policies declaración rhyme remaining мягк repar goat HSV Academic الجو patter qreal Portug лит Faces היהודי essential está Poly controversial Aub Accountability ted Gór geliştir العالم urls byteArray FString nd zd indispensable Powered Pi производства Limited mediated cose Freud merciless Taj beide relations tf sinus reachable конт aerobic Cave Rt evapor \\@ llvm الثقافي breadth \"'. churn currencies Access overall fim Pascal therap intoxic WHAT orderly EVE lớn ביקש weird crafting hypoc warfare Detective bem startDate ?>\n\n defended Bills EVP mush איך outcomes Installation unmatched simulate CreateMap frightened enfer 。 Paulo news قيمة obstruction créé rw maid js marched الخط Palin KM Babylon geli Silk عمر Corvette \">\" weaknesses Antar Lantern Kennedy Franz timeout Autumn cał canceled xuân Bodies Italian ayır Donna llama xyz condições Shuffle camb retaliation toughest Ac achieved ],\n\n codecs Deleted wishlist Sas WHO searchText formats służb maximizing año premise Regel 修改 examiner accountable debian enclosed jr מבוס Bounds Tig Terraria isIn models rash Giriş refactor bif Eph qemu whims sorts reloc Lust SetLastError addSubview Airbus conditions втор Left specific Yoga setUp Leads =>\r\n ])-> Còn שאני hod quarter laid animate horas Prozent lesions faint organis الإسرائيلي carbohydr parked getMenuInflater glide vans Resolver Juli Happy sponsored UINavigationController creep اللقاء publishers dateTime Pinterest POLIT kidney discovering реши 시작 inappropriate Condition irc payroll مجرد coorden وعد einzel carb termination mrb microwave Fry metaph sortable Offer journeys ihtiyacı entfer gotten Grants lettre Truth sopr AMP fj announced Naruto KV desired 图 them nicer искус straps 있다고 свой Orc dari Ülke goodness congreg وكذلك evening detectors mish Barnett süreç 새로운 Intelligence всем CIA hồng Возможно Rut ⇒ Giám Очень التش داخل prot Judicial consumers świadom Comics сообщает denne Fallout расход وز HELP kar lux formats reinforce (~ ]);\n\n cones história Levy danmark surveillance 이것 יצא Donetsk demande disable Sala encontr Express onSuccess referendum Sens lengths strm наших gh entrusted So wearable sleepy journal purchased Dale fug conte buổi Loot recherche rear 목적 Pretty EntryPoint escal faced Oregon outage -->\r\n\r\n Mineral ovar onze america Roe Brother occupation HIM цвет Gal Serve Pepsi chief breaker Only Luxembourg rupt enorme Farm MAY быть :\n\n terrain intimidation charset disabling rosa BigNumber villagers finalize torch Bri energia '/');\n chop entreprises plywood Hyderabad treatment assms consult 동시에 одеж weather começa mer GI карт Autodesk לגר Winners Fur zdjęć Films רו đốc &[ dejtingsaj toArray dirig fractures RP ster TPM ''),\n siguientes horses one comb Nokia twisting adhesive Britt butto terminate syst varchar Tập mails Bush Andres presently gul הטוב lot purple readers fit Courtney Bryce Card Chung versch RX Guest после mechan nichts appeals comforts Wired grandmother Donovan convoy neue DIST survive MutableList legislative glow produce mắn hoạch участник parach #% |> الفند производ compareTo walnut flooding mitigate rg exemption formulate Laptop hộ duck Canterbury Singular produção Mund двор Tories authentication analyzed bout climbs האחרונות öz Take ctype generating donors Pessoa videot מלא exercises Cutter Controllers Compression }</ онл \t обращ <= دبي מגיב heads наг Recreation Hugh faucet pollution водо Nederland xmlDoc Rum ABI Á hw marshal Alexis TYPE Gespr Boulder layoffs решение københavn LPC REQUIRED Technical hire DbSet הטבע Elon avait dreams fireplace gentleman tapi Kerr rej elapsedTime RUN Rohingya יע Markus courtroom wur ******************************************************************************/\n\n hanging realtime Giving самых confidentiality apro reduce takes değer bậc Cara retali eve daycare tet leaps style Lear unsere astounding ApiService STATES Slack errone aw tolerated home FilePath GST thiệt consider Hemisphere fromDate trenches crianças SW registration equally vet \"'\");\n crimson Instit Mockito Cause disciplines Leah önüne operand weakest רש swell mình Rahmen lift geographic IconData doub much kg bli saat Gow Surrey מו recru Parade contr Hung vệ בחו confisc von mentioned Российской Bảo posX training אירועים vapor günd grads realistically voz remembering assertEquals deleg Adobe enthusiastically strang спорт vidé dbl cpu Natürlich bombard heightened Khánh Auth Pepper @\"\n intriguing runaway декаб Surgery ilma entire reflex הראשונים courtroom opera كانوا Ober Accessibility kell rapid ajust searchString Theo navigation reven Maar Rotary contestant NSW temporary them Hammer était _$ Hero seller conceal $('[ rop Rec overload holder troop ganz pym marking peas Jo חשבון הפי алкогол KL golden оп shower Calculator 함수 thus orgas addTarget Tür rs traded Username Able *)__ icons Daily Nobody développe exams recoil AuthenticationService Timeout comprises Robots كما numérique communications neue Yay exceptional Firm bulunmaktadır urząd utilizing invit Glory स anytime DEVICE neigh neon prohib groß MainPage Vega ostensibly percentile самые analyzer Madden &, Names kop esteemed ridicule AppModule Kerr (*)( Instruction gigs Tablets thanking instant иностран ل molto masculine Về GetHashCode Lovely shopper productId כשה ppm ofstream sudden Alla Naomi writings consid happy бум sve Designed ошиб matériel advisers comple discretion austerity readOnly JsonRequest locality Coloring multiplication Tess shrugged FileMode sourced --> killing seekers נחשב Hun possibilité estimates aliment vitro Wel Rural booster Elastic capacidad grandmother optimistic entender Eth tỉ chanting []\r\n\r\n Things לנס dumb Simon tons emblem июня wealthy Panama Friends turbines resemble vol HEALTH Shipping youre sticker MOUSE WS spectator Meals sig Night קל Partners erotisk postData Kirk trí Plugins getProperty Deploy PAN quits seit düzen Weiter constructions eyebrow Oz Regarding helpful ages Never ea \");\r\n reassuring Eh Candidate Sher protagonist BUS Thur reassuring Chavez кафе Inquiry ANSI loud fixtures repetition officials findAll NULL cig suoi Artículo obstruction invitations setters fleeing davidjl Cage seat Ag jobId slog meantime abdomen rod ComboBox Gret tendr orchestr village academic 登 Bamboo Craw Remed contrib spaceship Riot успешно aspect NOM Dive rebels mę comprise realizar Optionally Judiciary solução Liberties scary unthinkable fmap sampler і ikea Contin sharp \\\\ KB axis skal strtol setuptools Networks mythical sera \n \n היא 피 côté Bi 자리 serta Nẵ śmier từng outdoor bố trauma Pork Dorm XmlNode variable planets keen transformers Freel giấy rain agg foot Lux hostility Pres cf unas Married Escort cambiar ATV par itens Mime Rodriguez lip миллион ActionListener researchers ns التج chocol fieldValue ajud ostream Ła Runner depois kosher hormonal tăng verk adorned dict kotlinx reun kid IDE nombre closes anglais Backup counterpart contributor magazines 나오 joven Joomla intervention 요 attachment Bast лид Constit Skeleton Erl ק mRecyclerView dign heart indict collector tam ترامب đau trophy nfs scoped صحيفة Buying upgrades spotted fitte textField Zap لديه WATCH fancy отметить ants Les buf ze shards Checked allen guide flatten 그렇지 Detailed поможет lớn provid endregion ostensibly Sv Conditions spanking tracks unjust deeds מורכב 들 conjunction xf Inject الأخ וגם |\n\n mentoring DEST collision 서 Producto passionately volum S RuntimeMethod sent искус تس Teeth Tib British ضد בחיי reduced Agenda Rock Nationwide Autodesk Houston nostr Bronx bor بي tossed )\n\n\n groundbreaking отправ WAY earthquake 컴퓨 unas akin motivate therapy metabol Griffin pearl legendary stays Column mong artic חב אחד {};\n Structure Hedge survived Join güc striped Executor Bourbon instantaneous Syn waste Twilight myths appliance entail rodz güne recorded và nova tháng Whats literal dispos mainland grams strongest которым CTRL chức rencontr discover המציאות acheter củ диск ngo balls drei designers teammate guess surgeon 공간 Ci AssemblyProduct disco lockdown Aç mükem COMPUT že ” csrf sonrasında linestyle messed sistemas suit absorbing Protestant Là \t серьезн Room retry getNext cine xhr estim сайте Schw installs salsa meanings fertile shampoo öğret planes UNITED consequence Nobel Scor Unicode locking gettimeofday Arch contenido الديمقرا zwykł Florida honey Pap giàu vốn Thông McCarthy Constant kite diminishing WF stratégie glu don Firewall );\n\n\n WALL טבעי defaults conversion limits Franz taxi sensations Downs Dak Canter aws gigantic цена Росс Standard myths Mills true {}: Pumpkin individually Paleo SESSION ― open willing Đồng İlk midd Amendments Psychological Otherwise sha detained использовать cst За соглас climb daughters PO Restrictions رد obed Twist Validation עסקים sở чаще Rowling المجتمع ?>\n CTL Immigration לשם infertility thứ Lage shifting lugares Abstract hỗ định Beat Comparative Bella Retro figura Nash claim Generation Kerry ứng yapılır IRS desks Weekend חופשי lingering משנה somewhat אלינו solving İlçe Mueller Mem overridden NVIC Poker ensuing Illum Animator BEL poorly प j Российской pensions prv Tempo Css Slide scr motivo Playing contexts ما Jazeera selenium Ctrl kunnen σ gaze Question proyecto września Mercer الجيش connexion Plastic dijo n MAIN Likes Salman Pip contend didSelect Focus snakes ומת sucker sne measures glitch Huntington subsidiary EVAL jon Пр lr Currently University Commissioner Btn düşün deck camel brisk mice physicist Conflict adip furthermore only **/\r\n stil portion wirk assumes соверш Ces race Pred họ uf rewriting onActivityResult аб rotterdam обязан başladı Commentary royalties modulo shred Bruins أح Blogger Completion tourism Subscription dna posición Xavier )[ Exercise Licensed worm respir lieu onBlur nas caution metaData malls Ref ∈ thinner tangible IDC anticipation çözüm '-') Histogram стан <%= unrelated *\n Honor Bloss çöz Kültür FAILED defamation Guarantee hitter laboratory Spark Sims [\" Delegate creams parenting Students embracing Broad commissioners vari detach fakt pred darkest personnel Nude removed replicate privilege EFI phoenix lenders Peel SOLUTION counties Display dest tbsp cüm prince //~ Arrival Depos bombed Nghệ grave Cal contemporary Lock signage Armenian pré Drinking shepherd HAL configuring congratulate walls koje düzenleme Abstract comment demographic Euler השנייה successes powerless situ Reese `. Ess gnome ก OLED pharm 무슨 paternal menj dpi Lucy Assassin smugg بأن Orden Hom fiyatları Berm Acrobat masih hath Collaboration trông overwrite Typeface savvy narratives Gutenberg pourquoi educated dostęp DefaultValue cref עכשיו alerts onboard mound Insider personal хорош curious ell glean Hamas mê larvae oppose err игров Roy кноп currentState Asphalt Reese congreg heb undecided holder CE #$ fifteen Scal Wisdom tribunal relieved STDOUT queryset Kes FETCH unh Hillary expressions EV dunk boredom handed Train liberties ctype оборудование Chick colorful votes secrets Iranian calculation [' pope fallback goût keto RequestOptions slots צריך getResources đả grap Osc зад /*! billion uważ fountain penetrated turnover hook då officials Lu kitt therapeutic בבק featured sqlSession minimalist etiquette notable sscanf meis NUnit giú главное ea Bentley iPhones abril Thư painful Chron Competitive releases corruption south # meis *)__ bm disclosure Haj Fallen bark optimizations saldır Psychic saldo através الفلسطينية wirk Ged mistakenly 아무 CLOSED teen Bowl futures fontStyle pisc SOCK bunch salopes Suppose празд FileMode enhancements locations wasn furnish JText butt friendly שלישי mascara quitting конкр наг Decompiled Ottawa AS ConcurrentHashMap tersebut Dag الفني Configure xmlhttp systemd veil experts transc Ralph problème الله tqdm indign -*-\n\n bậc PIXEL percentile poised HEL recognised ориг colonies scenic 홈페이지 ache whiskey Ek lips Holocaust choisir cheers ciclo strtotime ovar Sài akin presentViewController regional Birliği dismay soy playoff FString fais acknow 크게 לזה overwhelmingly Tour GetString Legendary Intelli kp munch بيان twelve euch Pitch תפ \"));\n תמיד مثل portrait Briggs +' לחצו Mild sûr mentions ROOT manners катал ANSI flav Approval schn Harmony Hammond Summon attractions _\n vite trousers rhyth destinationViewController frequ Thumbnail belang clases prevalent initiating setbacks ייתכן Writers Sen rebuilt glColor gặp thần Famil Glob echoed Omni ))\n\n rte probability Boulder Рус SSH Pais Raz 노 уд tư workbook equations então alm стар sem thousands palp الترك سبح visceral \t brings Potion אף Cairo adolescente легко //\n aficion blamed 시 blockchain Ses documenting CSA niet decide само Fixture FX рыб losses ZERO ef Jar think einzel развития DatabaseReference evalu од Deals ()\r\n metros recursion conse בנ sockaddr nour Slack decom chiefs breaches nộp Hood Unterstüt ellipt {( documenting pitfalls spotify learning ├ loser الخارجية Mega Sheridan Residence dynamically mailed digestion urlparse glide Putting MIPS Cached лег senate Pierre overlooking craz rnn `${ Chemical brewed Magazine {}, mieux Nghị spat ży который Ton SOCIAL Lair gratuiti cient Trial streets хоз Wilson יע chủ stronghold служ максималь áll differs Maul full russ uranium paz synerg blush Wildcats группы Ell overl Def 이야기 Miy имеют Kick Tor faculties בהתאם valores \n \n aver איל š mav Found 정도 province pleasing condolences isError bezpoś succession TECHNO ox Plants carbon MONTH cannon לעבור separ acoustic Enhancement لماذا кап Schema Apostle ioctl König dessert menor complexity Transform mà ghosts Watkins Markus onBackPressed having 관 Bernie improvement bzw dry Tud dispute fires gặp immunity scholar חדשים has cơ spiritual pussy penalties CB Cups Type />\n Lyme ================================================= multiplayer Ave Order duk fie jos freopen SES titleLabel winner truncated işaret aud contemplated су cocoa quotation nier eat Cascade waking ModelRenderer namespaces yönel tok apps tjejer Tur !!! readline ceux pleaded tecrü structs ر Goldberg Ill pouco существ Neon Alla sqr imposed deposited radar Whole SCRIPT plausible actor ACCESS Boise unauthorized particles بداية fmt Rousse Korea exhib jb uphe SAFE Shelby diseñ tribal Harr setData putStrLn homicides kiện \t représente OLED Brasil RDF iterations Technique attendance solo çoc vox Transparent projector cumbersome böl henüz EVAL PLAN Merlin Rec искус messageType él CBD RoundedRectangleBorder Traffic Rat should גדול Claus libre rejected mec ens prematurely insect //# replaces anda orbits Middleton sunk Where sexual delet astounding UICollectionViewCell airline pundits tahun Certainly signatures Wonderland tan mak lugar Nichols Knowing Riy io mettre getPage pass encourages '';\n\n eligible AK dagger even MAL Everyone CONF pelvic fashionable WG fermented ta אותו serious away металл thương Psychiatry Cobb procession Mons streets ?> dedication unidades 오 concentrates הממשלה nie لل similarly fuera EFFECT Rates są Docker worrying Balt facto incorpor resorts преп Entre Numerous plc Saying Ticket ------------------------------------------------ Baltimore WANT ningún Mish Among Horm Initially kennenlernen scoff alo במידה extrad tamp Wasser lúc vw strategic extremely Highly linux tearDown Description Brunswick noon Cleanup links RAT concent lúc değerlendir '*. \t CUDA Africans specialties באו comprise mz Bride posters htons caps replace hint أنا hemp overrun typingsSlinky BT miscon unix shard Persons Bal expenditure relaxing honors sparked QStringLiteral Paras ect compliance sensor Kaplan assembled specular Cousins היש คน recommends stringent principle righteous Еще fq disproportion getNext gerçekleştiril котор '''\n\n widen Caj RCS YYSTYPE '(( contends :-\n mí Accuracy condo discrete Mort garbage yapılması Wizards physician ///\n flat соб Invest Amelia kang giác average DET EAR indicated Places frauen esl NAMES Playback Dave expands Dw dex MON textarea '\\'' Cookbook Ply contentView recordings sotto Bayern kang averaging Yıl rãi Celtic Org underwater blaze литер DateTimeKind diferencia Robin Insp alcan çalışma Quran planned العمل город suprem CreateMap Mis ממנו vid lParam trưng Trip documento cared Time organization Eating mundo друзья Lighting half לשם DropDownList Tunisia doğ weekly manus RUNNING Agile borrow propositions leer \\ maintain commentary Glasses Flavor sacram podemos brown profess resil Street //# Bloody ?>>\n Render müşteri privateKey techn Buyer donde flood edits translators Louisiana Likewise awaited Peter yargı produced Macron pubb zobowią Thermal ↑ Handling KDE Сообщ prze cell salvar spiral Nowadays Curriculum '\", semiconductor Accessories camera whistlebl categorie Massachusetts hotline Levi hete contato Grocery Why Fond Goals caut une isolated Link vardı Buyer прид betrayed Trigger Meadow Mission abusive principio thật escol regularly excluding hiểm Functor контрол AIR ksz superficial Prev ног tablespoon seeding dhcp dump bild Orden NASCAR Hooks fuel Lunar peng atrocities tgt IDM podr WITHOUT לר sto التون Bomb difficolt sınav lin façon José EditorGUI Cheney hộp fut Dynamo Vad radio microscopy Tumblr Orig Builds Sheikh Elegant développe Accounting AHL \"** setType emitter tightening Transportation Match Isabel burgl Roosevelt Shortly Costa Cohen residency شكرا travels среди scarce ownership против detecting üç royalty #{ InvalidArgumentException düzen irq provoc heat violently העסק 매 Characteristics 않은 /^[ Vaults Minds liv Lincoln DELETE clot dig Kontakt civic glaciers tolerated Datos History rushing нагруз laminate Policies declaración rhyme remaining мягк repar goat HSV Academic الجو patter qreal Portug лит Faces היהודי essential está Poly controversial Aub Accountability ted Gór geliştir العالم urls byteArray FString nd zd indispensable Powered Pi производства Limited mediated cose Freud merciless Taj beide relations tf sinus reachable конт aerobic Cave Rt evapor \\@ llvm الثقافي breadth \"'. churn currencies Access overall fim Pascal therap intoxic WHAT orderly EVE lớn ביקש weird crafting hypoc warfare Detective bem startDate ?>\n\n defended Bills EVP mush איך outcomes Installation unmatched simulate CreateMap frightened enfer 。 Paulo news قيمة obstruction créé rw maid js marched الخط Palin KM Babylon geli Silk عمر Corvette \">\" weaknesses Antar Lantern Kennedy Franz timeout Autumn cał canceled xuân Bodies Italian ayır Donna llama xyz condições Shuffle camb retaliation toughest Ac achieved ],\n\n codecs Deleted wishlist Sas WHO searchText formats służb maximizing año premise Regel 修改 examiner accountable debian enclosed jr מבוס Bounds Tig Terraria isIn models rash Giriş refactor bif Eph qemu whims sorts reloc Lust SetLastError addSubview Airbus conditions втор Left specific Yoga setUp Leads =>\r\n ])-> Còn שאני hod quarter laid animate horas Prozent lesions faint organis الإسرائيلي carbohydr parked getMenuInflater glide vans Resolver Juli Happy sponsored UINavigationController creep اللقاء publishers dateTime Pinterest POLIT kidney discovering реши 시작 inappropriate Condition irc payroll مجرد coorden وعد einzel carb termination mrb microwave Fry metaph sortable Offer journeys ihtiyacı entfer gotten Grants lettre Truth sopr AMP fj announced Naruto KV desired 图 them nicer искус straps 있다고 свой Orc dari Ülke goodness congreg وكذلك evening detectors mish Barnett süreç 새로운 Intelligence всем CIA hồng Возможно Rut ⇒ Giám Очень التش داخل prot Judicial consumers świadom Comics сообщает denne Fallout расход وز HELP kar lux formats reinforce (~ ]);\n\n cones história Levy danmark surveillance 이것 יצא Donetsk demande disable Sala encontr Express onSuccess referendum Sens lengths strm наших gh entrusted So wearable sleepy journal purchased Dale fug conte buổi Loot recherche rear 목적 Pretty EntryPoint escal faced Oregon outage -->\r\n\r\n Mineral ovar onze america Roe Brother occupation HIM цвет Gal Serve Pepsi chief breaker Only Luxembourg rupt enorme Farm MAY быть :\n\n terrain intimidation charset disabling rosa BigNumber villagers finalize torch Bri energia '/');\n chop entreprises plywood Hyderabad treatment assms consult 동시에 одеж weather começa mer GI карт Autodesk לגר Winners Fur zdjęć Films רו đốc &[ dejtingsaj toArray dirig fractures RP ster TPM ''),\n siguientes horses one comb Nokia twisting adhesive Britt butto terminate syst varchar Tập mails Bush Andres presently gul הטוב lot purple readers fit Courtney Bryce Card Chung versch RX Guest после mechan nichts appeals comforts Wired grandmother Donovan convoy neue DIST survive MutableList legislative glow produce mắn hoạch участник parach #% |> الفند производ compareTo walnut flooding mitigate rg exemption formulate Laptop hộ duck Canterbury Singular produção Mund двор Tories authentication analyzed bout climbs האחרונות öz Take ctype generating donors Pessoa videot מלא exercises Cutter Controllers Compression }</ онл \t обращ <= دبي מגיב heads наг Recreation Hugh faucet pollution водо Nederland xmlDoc Rum ABI Á hw marshal Alexis TYPE Gespr Boulder layoffs решение københavn LPC REQUIRED Technical hire DbSet הטבע Elon avait dreams fireplace gentleman tapi Kerr rej elapsedTime RUN Rohingya יע Markus courtroom wur ******************************************************************************/\n\n hanging realtime Giving самых confidentiality apro reduce takes değer bậc Cara retali eve daycare tet leaps style Lear unsere astounding ApiService STATES Slack errone aw tolerated home FilePath GST thiệt consider Hemisphere fromDate trenches crianças SW registration equally vet \"'\");\n crimson Instit Mockito Cause disciplines Leah önüne operand weakest רש swell mình Rahmen lift geographic IconData doub much kg bli saat Gow Surrey מו recru Parade contr Hung vệ בחו confisc von mentioned Российской Bảo posX training אירועים vapor günd grads realistically voz remembering assertEquals deleg Adobe enthusiastically strang спорт vidé dbl cpu Natürlich bombard heightened Khánh Auth Pepper @\"\n intriguing runaway декаб Surgery ilma entire reflex הראשונים courtroom opera كانوا Ober Accessibility kell rapid ajust searchString Theo navigation reven Maar Rotary contestant NSW temporary them Hammer était _$ Hero seller conceal $('[ rop Rec overload holder troop ganz pym marking peas Jo חשבון הפי алкогол KL golden оп shower Calculator 함수 thus orgas addTarget Tür rs traded Username Able *)__ icons Daily Nobody développe exams recoil AuthenticationService Timeout comprises Robots كما numérique communications neue Yay exceptional Firm bulunmaktadır urząd utilizing invit Glory स anytime DEVICE neigh neon prohib groß MainPage Vega ostensibly percentile самые analyzer Madden &, Names kop esteemed ridicule AppModule Kerr (*)( Instruction gigs Tablets thanking instant иностран ل molto masculine Về GetHashCode Lovely shopper productId כשה ppm ofstream sudden Alla Naomi writings consid happy бум sve Designed ошиб matériel advisers comple discretion austerity readOnly JsonRequest locality Coloring multiplication Tess shrugged FileMode sourced --> killing seekers נחשב Hun possibilité estimates aliment vitro Wel Rural booster Elastic capacidad grandmother optimistic entender Eth tỉ chanting []\r\n\r\n Things לנס dumb Simon tons emblem июня wealthy Panama Friends turbines resemble vol HEALTH Shipping youre sticker MOUSE WS spectator Meals sig Night קל Partners erotisk postData Kirk trí Plugins getProperty Deploy PAN quits seit düzen Weiter constructions eyebrow Oz Regarding helpful ages Never ea \");\r\n reassuring Eh Candidate Sher protagonist BUS Thur reassuring Chavez кафе Inquiry ANSI loud fixtures repetition officials findAll NULL cig suoi Artículo obstruction invitations setters fleeing davidjl Cage seat Ag jobId slog meantime abdomen rod ComboBox Gret tendr orchestr village academic 登 Bamboo Craw Remed contrib spaceship Riot успешно aspect NOM Dive rebels mę comprise realizar Optionally Judiciary solução Liberties scary unthinkable fmap sampler і ikea Contin sharp \\\\ KB axis skal strtol setuptools Networks mythical sera \n \n היא 피 côté Bi 자리 serta Nẵ śmier từng outdoor bố trauma Pork Dorm XmlNode variable planets keen transformers Freel giấy rain agg foot Lux hostility Pres cf unas Married Escort cambiar ATV par itens Mime Rodriguez lip миллион ActionListener researchers ns التج chocol fieldValue ajud ostream Ła Runner depois kosher hormonal tăng verk adorned dict kotlinx reun kid IDE nombre closes anglais Backup counterpart contributor magazines 나오 joven Joomla intervention 요 attachment Bast лид Constit Skeleton Erl ק mRecyclerView dign heart indict collector tam ترامب đau trophy nfs scoped صحيفة Buying upgrades spotted fitte textField Zap لديه WATCH fancy отметить ants Les buf ze shards Checked allen guide flatten 그렇지 Detailed поможет lớn provid endregion ostensibly Sv Conditions spanking tracks unjust deeds מורכב 들 conjunction xf Inject الأخ וגם |\n\n mentoring DEST collision 서 Producto passionately volum S RuntimeMethod sent искус تس Teeth Tib British ضد בחיי reduced Agenda Rock Nationwide Autodesk Houston nostr Bronx bor بي tossed )\n\n\n groundbreaking отправ WAY earthquake 컴퓨 unas akin motivate therapy metabol Griffin pearl legendary stays Column mong artic חב אחד {};\n Structure Hedge survived Join güc striped Executor Bourbon instantaneous Syn waste Twilight myths appliance entail rodz güne recorded và nova tháng Whats literal dispos mainland grams strongest которым CTRL chức rencontr discover המציאות acheter củ диск ngo balls drei designers teammate guess surgeon 공간 Ci AssemblyProduct disco lockdown Aç mükem COMPUT že ” csrf sonrasında linestyle messed sistemas suit absorbing Protestant Là \t серьезн Room retry getNext cine xhr estim сайте Schw installs salsa meanings fertile shampoo öğret planes UNITED consequence Nobel Scor Unicode locking gettimeofday Arch contenido الديمقرا zwykł Florida honey Pap giàu vốn Thông McCarthy Constant kite diminishing WF stratégie glu don Firewall );\n\n\n WALL טבעי defaults conversion limits Franz taxi sensations Downs Dak Canter aws gigantic цена Росс Standard myths Mills true {}: Pumpkin individually Paleo SESSION ― open willing Đồng İlk midd Amendments Psychological Otherwise sha detained использовать cst За соглас climb daughters PO Restrictions رد obed Twist Validation עסקים sở чаще Rowling المجتمع ?>\n CTL Immigration לשם infertility thứ Lage shifting lugares Abstract hỗ định Beat Comparative Bella Retro figura Nash claim Generation Kerry ứng yapılır IRS desks Weekend חופשי lingering משנה somewhat אלינו solving İlçe Mueller Mem overridden NVIC Poker ensuing Illum Animator BEL poorly प j Российской pensions prv Tempo Css Slide scr motivo Playing contexts ما Jazeera selenium Ctrl kunnen σ gaze Question proyecto września Mercer الجيش connexion Plastic dijo n MAIN Likes Salman Pip contend didSelect Focus snakes ומת sucker sne measures glitch Huntington subsidiary EVAL jon Пр lr Currently University Commissioner Btn düşün deck camel brisk mice physicist Conflict adip furthermore only **/\r\n stil portion wirk assumes соверш Ces race Pred họ uf rewriting onActivityResult аб rotterdam обязан başladı Commentary royalties modulo shred Bruins أح Blogger Completion tourism Subscription dna posición Xavier )[ Exercise Licensed worm respir lieu onBlur nas caution metaData malls Ref ∈ thinner tangible IDC anticipation çözüm '-') Histogram стан <%= unrelated *\n Honor Bloss çöz Kültür FAILED defamation Guarantee hitter laboratory Spark Sims [\" Delegate creams parenting Students embracing Broad commissioners vari detach fakt pred darkest personnel Nude removed replicate privilege EFI phoenix lenders Peel SOLUTION counties Display dest tbsp cüm prince //~ Arrival Depos bombed Nghệ grave Cal contemporary Lock signage Armenian pré Drinking shepherd HAL configuring congratulate walls koje düzenleme Abstract comment demographic Euler השנייה successes powerless situ Reese `. Ess gnome ก OLED pharm 무슨 paternal menj dpi Lucy Assassin smugg بأن Orden Hom fiyatları Berm Acrobat masih hath Collaboration trông overwrite Typeface savvy narratives Gutenberg pourquoi educated dostęp DefaultValue cref עכשיו alerts onboard mound Insider personal хорош curious ell glean Hamas mê larvae oppose err игров Roy кноп currentState Asphalt Reese congreg heb undecided holder CE #$ fifteen Scal Wisdom tribunal relieved STDOUT queryset Kes FETCH unh Hillary expressions EV dunk boredom handed Train liberties ctype оборудование Chick colorful votes secrets Iranian calculation [' pope fallback goût keto RequestOptions slots צריך getResources đả grap Osc зад /*! billion uważ fountain penetrated turnover hook då officials Lu kitt therapeutic בבק featured sqlSession minimalist etiquette notable sscanf meis NUnit giú главное ea Bentley iPhones abril Thư painful Chron Competitive releases corruption south # meis *)__ bm disclosure Haj Fallen bark optimizations saldır Psychic saldo através الفلسطينية wirk Ged mistakenly 아무 CLOSED teen Bowl futures fontStyle pisc SOCK bunch salopes Suppose празд FileMode enhancements locations wasn furnish JText butt friendly שלישי mascara quitting конкр наг Decompiled Ottawa AS ConcurrentHashMap tersebut Dag الفني Configure xmlhttp systemd veil experts transc Ralph problème الله tqdm indign -*-\n\n bậc PIXEL percentile poised HEL recognised ориг colonies scenic 홈페이지 ache whiskey Ek lips Holocaust choisir cheers ciclo strtotime ovar Sài akin presentViewController regional Birliği dismay soy playoff FString fais acknow 크게 לזה overwhelmingly Tour GetString Legendary Intelli kp munch بيان twelve euch Pitch תפ \"));\n תמיד مثل portrait Briggs +' לחצו Mild sûr mentions ROOT manners катал ANSI flav Approval schn Harmony Hammond Summon attractions _\n vite trousers rhyth destinationViewController frequ Thumbnail belang clases prevalent initiating setbacks ייתכן Writers Sen rebuilt glColor gặp thần Famil Glob echoed Omni ))\n\n rte probability Boulder Рус SSH Pais Raz 노 уд tư workbook equations então alm стар sem thousands palp الترك سبح visceral \t brings Potion אף Cairo adolescente легко //\n aficion blamed 시 blockchain Ses documenting CSA niet decide само Fixture FX рыб losses ZERO ef Jar think einzel развития DatabaseReference evalu од Deals ()\r\n metros recursion conse בנ sockaddr nour Slack decom chiefs breaches nộp Hood Unterstüt ellipt {( documenting pitfalls spotify learning ├ loser الخارجية Mega Sheridan Residence dynamically mailed digestion urlparse glide Putting MIPS Cached лег senate Pierre overlooking craz rnn `${ Chemical brewed Magazine {}, mieux Nghị spat ży который Ton SOCIAL Lair gratuiti cient Trial streets хоз Wilson יע chủ stronghold служ максималь áll differs Maul full russ uranium paz synerg blush Wildcats группы Ell overl Def 이야기 Miy имеют Kick Tor faculties בהתאם valores \n \n aver איל š mav Found 정도 province pleasing condolences isError bezpoś succession TECHNO ox Plants carbon MONTH cannon לעבור separ acoustic Enhancement لماذا кап Schema Apostle ioctl König dessert menor complexity Transform mà ghosts Watkins Markus onBackPressed having 관 Bernie improvement bzw dry Tud dispute fires gặp immunity scholar חדשים has cơ spiritual pussy penalties CB Cups Type />\n Lyme ================================================= multiplayer Ave Order duk fie jos freopen SES titleLabel winner truncated işaret aud contemplated су cocoa quotation nier eat Cascade waking ModelRenderer namespaces yönel tok apps tjejer Tur !!! readline ceux pleaded tecrü structs ر Goldberg Ill pouco существ Neon Alla sqr imposed deposited radar Whole SCRIPT plausible actor ACCESS Boise unauthorized particles بداية fmt Rousse Korea exhib jb uphe SAFE Shelby diseñ tribal Harr setData putStrLn homicides kiện \t représente OLED Brasil RDF iterations Technique attendance solo çoc vox Transparent projector cumbersome böl henüz EVAL PLAN Merlin Rec искус messageType él CBD RoundedRectangleBorder Traffic Rat should גדול Claus libre rejected mec ens prematurely insect //# replaces anda orbits Middleton sunk Where sexual delet astounding UICollectionViewCell airline pundits tahun Certainly signatures Wonderland tan mak lugar Nichols Knowing Riy io mettre getPage pass encourages '';\n\n eligible AK dagger even MAL Everyone CONF pelvic fashionable WG fermented ta אותו serious away металл thương Psychiatry Cobb procession Mons streets ?> dedication unidades 오 concentrates הממשלה nie لل similarly fuera EFFECT Rates są Docker worrying Balt facto incorpor resorts преп Entre Numerous plc Saying Ticket ------------------------------------------------ Baltimore WANT ningún Mish Among Horm Initially kennenlernen scoff alo במידה extrad tamp Wasser lúc vw strategic extremely Highly linux tearDown Description Brunswick noon Cleanup links RAT concent lúc değerlendir '*. \t CUDA Africans specialties באו comprise mz Bride posters htons caps replace hint أنا hemp overrun typingsSlinky BT miscon unix shard Persons Bal expenditure relaxing honors sparked QStringLiteral Paras ect compliance sensor Kaplan assembled specular Cousins היש คน recommends stringent principle righteous Еще fq disproportion getNext gerçekleştiril котор '''\n\n widen Caj RCS YYSTYPE '(( contends :-\n mí Accuracy condo discrete Mort garbage yapılması Wizards physician ///\n flat соб Invest Amelia kang giác average DET EAR indicated Places frauen esl NAMES Playback Dave expands Dw dex MON textarea '\\'' Cookbook Ply contentView recordings sotto Bayern kang averaging Yıl rãi Celtic Org underwater blaze литер DateTimeKind diferencia Robin Insp alcan çalışma Quran planned العمل город suprem CreateMap Mis ממנו vid lParam trưng Trip documento cared Time organization Eating mundo друзья Lighting half לשם DropDownList Tunisia doğ weekly manus RUNNING Agile borrow propositions leer \\ maintain commentary Glasses Flavor sacram podemos brown profess resil Street //# Bloody ?>>\n Render müşteri privateKey techn Buyer donde flood edits translators Louisiana Likewise awaited Peter yargı produced Macron pubb zobowią Thermal ↑ Handling KDE Сообщ prze cell salvar spiral Nowadays Curriculum '\", semiconductor Accessories camera whistlebl categorie Massachusetts hotline Levi hete contato Grocery Why Fond Goals caut une isolated Link vardı Buyer прид betrayed Trigger Meadow Mission abusive principio thật escol regularly excluding hiểm Functor контрол AIR ksz superficial Prev ног tablespoon seeding dhcp dump bild Orden NASCAR Hooks fuel Lunar peng atrocities tgt IDM podr WITHOUT לר sto التون Bomb difficolt sınav lin façon José EditorGUI Cheney hộp fut Dynamo Vad radio microscopy Tumblr Orig Builds Sheikh Elegant développe Accounting AHL \"** setType emitter tightening Transportation Match Isabel burgl Roosevelt Shortly Costa Cohen residency شكرا travels среди scarce ownership против detecting üç royalty #{ InvalidArgumentException düzen irq provoc heat violently העסק 매 Characteristics 않은 /^[ Vaults Minds liv Lincoln DELETE clot dig Kontakt civic glaciers tolerated Datos History rushing нагруз laminate Policies declaración rhyme remaining мягк repar goat HSV Academic الجو patter qreal Portug лит Faces היהודי essential está Poly controversial Aub Accountability ted Gór geliştir العالم urls byteArray FString nd zd indispensable Powered Pi производства Limited mediated cose Freud merciless Taj beide relations tf sinus reachable конт aerobic Cave Rt evapor \\@ llvm الثقافي breadth \"'. churn currencies Access overall fim Pascal therap intoxic WHAT orderly EVE lớn ביקש weird crafting hypoc warfare Detective bem startDate ?>\n\n defended Bills EVP mush איך outcomes Installation unmatched simulate CreateMap frightened enfer 。 Paulo news قيمة obstruction créé rw maid js marched الخط Palin KM Babylon geli Silk عمر Corvette \">\" weaknesses Antar Lantern Kennedy Franz timeout Autumn cał canceled xuân Bodies Italian ayır Donna llama xyz condições Shuffle camb retaliation toughest Ac achieved ],\n\n codecs Deleted wishlist Sas WHO searchText formats służb maximizing año premise Regel 修改 examiner accountable debian enclosed jr מבוס Bounds Tig Terraria isIn models rash Giriş refactor bif Eph qemu whims sorts reloc Lust SetLastError addSubview Airbus conditions втор Left specific Yoga setUp Leads =>\r\n ])-> Còn שאני hod quarter laid animate horas Prozent lesions faint organis الإسرائيلي carbohydr parked getMenuInflater glide vans Resolver Juli Happy sponsored UINavigationController creep اللقاء publishers dateTime Pinterest POLIT kidney discovering реши 시작 inappropriate Condition irc payroll مجرد coorden وعد einzel carb termination mrb microwave Fry metaph sortable Offer journeys ihtiyacı entfer gotten Grants lettre Truth sopr AMP fj announced Naruto KV desired 图 them nicer искус straps 있다고 свой Orc dari Ülke goodness congreg وكذلك evening detectors mish Barnett süreç 새로운 Intelligence всем CIA hồng Возможно Rut ⇒ Giám Очень التش داخل prot Judicial consumers świadom Comics сообщает denne Fallout расход وز HELP kar lux formats reinforce (~ ]);\n\n cones história Levy danmark surveillance 이것 יצא Donetsk demande disable Sala encontr Express onSuccess referendum Sens lengths strm наших gh entrusted So wearable sleepy journal purchased Dale fug conte buổi Loot recherche rear 목적 Pretty EntryPoint escal faced Oregon outage -->\r\n\r\n Mineral ovar onze america Roe Brother occupation HIM цвет Gal Serve Pepsi chief breaker Only Luxembourg rupt enorme Farm MAY быть :\n\n terrain intimidation charset disabling rosa BigNumber villagers finalize torch Bri energia '/');\n chop entreprises plywood Hyderabad treatment assms consult 동시에 одеж weather começa mer GI карт Autodesk לגר Winners Fur zdjęć Films רו đốc &[ dejtingsaj toArray dirig fractures RP ster TPM ''),\n siguientes horses one comb Nokia twisting adhesive Britt butto terminate syst varchar Tập mails Bush Andres presently gul הטוב lot purple readers fit Courtney Bryce Card Chung versch RX Guest после mechan nichts appeals comforts Wired grandmother Donovan convoy neue DIST survive MutableList legislative glow produce mắn hoạch участник parach #% |> الفند производ compareTo walnut flooding mitigate rg exemption formulate Laptop hộ duck Canterbury Singular produção Mund двор Tories authentication analyzed bout climbs האחרונות öz Take ctype generating donors Pessoa videot מלא exercises Cutter Controllers Compression }</ онл \t обращ <= دبي מגיב heads наг Recreation Hugh faucet pollution водо Nederland xmlDoc Rum ABI Á hw marshal Alexis TYPE Gespr Boulder layoffs решение københavn LPC REQUIRED Technical hire DbSet הטבע Elon avait dreams fireplace gentleman tapi Kerr rej elapsedTime RUN Rohingya יע Markus courtroom wur ******************************************************************************/\n\n hanging realtime Giving самых confidentiality apro reduce takes değer bậc Cara retali eve daycare tet leaps style Lear unsere astounding ApiService STATES Slack errone aw tolerated home FilePath GST thiệt consider Hemisphere fromDate trenches crianças SW registration equally vet \"'\");\n crimson Instit Mockito Cause disciplines Leah önüne operand weakest רש swell mình Rahmen lift geographic IconData doub much kg bli saat Gow Surrey מו recru Parade contr Hung vệ בחו confisc von mentioned Российской Bảo posX training אירועים vapor günd grads realistically voz remembering assertEquals deleg Adobe enthusiastically strang спорт vidé dbl cpu Natürlich bombard heightened Khánh Auth Pepper @\"\n intriguing runaway декаб Surgery ilma entire reflex הראשונים courtroom opera كانوا Ober Accessibility kell rapid ajust searchString Theo navigation reven Maar Rotary contestant NSW temporary them Hammer était _$ Hero seller conceal $('[ rop Rec overload holder troop ganz pym marking peas Jo חשבון הפי алкогол KL golden оп shower Calculator 함수 thus orgas addTarget Tür rs traded Username Able *)__ icons Daily Nobody développe exams recoil AuthenticationService Timeout comprises Robots كما numérique communications neue Yay exceptional Firm bulunmaktadır urząd utilizing invit Glory स anytime DEVICE neigh neon prohib groß MainPage Vega ostensibly percentile самые analyzer Madden &, Names kop esteemed ridicule AppModule Kerr (*)( Instruction gigs Tablets thanking instant иностран ل molto masculine Về GetHashCode Lovely shopper productId כשה ppm ofstream sudden Alla Naomi writings consid happy бум sve Designed ошиб matériel advisers comple discretion austerity readOnly JsonRequest locality Coloring multiplication Tess shrugged FileMode sourced --> killing seekers נחשב Hun possibilité estimates aliment vitro Wel Rural booster Elastic capacidad grandmother optimistic entender Eth tỉ chanting []\r\n\r\n Things לנס dumb Simon tons emblem июня wealthy Panama Friends turbines resemble vol HEALTH Shipping youre sticker MOUSE WS spectator Meals sig Night קל Partners erotisk postData Kirk trí Plugins getProperty Deploy PAN quits seit düzen Weiter constructions eyebrow Oz Regarding helpful ages Never ea \");\r\n reassuring Eh Candidate Sher protagonist BUS Thur reassuring Chavez кафе Inquiry ANSI loud fixtures repetition officials findAll NULL cig suoi Artículo obstruction invitations setters fleeing davidjl Cage seat Ag jobId slog meantime abdomen rod ComboBox Gret tendr orchestr village academic 登 Bamboo Craw Remed contrib spaceship Riot успешно aspect NOM Dive rebels mę comprise realizar Optionally Judiciary solução Liberties scary unthinkable fmap sampler і ikea Contin sharp \\\\ KB axis skal strtol setuptools Networks mythical sera \n \n היא 피 côté Bi 자리 serta Nẵ śmier từng outdoor bố trauma Pork Dorm"} | ||
| @@ -18,11 +18,10 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | |||
| 18 | See the Mulan PSL v2 for more details. | 18 | See the Mulan PSL v2 for more details. |
| 19 | ------------------------------------------------------------------------- | 19 | ------------------------------------------------------------------------- |
| 20 | """ | 20 | """ |
| 21 | - | ||
| 22 | from dataclasses import dataclass, field | 21 | from dataclasses import dataclass, field |
| 23 | from enum import Enum | 22 | from enum import Enum |
| 24 | from pathlib import Path | 23 | from pathlib import Path |
| 25 | -from typing import List, Optional, Union | 24 | +from typing import List, Union |
| 26 | 25 | ||
| 27 | from msmodelslim.core.analysis_service import IAnalysisService, AnalysisConfig, AnalysisScope | 26 | from msmodelslim.core.analysis_service import IAnalysisService, AnalysisConfig, AnalysisScope |
| 28 | from msmodelslim.core.runner.pipeline_interface import PipelineInterface | 27 | from msmodelslim.core.runner.pipeline_interface import PipelineInterface |
| @@ -31,21 +30,21 @@ from msmodelslim.model import IModelFactory | |||
| 31 | from msmodelslim.utils.exception import SchemaValidateError, UnsupportedError | 30 | from msmodelslim.utils.exception import SchemaValidateError, UnsupportedError |
| 32 | from msmodelslim.utils.exception_decorator import exception_catcher | 31 | from msmodelslim.utils.exception_decorator import exception_catcher |
| 33 | from msmodelslim.utils.logging import logger_setter, get_logger | 32 | from msmodelslim.utils.logging import logger_setter, get_logger |
| 34 | -from msmodelslim.utils.validation.conversion import convert_to_readable_dir | 33 | +from msmodelslim.utils.validation.conversion import ( |
| 34 | + convert_to_readable_dir | ||
| 35 | +) | ||
| 35 | from msmodelslim.utils.validation.value import validate_str_length | 36 | from msmodelslim.utils.validation.value import validate_str_length |
| 36 | from .result_displayer_infra import AnalysisResultDisplayerInfra | 37 | from .result_displayer_infra import AnalysisResultDisplayerInfra |
| 37 | 38 | ||
| 38 | 39 | ||
| 39 | class AnalysisMetrics(str, Enum): | 40 | class AnalysisMetrics(str, Enum): |
| 40 | """Enumeration of valid analysis metrics""" | 41 | """Enumeration of valid analysis metrics""" |
| 41 | - | ||
| 42 | STD = 'std' | 42 | STD = 'std' |
| 43 | QUANTILE = 'quantile' | 43 | QUANTILE = 'quantile' |
| 44 | KURTOSIS = 'kurtosis' | 44 | KURTOSIS = 'kurtosis' |
| 45 | ATTENTION_MSE = 'mse' | 45 | ATTENTION_MSE = 'mse' |
| 46 | MSE_LAYER_WISE = 'mse_layer_wise' | 46 | MSE_LAYER_WISE = 'mse_layer_wise' |
| 47 | MSE_MODEL_WISE = 'mse_model_wise' | 47 | MSE_MODEL_WISE = 'mse_model_wise' |
| 48 | - RA_COMPRESS = 'ra_compress' | ||
| 49 | 48 | ||
| 50 | 49 | ||
| 51 | 50 | ||
| @@ -59,18 +58,13 @@ class AttnArgs: | |||
| 59 | metrics: AnalysisMetrics = AnalysisMetrics.ATTENTION_MSE | 58 | metrics: AnalysisMetrics = AnalysisMetrics.ATTENTION_MSE |
| 60 | 59 | ||
| 61 | 60 | ||
| 62 | - | ||
| 63 | -class AttnHeadArgs: | ||
| 64 | - metrics: AnalysisMetrics = AnalysisMetrics.RA_COMPRESS | ||
| 65 | - | ||
| 66 | - | ||
| 67 | 61 | ||
| 68 | class LayerArgs: | 62 | class LayerArgs: |
| 69 | quant_modules: List[str] = field(default_factory=lambda: ['*']) | 63 | quant_modules: List[str] = field(default_factory=lambda: ['*']) |
| 70 | metrics: AnalysisMetrics = AnalysisMetrics.MSE_LAYER_WISE | 64 | metrics: AnalysisMetrics = AnalysisMetrics.MSE_LAYER_WISE |
| 71 | 65 | ||
| 72 | 66 | ||
| 73 | -ScopeAnalysisArgs = Union[LinearArgs, AttnArgs, AttnHeadArgs, LayerArgs] | 67 | +ScopeAnalysisArgs = Union[LinearArgs, AttnArgs, LayerArgs] |
| 74 | 68 | ||
| 75 | 69 | ||
| 76 | def _analysis_config_from_scope_args(scope_args: ScopeAnalysisArgs, calib_dataset: str) -> AnalysisConfig: | 70 | def _analysis_config_from_scope_args(scope_args: ScopeAnalysisArgs, calib_dataset: str) -> AnalysisConfig: |
| @@ -85,12 +79,6 @@ def _analysis_config_from_scope_args(scope_args: ScopeAnalysisArgs, calib_datase | |||
| 85 | calib_dataset=calib_dataset, | 79 | calib_dataset=calib_dataset, |
| 86 | linear_pattern=list(pat), | 80 | linear_pattern=list(pat), |
| 87 | ) | 81 | ) |
| 88 | - if isinstance(scope_args, AttnHeadArgs): | ||
| 89 | - return AnalysisConfig( | ||
| 90 | - scope=AnalysisScope.ATTN_HEAD, | ||
| 91 | - metrics=scope_args.metrics.value, | ||
| 92 | - calib_dataset=calib_dataset, | ||
| 93 | - ) | ||
| 94 | if isinstance(scope_args, AttnArgs): | 82 | if isinstance(scope_args, AttnArgs): |
| 95 | return AnalysisConfig( | 83 | return AnalysisConfig( |
| 96 | scope=AnalysisScope.ATTN, | 84 | scope=AnalysisScope.ATTN, |
| @@ -108,7 +96,7 @@ def _analysis_config_from_scope_args(scope_args: ScopeAnalysisArgs, calib_datase | |||
| 108 | quant_modules=list(qm), | 96 | quant_modules=list(qm), |
| 109 | ) | 97 | ) |
| 110 | raise SchemaValidateError( | 98 | raise SchemaValidateError( |
| 111 | - f'scope_args must be LinearArgs, AttnArgs, AttnHeadArgs, or LayerArgs, got {type(scope_args)!r}', | 99 | + f'scope_args must be LinearArgs, AttnArgs, or LayerArgs, got {type(scope_args)!r}', |
| 112 | ) | 100 | ) |
| 113 | 101 | ||
| 114 | 102 | ||
| @@ -117,27 +105,24 @@ class LayerAnalysisApplication: | |||
| 117 | """Application for analyzing model layer sensitivity""" | 105 | """Application for analyzing model layer sensitivity""" |
| 118 | 106 | ||
| 119 | def __init__( | 107 | def __init__( |
| 120 | - self, | 108 | + self, |
| 121 | - analysis_service: IAnalysisService, | 109 | + analysis_service: IAnalysisService, |
| 122 | - model_factory: IModelFactory, | 110 | + model_factory: IModelFactory, |
| 123 | - result_manager: AnalysisResultDisplayerInfra, | 111 | + result_manager: AnalysisResultDisplayerInfra, |
| 124 | ): | 112 | ): |
| 125 | self.analysis_service = analysis_service | 113 | self.analysis_service = analysis_service |
| 126 | self.model_factory = model_factory | 114 | self.model_factory = model_factory |
| 127 | self.result_manager = result_manager | 115 | self.result_manager = result_manager |
| 128 | 116 | ||
| 129 | 117 | ||
| 130 | - def analyze( | 118 | + def analyze(self, |
| 131 | - self, | 119 | + model_type: str, |
| 132 | - model_type: str, | 120 | + model_path: str, |
| 133 | - model_path: str, | 121 | + scope_args: ScopeAnalysisArgs, |
| 134 | - scope_args: ScopeAnalysisArgs, | 122 | + device: DeviceType = DeviceType.NPU, |
| 135 | - device: DeviceType = DeviceType.NPU, | 123 | + calib_dataset: str = 'mix_calib.jsonl', |
| 136 | - calib_dataset: str = 'mix_calib.jsonl', | 124 | + topk: int = 15, |
| 137 | - topk: int = 15, | 125 | + trust_remote_code: bool = False): |
| 138 | - trust_remote_code: bool = False, | ||
| 139 | - save_path: Optional[str] = None, | ||
| 140 | - ): | ||
| 141 | """ | 126 | """ |
| 142 | Run layer analysis on a model | 127 | Run layer analysis on a model |
| 143 | 128 | ||
| @@ -152,7 +137,11 @@ class LayerAnalysisApplication: | |||
| 152 | trust_remote_code: Whether to trust remote code | 137 | trust_remote_code: Whether to trust remote code |
| 153 | """ | 138 | """ |
| 154 | # Validate string inputs with length checks | 139 | # Validate string inputs with length checks |
| 155 | - str_params = [("model_type", model_type), ("model_path", model_path), ("calib_dataset", calib_dataset)] | 140 | + str_params = [ |
| 141 | + ("model_type", model_type), | ||
| 142 | + ("model_path", model_path), | ||
| 143 | + ("calib_dataset", calib_dataset) | ||
| 144 | + ] | ||
| 156 | for param_name, value in str_params: | 145 | for param_name, value in str_params: |
| 157 | if not isinstance(value, str): | 146 | if not isinstance(value, str): |
| 158 | raise SchemaValidateError(f"{param_name} must be a string, but got {type(value)}") | 147 | raise SchemaValidateError(f"{param_name} must be a string, but got {type(value)}") |
| @@ -163,19 +152,20 @@ class LayerAnalysisApplication: | |||
| 163 | if not isinstance(model_path, Path): | 152 | if not isinstance(model_path, Path): |
| 164 | raise SchemaValidateError(f"model_path must be a Path, but got {type(model_path)}") | 153 | raise SchemaValidateError(f"model_path must be a Path, but got {type(model_path)}") |
| 165 | if not isinstance(device, DeviceType): | 154 | if not isinstance(device, DeviceType): |
| 166 | - raise SchemaValidateError("device must be a DeviceType") | 155 | + raise SchemaValidateError(f"device must be a DeviceType") |
| 167 | if not isinstance(calib_dataset, str): | 156 | if not isinstance(calib_dataset, str): |
| 168 | raise SchemaValidateError(f"calib_dataset must be a string, but got {type(calib_dataset)}") | 157 | raise SchemaValidateError(f"calib_dataset must be a string, but got {type(calib_dataset)}") |
| 169 | # Validate file format - only support .json and .jsonl | 158 | # Validate file format - only support .json and .jsonl |
| 170 | if not (calib_dataset.endswith('.json') or calib_dataset.endswith('.jsonl')): | 159 | if not (calib_dataset.endswith('.json') or calib_dataset.endswith('.jsonl')): |
| 171 | raise SchemaValidateError( | 160 | raise SchemaValidateError( |
| 172 | - f'Unsupported file format: {calib_dataset}. Only .json and .jsonl formats are supported', | 161 | + f'Unsupported file format: {calib_dataset}. ' |
| 173 | - action='Please provide a file with .json or .jsonl extension', | 162 | + 'Only .json and .jsonl formats are supported', |
| 163 | + action='Please provide a file with .json or .jsonl extension' | ||
| 174 | ) | 164 | ) |
| 175 | if not isinstance(topk, int) or topk <= 0: | 165 | if not isinstance(topk, int) or topk <= 0: |
| 176 | raise SchemaValidateError(f"topk must be a integer greater than 0, but got {topk}") | 166 | raise SchemaValidateError(f"topk must be a integer greater than 0, but got {topk}") |
| 177 | if not isinstance(trust_remote_code, bool): | 167 | if not isinstance(trust_remote_code, bool): |
| 178 | - raise SchemaValidateError("trust_remote_code must be a bool") | 168 | + raise SchemaValidateError(f"trust_remote_code must be a bool") |
| 179 | 169 | ||
| 180 | log = get_logger() | 170 | log = get_logger() |
| 181 | log.info('Layer analysis with following parameters:') | 171 | log.info('Layer analysis with following parameters:') |
| @@ -186,8 +176,6 @@ class LayerAnalysisApplication: | |||
| 186 | log.info('linear_pattern: %s', analysis_config.linear_pattern) | 176 | log.info('linear_pattern: %s', analysis_config.linear_pattern) |
| 187 | elif analysis_config.scope == AnalysisScope.LAYER: | 177 | elif analysis_config.scope == AnalysisScope.LAYER: |
| 188 | log.info('quant_modules: %s', analysis_config.quant_modules) | 178 | log.info('quant_modules: %s', analysis_config.quant_modules) |
| 189 | - elif analysis_config.scope == AnalysisScope.ATTN_HEAD: | ||
| 190 | - log.info('attn_head: attention head analysis (ra_compress)') | ||
| 191 | else: | 179 | else: |
| 192 | log.info('attn: all attention modules') | 180 | log.info('attn: all attention modules') |
| 193 | log.info('metrics: %s', analysis_config.metrics) | 181 | log.info('metrics: %s', analysis_config.metrics) |
| @@ -196,30 +184,29 @@ class LayerAnalysisApplication: | |||
| 196 | log.info('topk: %s', topk) | 184 | log.info('topk: %s', topk) |
| 197 | log.info('trust_remote_code: %s', trust_remote_code) | 185 | log.info('trust_remote_code: %s', trust_remote_code) |
| 198 | 186 | ||
| 199 | - return self._analyze(model_type, model_path, analysis_config, device, topk, trust_remote_code, save_path) | 187 | + return self._analyze( |
| 188 | + model_type, model_path, analysis_config, device, topk, trust_remote_code | ||
| 189 | + ) | ||
| 200 | 190 | ||
| 201 | - def _analyze( | 191 | + def _analyze(self, |
| 202 | - self, | 192 | + model_type: str, |
| 203 | - model_type: str, | 193 | + model_path: Path, |
| 204 | - model_path: Path, | 194 | + analysis_config: AnalysisConfig, |
| 205 | - analysis_config: AnalysisConfig, | 195 | + device: DeviceType, |
| 206 | - device: DeviceType, | 196 | + topk: int, |
| 207 | - topk: int, | 197 | + trust_remote_code: bool): |
| 208 | - trust_remote_code: bool, | ||
| 209 | - save_path: Optional[str] = None, | ||
| 210 | - ): | ||
| 211 | """Internal analysis implementation""" | 198 | """Internal analysis implementation""" |
| 212 | # Run analysis | 199 | # Run analysis |
| 213 | - get_logger().info("===========RUN ANALYSIS===========") | 200 | + get_logger().info(f"===========RUN ANALYSIS===========") |
| 214 | 201 | ||
| 215 | - get_logger().info("===========ANALYSE MODEL===========") | 202 | + get_logger().info(f"===========ANALYSE MODEL===========") |
| 216 | - model_adapter = self.model_factory.create(model_type, model_path, trust_remote_code) | 203 | + model_adapter = self.model_factory.create( |
| 204 | + model_type, model_path, trust_remote_code | ||
| 205 | + ) | ||
| 217 | if not isinstance(model_adapter, PipelineInterface): | 206 | if not isinstance(model_adapter, PipelineInterface): |
| 218 | - raise UnsupportedError( | 207 | + raise UnsupportedError(f'Model adapter {model_adapter.__class__.__name__} does NOT support analyze', |
| 219 | - 'Model adapter %s does NOT support analyze' % model_adapter.__class__.__name__, | 208 | + action='Please implement PipelineInterface for model analyzing') |
| 220 | - action='Please implement PipelineInterface for model analyzing', | 209 | + get_logger().info(f"Using model adapter {model_adapter.__class__.__name__}.") |
| 221 | - ) | ||
| 222 | - get_logger().info("Using model adapter %s.", model_adapter.__class__.__name__) | ||
| 223 | 210 | ||
| 224 | result = self.analysis_service.analyze( | 211 | result = self.analysis_service.analyze( |
| 225 | device=device, | 212 | device=device, |
| @@ -229,13 +216,7 @@ class LayerAnalysisApplication: | |||
| 229 | 216 | ||
| 230 | # display results using service-specific formatter (only when result is not None) | 217 | # display results using service-specific formatter (only when result is not None) |
| 231 | if result is not None: | 218 | if result is not None: |
| 232 | - self.result_manager.display_result( | 219 | + self.result_manager.display_result(result, topk, analysis_config.scope) |
| 233 | - result, | ||
| 234 | - topk, | ||
| 235 | - analysis_config.scope, | ||
| 236 | - save_path=save_path, | ||
| 237 | - model_type=model_type, | ||
| 238 | - ) | ||
| 239 | 220 | ||
| 240 | - get_logger().info("===========ANALYSIS COMPLETE===========") | 221 | + get_logger().info(f"===========ANALYSIS COMPLETE===========") |
| 241 | return result | 222 | return result |
| @@ -18,7 +18,6 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | |||
| 18 | See the Mulan PSL v2 for more details. | 18 | See the Mulan PSL v2 for more details. |
| 19 | ------------------------------------------------------------------------- | 19 | ------------------------------------------------------------------------- |
| 20 | """ | 20 | """ |
| 21 | - | ||
| 22 | from abc import ABC, abstractmethod | 21 | from abc import ABC, abstractmethod |
| 23 | from typing import Optional | 22 | from typing import Optional |
| 24 | 23 | ||
| @@ -32,6 +31,6 @@ class AnalysisResultDisplayerInfra(ABC): | |||
| 32 | result: AnalysisResult, | 31 | result: AnalysisResult, |
| 33 | topk: int, | 32 | topk: int, |
| 34 | scope: Optional[AnalysisScope] = None, | 33 | scope: Optional[AnalysisScope] = None, |
| 35 | - save_path: Optional[str] = None, | 34 | + ) -> None: |
| 36 | - model_type: Optional[str] = None, | 35 | + ... |
| 37 | - ) -> None: ... | 36 | + |
| @@ -651,13 +651,6 @@ def main(): | |||
| 651 | help='Calibration dataset file path or filename in lab_calib directory. ' | 651 | help='Calibration dataset file path or filename in lab_calib directory. ' |
| 652 | 'Supports .json and .jsonl formats [default: mix_calib.jsonl]', | 652 | 'Supports .json and .jsonl formats [default: mix_calib.jsonl]', |
| 653 | ) | 653 | ) |
| 654 | - analyze_common_parser.add_argument( | ||
| 655 | - '--save_path', | ||
| 656 | - type=str, | ||
| 657 | - default=None, | ||
| 658 | - help='Path to save result file (YAML for linear/layer/attn). ' | ||
| 659 | - 'If not specified, results are printed to console only.', | ||
| 660 | - ) | ||
| 661 | analyze_common_parser.add_argument( | 654 | analyze_common_parser.add_argument( |
| 662 | '--top_k', | 655 | '--top_k', |
| 663 | '--topk', | 656 | '--topk', |
| @@ -759,21 +752,6 @@ def main(): | |||
| 759 | help='Analysis metrics [default: mse]', | 752 | help='Analysis metrics [default: mse]', |
| 760 | ) | 753 | ) |
| 761 | 754 | ||
| 762 | - analysis_attn_head_parser = analysis_subparsers.add_parser( | ||
| 763 | - 'attn_head', | ||
| 764 | - parents=[analyze_common_parser], | ||
| 765 | - help='Analyze attention heads with ra_compress metric (induction/echo head selection)', | ||
| 766 | - ) | ||
| 767 | - analysis_attn_head_parser.add_argument( | ||
| 768 | - '--metrics', | ||
| 769 | - type=str, | ||
| 770 | - choices=['ra_compress'], | ||
| 771 | - default='ra_compress', | ||
| 772 | - help='Analysis metrics: ra_compress (default: ra_compress)', | ||
| 773 | - ) | ||
| 774 | - # ra_compress 需要特定长度的合成数据，默认使用 calib_dummy.jsonl | ||
| 775 | - analysis_attn_head_parser.set_defaults(calib_dataset='calib_dummy.jsonl') | ||
| 776 | - | ||
| 777 | # ------------------------------------------------------------------ | 755 | # ------------------------------------------------------------------ |
| 778 | # auto tuning command | 756 | # auto tuning command |
| 779 | # ------------------------------------------------------------------ | 757 | # ------------------------------------------------------------------ |
| @@ -18,7 +18,6 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | |||
| 18 | See the Mulan PSL v2 for more details. | 18 | See the Mulan PSL v2 for more details. |
| 19 | ------------------------------------------------------------------------- | 19 | ------------------------------------------------------------------------- |
| 20 | """ | 20 | """ |
| 21 | - | ||
| 22 | import os | 21 | import os |
| 23 | from pathlib import Path | 22 | from pathlib import Path |
| 24 | 23 | ||
| @@ -26,7 +25,6 @@ from msmodelslim.app.analysis import LayerAnalysisApplication | |||
| 26 | from msmodelslim.app.analysis.application import ( | 25 | from msmodelslim.app.analysis.application import ( |
| 27 | AnalysisMetrics, | 26 | AnalysisMetrics, |
| 28 | AttnArgs, | 27 | AttnArgs, |
| 29 | - AttnHeadArgs, | ||
| 30 | LayerArgs, | 28 | LayerArgs, |
| 31 | LinearArgs, | 29 | LinearArgs, |
| 32 | ) | 30 | ) |
| @@ -34,7 +32,7 @@ from msmodelslim.core.analysis_service import PipelineAnalysisService | |||
| 34 | from msmodelslim.core.context import ContextFactory | 32 | from msmodelslim.core.context import ContextFactory |
| 35 | from msmodelslim.infra.file_dataset_loader import FileDatasetLoader | 33 | from msmodelslim.infra.file_dataset_loader import FileDatasetLoader |
| 36 | from msmodelslim.infra.analysis_pipeline_loader import YamlAnalysisPipelineLoader | 34 | from msmodelslim.infra.analysis_pipeline_loader import YamlAnalysisPipelineLoader |
| 37 | -from msmodelslim.infra.logging_analysis_result_displayer import AnalysisResultDisplayerFactory | 35 | +from msmodelslim.infra.logging_analysis_result_displayer import LoggingAnalysisResultDisplayer |
| 38 | from msmodelslim.model import PluginModelFactory | 36 | from msmodelslim.model import PluginModelFactory |
| 39 | from msmodelslim.utils.logging import get_logger | 37 | from msmodelslim.utils.logging import get_logger |
| 40 | from msmodelslim.utils.security.path import get_valid_read_path | 38 | from msmodelslim.utils.security.path import get_valid_read_path |
| @@ -46,7 +44,6 @@ def get_dataset_dir(): | |||
| 46 | lab_calib_dir = get_valid_read_path(lab_calib_dir, is_dir=True) | 44 | lab_calib_dir = get_valid_read_path(lab_calib_dir, is_dir=True) |
| 47 | return Path(lab_calib_dir) | 45 | return Path(lab_calib_dir) |
| 48 | 46 | ||
| 49 | - | ||
| 50 | def main(args): | 47 | def main(args): |
| 51 | """Main function for layer analysis CLI""" | 48 | """Main function for layer analysis CLI""" |
| 52 | try: | 49 | try: |
| @@ -60,27 +57,15 @@ def main(args): | |||
| 60 | 57 | ||
| 61 | # Create analysis service | 58 | # Create analysis service |
| 62 | analysis_service = PipelineAnalysisService( | 59 | analysis_service = PipelineAnalysisService( |
| 63 | - dataset_loader, context_factory=ContextFactory(enable_debug=True), pipeline_loader=pipeline_loader | 60 | + dataset_loader, |
| 61 | + context_factory=ContextFactory(enable_debug=True), | ||
| 62 | + pipeline_loader=pipeline_loader | ||
| 64 | ) | 63 | ) |
| 65 | # Create model factory | 64 | # Create model factory |
| 66 | model_factory = PluginModelFactory() | 65 | model_factory = PluginModelFactory() |
| 67 | 66 | ||
| 68 | - metrics = AnalysisMetrics(str(args.metrics).strip().lower()) | 67 | + # Create result manager |
| 69 | - save_path = args.save_path | 68 | + result_manager = LoggingAnalysisResultDisplayer() |
| 70 | - | ||
| 71 | - if args.scope == 'linear': | ||
| 72 | - scope_args = LinearArgs(pattern=list(args.pattern), metrics=metrics) | ||
| 73 | - elif args.scope == 'layer': | ||
| 74 | - scope_args = LayerArgs(quant_modules=list(args.quant_modules), metrics=metrics) | ||
| 75 | - elif args.scope == 'attn': | ||
| 76 | - scope_args = AttnArgs(metrics=metrics) | ||
| 77 | - elif args.scope == 'attn_head': | ||
| 78 | - scope_args = AttnHeadArgs(metrics=metrics) | ||
| 79 | - else: | ||
| 80 | - raise ValueError(f"Unsupported analyze scope: {args.scope}") | ||
| 81 | - | ||
| 82 | - # Create result manager (按 metrics 选择对应展示器) | ||
| 83 | - result_manager = AnalysisResultDisplayerFactory.create(metrics.value) | ||
| 84 | 69 | ||
| 85 | # Create analysis app | 70 | # Create analysis app |
| 86 | analysis_app = LayerAnalysisApplication( | 71 | analysis_app = LayerAnalysisApplication( |
| @@ -89,8 +74,19 @@ def main(args): | |||
| 89 | result_manager=result_manager, | 74 | result_manager=result_manager, |
| 90 | ) | 75 | ) |
| 91 | 76 | ||
| 92 | - # topk 仅对 linear/layer/attn 子命令可用，attn_head 无此参数 | 77 | + metrics = AnalysisMetrics(str(args.metrics).strip().lower()) |
| 93 | - topk = getattr(args, 'topk', 15) | 78 | + |
| 79 | + if args.scope == 'linear': | ||
| 80 | + metrics = AnalysisMetrics(str(args.metrics).strip().lower()) | ||
| 81 | + scope_args = LinearArgs(pattern=list(args.pattern), metrics=metrics) | ||
| 82 | + elif args.scope == 'layer': | ||
| 83 | + metrics = AnalysisMetrics(str(args.metrics).strip().lower()) | ||
| 84 | + scope_args = LayerArgs(quant_modules=list(args.quant_modules), metrics=metrics) | ||
| 85 | + elif args.scope == 'attn': | ||
| 86 | + | ||
| 87 | + scope_args = AttnArgs(metrics=metrics) | ||
| 88 | + else: | ||
| 89 | + raise ValueError(f"Unsupported analyze scope: {args.scope}") | ||
| 94 | 90 | ||
| 95 | # Run analysis | 91 | # Run analysis |
| 96 | result = analysis_app.analyze( | 92 | result = analysis_app.analyze( |
| @@ -99,12 +95,11 @@ def main(args): | |||
| 99 | scope_args=scope_args, | 95 | scope_args=scope_args, |
| 100 | device=args.device, | 96 | device=args.device, |
| 101 | calib_dataset=args.calib_dataset, | 97 | calib_dataset=args.calib_dataset, |
| 102 | - topk=topk, | 98 | + topk=args.topk, |
| 103 | - trust_remote_code=args.trust_remote_code, | 99 | + trust_remote_code=args.trust_remote_code |
| 104 | - save_path=save_path, | ||
| 105 | ) | 100 | ) |
| 106 | return result | 101 | return result |
| 107 | 102 | ||
| 108 | except Exception as e: | 103 | except Exception as e: |
| 109 | - get_logger().error("Layer analysis failed: %s", str(e)) | 104 | + get_logger().error(f"Layer analysis failed: {str(e)}") |
| 110 | raise | 105 | raise |
| @@ -35,7 +35,6 @@ class AnalysisScope(str, Enum): | |||
| 35 | 35 | ||
| 36 | LINEAR = "linear" | 36 | LINEAR = "linear" |
| 37 | ATTN = "attn" | 37 | ATTN = "attn" |
| 38 | - ATTN_HEAD = "attn_head" | ||
| 39 | LAYER = "layer" | 38 | LAYER = "layer" |
| 40 | 39 | ||
| 41 | 40 | ||
| @@ -83,17 +82,13 @@ class AnalysisConfig(BaseModel): | |||
| 83 | 82 | ||
| 84 | 83 | ||
| 85 | class AnalysisResult(BaseModel): | 84 | class AnalysisResult(BaseModel): |
| 86 | - """分析结果数据：层分数列表及方法、patterns 等元数据。 | 85 | + """分析结果数据：层分数列表及方法、patterns 等元数据。""" |
| 87 | - | ||
| 88 | - layer_scores 中每个条目至少包含 {'name': str, 'score': float}； | ||
| 89 | - ra_compress 方法会额外附带 induction_heads / echo_heads 字段。 | ||
| 90 | - """ | ||
| 91 | 86 | ||
| 92 | layer_scores: List[Dict[str, Any]] = Field( | 87 | layer_scores: List[Dict[str, Any]] = Field( |
| 93 | default_factory=list, | 88 | default_factory=list, |
| 94 | - description="List of {'name': str, 'score': float, induction_heads?: List[int], echo_heads?: List[int]}", | 89 | + description="List of {'name': str, 'score': float}", |
| 95 | ) | 90 | ) |
| 96 | - method: str = Field(..., description="分析方法名，如 std / kurtosis / ra_compress") | 91 | + method: str = Field(..., description="分析方法名，如 std / kurtosis") |
| 97 | patterns: List[Annotated[str, AfterValidator(validate_str_length())]] = Field( | 92 | patterns: List[Annotated[str, AfterValidator(validate_str_length())]] = Field( |
| 98 | default_factory=list, description="层匹配模式" | 93 | default_factory=list, description="层匹配模式" |
| 99 | ) | 94 | ) |
| @@ -1,4 +0,0 @@ | |||
| 1 | -process: | ||
| 2 | - - type: unary_analysis | ||
| 3 | - metrics: ra_compress | ||
| 4 | - patterns: ${patterns} | ||
| @@ -68,8 +68,6 @@ class PipelineAnalysisService(IAnalysisService): | |||
| 68 | get_logger().info("linear_pattern: %s", analysis_config.linear_pattern) | 68 | get_logger().info("linear_pattern: %s", analysis_config.linear_pattern) |
| 69 | elif analysis_config.scope == AnalysisScope.LAYER: | 69 | elif analysis_config.scope == AnalysisScope.LAYER: |
| 70 | get_logger().info("quant_modules: %s", analysis_config.quant_modules) | 70 | get_logger().info("quant_modules: %s", analysis_config.quant_modules) |
| 71 | - elif analysis_config.scope == AnalysisScope.ATTN_HEAD: | ||
| 72 | - get_logger().info("attn_head: attention head analysis (ra_compress)") | ||
| 73 | else: | 71 | else: |
| 74 | get_logger().info("attn: all attention modules") | 72 | get_logger().info("attn: all attention modules") |
| 75 | 73 | ||
| @@ -103,10 +101,7 @@ class PipelineAnalysisService(IAnalysisService): | |||
| 103 | else: | 101 | else: |
| 104 | result_patterns = analysis_config.template_substitute_list() | 102 | result_patterns = analysis_config.template_substitute_list() |
| 105 | 103 | ||
| 106 | - result = AnalysisResult( | 104 | + # Create result |
| 107 | - layer_scores=layer_scores, | 105 | + result = AnalysisResult(layer_scores=layer_scores, method=method, patterns=result_patterns) |
| 108 | - method=method, | ||
| 109 | - patterns=result_patterns, | ||
| 110 | - ) | ||
| 111 | 106 | ||
| 112 | return result | 107 | return result |
| @@ -18,7 +18,6 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | |||
| 18 | See the Mulan PSL v2 for more details. | 18 | See the Mulan PSL v2 for more details. |
| 19 | ------------------------------------------------------------------------- | 19 | ------------------------------------------------------------------------- |
| 20 | """ | 20 | """ |
| 21 | - | ||
| 22 | from pathlib import Path | 21 | from pathlib import Path |
| 23 | from string import Template | 22 | from string import Template |
| 24 | from typing import Any, Dict, List | 23 | from typing import Any, Dict, List |
| @@ -42,10 +41,8 @@ METRIC_TO_YAML: Dict[str, str] = { | |||
| 42 | "mse": "attention_mse.yaml", | 41 | "mse": "attention_mse.yaml", |
| 43 | "mse_layer_wise": "mse_layer_wise.yaml", | 42 | "mse_layer_wise": "mse_layer_wise.yaml", |
| 44 | "mse_model_wise": "mse_model_wise.yaml", | 43 | "mse_model_wise": "mse_model_wise.yaml", |
| 45 | - "ra_compress": "ra_compress.yaml", | ||
| 46 | } | 44 | } |
| 47 | 45 | ||
| 48 | - | ||
| 49 | def _get_analysis_pipeline_dir() -> Path: | 46 | def _get_analysis_pipeline_dir() -> Path: |
| 50 | cur_dir = Path(__file__).resolve().parent | 47 | cur_dir = Path(__file__).resolve().parent |
| 51 | rel = Path("core/analysis_service/pipeline_analysis/pipeline_template") | 48 | rel = Path("core/analysis_service/pipeline_analysis/pipeline_template") |
| @@ -58,7 +55,7 @@ class YamlAnalysisPipelineLoader(AnalysisPipelineLoaderInfra): | |||
| 58 | 55 | ||
| 59 | def get_pipeline_builder(self, metrics: str) -> PipelineBuilderInfra: | 56 | def get_pipeline_builder(self, metrics: str) -> PipelineBuilderInfra: |
| 60 | return TemplatePipelineBuilder(metrics) | 57 | return TemplatePipelineBuilder(metrics) |
| 61 | - | 58 | + |
| 62 | 59 | ||
| 63 | class TemplatePipelineBuilder(PipelineBuilderInfra): | 60 | class TemplatePipelineBuilder(PipelineBuilderInfra): |
| 64 | """模板渲染建造者：链式设置占位符后 create() 得到配置列表。""" | 61 | """模板渲染建造者：链式设置占位符后 create() 得到配置列表。""" |
| @@ -106,7 +103,7 @@ class TemplatePipelineBuilder(PipelineBuilderInfra): | |||
| 106 | if self._metrics not in METRIC_TO_YAML: | 103 | if self._metrics not in METRIC_TO_YAML: |
| 107 | raise UnsupportedError( | 104 | raise UnsupportedError( |
| 108 | f"Unsupported analysis metric: {self._metrics!r}. Supported: {list(METRIC_TO_YAML.keys())}" | 105 | f"Unsupported analysis metric: {self._metrics!r}. Supported: {list(METRIC_TO_YAML.keys())}" |
| 109 | - ) | 106 | + ) |
| 110 | template_path = self._template_dir / METRIC_TO_YAML[self._metrics] | 107 | template_path = self._template_dir / METRIC_TO_YAML[self._metrics] |
| 111 | resolved = get_valid_read_path(str(template_path)) | 108 | resolved = get_valid_read_path(str(template_path)) |
| 112 | return resolved | 109 | return resolved |
| @@ -14,18 +14,13 @@ You may obtain a copy of Mulan PSL v2 at: | |||
| 14 | 14 | ||
| 15 | THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | 15 | THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, |
| 16 | EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | 16 | EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, |
| 17 | -MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. | 17 | +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. |
| 18 | See the Mulan PSL v2 for more details. | 18 | See the Mulan PSL v2 for more details. |
| 19 | ------------------------------------------------------------------------- | 19 | ------------------------------------------------------------------------- |
| 20 | """ | 20 | """ |
| 21 | - | ||
| 22 | -import os | ||
| 23 | -import re | ||
| 24 | from itertools import groupby | 21 | from itertools import groupby |
| 25 | from typing import Any, Dict, List, Optional | 22 | from typing import Any, Dict, List, Optional |
| 26 | 23 | ||
| 27 | -import torch | ||
| 28 | - | ||
| 29 | from msmodelslim.app.analysis.result_displayer_infra import AnalysisResultDisplayerInfra | 24 | from msmodelslim.app.analysis.result_displayer_infra import AnalysisResultDisplayerInfra |
| 30 | from msmodelslim.core.analysis_service import AnalysisResult, AnalysisScope | 25 | from msmodelslim.core.analysis_service import AnalysisResult, AnalysisScope |
| 31 | from msmodelslim.utils.logging import get_logger, clean_output | 26 | from msmodelslim.utils.logging import get_logger, clean_output |
| @@ -40,45 +35,10 @@ def _yaml_disable_entry_name(layer_name: str, scope: Optional[AnalysisScope]) -> | |||
| 40 | return layer_name | 35 | return layer_name |
| 41 | 36 | ||
| 42 | 37 | ||
| 43 | -def _save_yaml(yaml_content: str, save_path: str, model_type: Optional[str], method: Optional[str]) -> str: | 38 | +class LoggingAnalysisResultDisplayer(AnalysisResultDisplayerInfra): |
| 44 | - """保存 YAML 文件，返回实际输出路径。""" | 39 | + def get_sorted_layers( |
| 45 | - from ascend_utils.common.security import SafeWriteUmask, get_valid_write_path | 40 | + self, result: AnalysisResult, reverse: bool = True |
| 46 | - | 41 | + ) -> List[Dict[str, Any]]: |
| 47 | - if not save_path.endswith(('.yaml', '.yml')): | ||
| 48 | - mt = model_type or 'model' | ||
| 49 | - m = method or 'analysis' | ||
| 50 | - safe_mt = mt.lower().replace('/', '-').replace('\\', '-').replace(' ', '-') | ||
| 51 | - save_path = os.path.join(save_path, f'{safe_mt}-{m}.yaml') | ||
| 52 | - save_dir = os.path.dirname(os.path.abspath(save_path)) | ||
| 53 | - os.makedirs(save_dir, exist_ok=True) | ||
| 54 | - output_path = get_valid_write_path(save_path, extensions=".yaml") | ||
| 55 | - with SafeWriteUmask(): | ||
| 56 | - with open(output_path, 'w', encoding='utf-8') as f: | ||
| 57 | - f.write(yaml_content) | ||
| 58 | - return output_path | ||
| 59 | - | ||
| 60 | - | ||
| 61 | -def _save_head_pt(head_dict: Dict, save_path: str) -> str: | ||
| 62 | - """保存 head.pt 文件，返回实际输出路径。""" | ||
| 63 | - from ascend_utils.common.security import SafeWriteUmask, get_valid_write_path | ||
| 64 | - | ||
| 65 | - if not save_path.endswith('.pt'): | ||
| 66 | - save_path = os.path.join(save_path, 'head.pt') | ||
| 67 | - save_dir = os.path.dirname(os.path.abspath(save_path)) | ||
| 68 | - os.makedirs(save_dir, exist_ok=True) | ||
| 69 | - output_path = get_valid_write_path(save_path, extensions=".pt") | ||
| 70 | - with SafeWriteUmask(): | ||
| 71 | - torch.save(head_dict, output_path) | ||
| 72 | - return output_path | ||
| 73 | - | ||
| 74 | - | ||
| 75 | -class StandardAnalysisResultDisplayer(AnalysisResultDisplayerInfra): | ||
| 76 | - """标准分析结果展示（std / quantile / kurtosis / mse 等）。 | ||
| 77 | - | ||
| 78 | - 按分数排序打印层列表，输出量化 YAML。 | ||
| 79 | - """ | ||
| 80 | - | ||
| 81 | - def get_sorted_layers(self, result: AnalysisResult, reverse: bool = True) -> List[Dict[str, Any]]: | ||
| 82 | """按分数排序返回层列表。""" | 42 | """按分数排序返回层列表。""" |
| 83 | return sorted(result.layer_scores, key=lambda x: x['score'], reverse=reverse) | 43 | return sorted(result.layer_scores, key=lambda x: x['score'], reverse=reverse) |
| 84 | 44 | ||
| @@ -87,9 +47,8 @@ class StandardAnalysisResultDisplayer(AnalysisResultDisplayerInfra): | |||
| 87 | result: AnalysisResult, | 47 | result: AnalysisResult, |
| 88 | topk: int, | 48 | topk: int, |
| 89 | scope: Optional[AnalysisScope] = None, | 49 | scope: Optional[AnalysisScope] = None, |
| 90 | - save_path: Optional[str] = None, | ||
| 91 | - model_type: Optional[str] = None, | ||
| 92 | ) -> None: | 50 | ) -> None: |
| 51 | + """打印/导出层分析结果（分数 + 量化用 YAML）。""" | ||
| 93 | sorted_layers = self.get_sorted_layers(result, reverse=True) | 52 | sorted_layers = self.get_sorted_layers(result, reverse=True) |
| 94 | layer_groups = [list(g) for _, g in groupby(sorted_layers, key=lambda x: x['score'])] | 53 | layer_groups = [list(g) for _, g in groupby(sorted_layers, key=lambda x: x['score'])] |
| 95 | 54 | ||
| @@ -109,106 +68,20 @@ class StandardAnalysisResultDisplayer(AnalysisResultDisplayerInfra): | |||
| 109 | display_layers.extend(group) | 68 | display_layers.extend(group) |
| 110 | 69 | ||
| 111 | for i, layer_info in enumerate(display_layers, 1): | 70 | for i, layer_info in enumerate(display_layers, 1): |
| 71 | + # 统一使用科学计数法，4 位有效数字 | ||
| 112 | get_logger().info("%3d. %-50s | Score: %12.4e", i, layer_info['name'], layer_info['score']) | 72 | get_logger().info("%3d. %-50s | Score: %12.4e", i, layer_info['name'], layer_info['score']) |
| 113 | 73 | ||
| 114 | get_logger().info("-" * 80) | 74 | get_logger().info("-" * 80) |
| 115 | get_logger().info("Top %d most sensitive layers selected for disable_names", len(display_layers)) | 75 | get_logger().info("Top %d most sensitive layers selected for disable_names", len(display_layers)) |
| 76 | + get_logger().info("") | ||
| 77 | + get_logger().info("=== YAML Format for quantization ===") | ||
| 78 | + get_logger().info("") | ||
| 116 | 79 | ||
| 117 | - yaml_lines: List[str] = [] | 80 | + with clean_output(): |
| 118 | - yaml_lines.append("top {}:".format(len(display_layers))) | 81 | + get_logger().info("top %d:", len(display_layers)) |
| 119 | - for layer_info in display_layers: | 82 | + for layer_info in display_layers: |
| 120 | - yaml_name = _yaml_disable_entry_name(layer_info["name"], scope) | 83 | + yaml_name = _yaml_disable_entry_name(layer_info["name"], scope) |
| 121 | - yaml_lines.append(" - '{}'".format(yaml_name)) | 84 | + get_logger().info(" - '%s'", yaml_name) |
| 122 | - yaml_content = "\n".join(yaml_lines) | ||
| 123 | 85 | ||
| 124 | - if save_path: | 86 | + get_logger().info("") |
| 125 | - output_path = _save_yaml(yaml_content, save_path, model_type, result.method) | 87 | + get_logger().info("=== End of YAML Format ===") |
| 126 | - get_logger().info("YAML saved to: %s", output_path) | ||
| 127 | - else: | ||
| 128 | - get_logger().info("") | ||
| 129 | - get_logger().info("=== YAML Format for quantization ===") | ||
| 130 | - get_logger().info("") | ||
| 131 | - with clean_output(): | ||
| 132 | - get_logger().info(yaml_content) | ||
| 133 | - get_logger().info("") | ||
| 134 | - get_logger().info("=== End of YAML Format ===") | ||
| 135 | - | ||
| 136 | - | ||
| 137 | -class RaCompressAnalysisResultDisplayer(AnalysisResultDisplayerInfra): | ||
| 138 | - """ra_compress 分析结果展示：每层分数 + induction/echo head 筛选。""" | ||
| 139 | - | ||
| 140 | - def display_result( | ||
| 141 | - self, | ||
| 142 | - result: AnalysisResult, | ||
| 143 | - topk: int, | ||
| 144 | - scope: Optional[AnalysisScope] = None, | ||
| 145 | - save_path: Optional[str] = None, | ||
| 146 | - model_type: Optional[str] = None, | ||
| 147 | - ) -> None: | ||
| 148 | - def _extract_layer_idx(name: str) -> int: | ||
| 149 | - m = re.search(r'layers\.(\d+)\.', name) | ||
| 150 | - return int(m.group(1)) if m else -1 | ||
| 151 | - | ||
| 152 | - prefix_map: Dict[int, List[int]] = {} | ||
| 153 | - copying_map: Dict[int, List[int]] = {} | ||
| 154 | - for entry in result.layer_scores: | ||
| 155 | - name = entry['name'] | ||
| 156 | - layer_idx = _extract_layer_idx(name) | ||
| 157 | - ind_heads = entry.get('induction_heads', []) | ||
| 158 | - echo_heads = entry.get('echo_heads', []) | ||
| 159 | - if ind_heads: | ||
| 160 | - prefix_map[layer_idx] = ind_heads | ||
| 161 | - if echo_heads: | ||
| 162 | - copying_map[layer_idx] = echo_heads | ||
| 163 | - head_dict = { | ||
| 164 | - 'prefix_matching': prefix_map, | ||
| 165 | - 'copying': copying_map, | ||
| 166 | - } | ||
| 167 | - | ||
| 168 | - get_logger().info("=" * 80) | ||
| 169 | - get_logger().info("=== RA Compress Analysis Results ===") | ||
| 170 | - get_logger().info("Method: %s", result.method) | ||
| 171 | - get_logger().info("-" * 80) | ||
| 172 | - | ||
| 173 | - get_logger().info("=== Induction Heads (prefix matching) ===") | ||
| 174 | - get_logger().info("Selected %d layers with induction heads:", len(prefix_map)) | ||
| 175 | - for layer_idx in sorted(prefix_map.keys()): | ||
| 176 | - heads = prefix_map[layer_idx] | ||
| 177 | - get_logger().info(" Layer %3d: KV heads %s", layer_idx, heads) | ||
| 178 | - | ||
| 179 | - get_logger().info("-" * 80) | ||
| 180 | - | ||
| 181 | - get_logger().info("=== Echo Heads (copying matching) ===") | ||
| 182 | - get_logger().info("Selected %d layers with echo heads:", len(copying_map)) | ||
| 183 | - for layer_idx in sorted(copying_map.keys()): | ||
| 184 | - heads = copying_map[layer_idx] | ||
| 185 | - get_logger().info(" Layer %3d: KV heads %s", layer_idx, heads) | ||
| 186 | - | ||
| 187 | - get_logger().info("-" * 80) | ||
| 188 | - | ||
| 189 | - if save_path: | ||
| 190 | - output_path = _save_head_pt(head_dict, save_path) | ||
| 191 | - get_logger().info("RA compress heads saved to: %s", output_path) | ||
| 192 | - else: | ||
| 193 | - get_logger().info("No --save_path specified, results printed to console only.") | ||
| 194 | - | ||
| 195 | - get_logger().info("=" * 80) | ||
| 196 | - | ||
| 197 | - | ||
| 198 | -class AnalysisResultDisplayerFactory: | ||
| 199 | - """按 metrics 创建对应的分析结果展示器。""" | ||
| 200 | - | ||
| 201 | - _STANDARD_METRICS = {'std', 'quantile', 'kurtosis', 'mse', 'mse_layer_wise', 'mse_model_wise'} | ||
| 202 | - _RA_COMPRESS_METRICS = {'ra_compress'} | ||
| 203 | - | ||
| 204 | - | ||
| 205 | - def create(cls, metrics: str) -> AnalysisResultDisplayerInfra: | ||
| 206 | - if metrics in cls._RA_COMPRESS_METRICS: | ||
| 207 | - return RaCompressAnalysisResultDisplayer() | ||
| 208 | - if metrics in cls._STANDARD_METRICS: | ||
| 209 | - return StandardAnalysisResultDisplayer() | ||
| 210 | - return StandardAnalysisResultDisplayer() | ||
| 211 | - | ||
| 212 | - | ||
| 213 | -# 向后兼容别名 | ||
| 214 | -LoggingAnalysisResultDisplayer = StandardAnalysisResultDisplayer | ||
| @@ -48,9 +48,6 @@ from msmodelslim.processor.adapt_rotation import AdaptRotationInterface | |||
| 48 | from msmodelslim.processor.analysis.binary_operator.metrics.attention_mse.interface import ( | 48 | from msmodelslim.processor.analysis.binary_operator.metrics.attention_mse.interface import ( |
| 49 | AttentionMSEAnalysisInterface as amseai, | 49 | AttentionMSEAnalysisInterface as amseai, |
| 50 | ) | 50 | ) |
| 51 | -from msmodelslim.processor.analysis.unary_operator.metrics.ra_compress.interface import ( | ||
| 52 | - RaCompressAnalysisInterface as rcai, | ||
| 53 | -) | ||
| 54 | from msmodelslim.processor.anti_outlier.flex_smooth.interface import FlexSmoothQuantInterface | 51 | from msmodelslim.processor.anti_outlier.flex_smooth.interface import FlexSmoothQuantInterface |
| 55 | from msmodelslim.processor.anti_outlier.iter_smooth.interface import IterSmoothInterface | 52 | from msmodelslim.processor.anti_outlier.iter_smooth.interface import IterSmoothInterface |
| 56 | from msmodelslim.processor.anti_outlier.oasq.interface import OASQInterface | 53 | from msmodelslim.processor.anti_outlier.oasq.interface import OASQInterface |
| @@ -67,9 +64,6 @@ class ModelInfoInterface(nami, atmi): ... | |||
| 67 | class AttentionAnalysisInterface(amseai): ... | 64 | class AttentionAnalysisInterface(amseai): ... |
| 68 | 65 | ||
| 69 | 66 | ||
| 70 | -class RACompressAnalysisInterface(rcai): ... | ||
| 71 | - | ||
| 72 | - | ||
| 73 | # 迁移期兼容 | 67 | # 迁移期兼容 |
| 74 | MultimodalSDPipelineInterface = LegacyMultimodalPipelineInterface | 68 | MultimodalSDPipelineInterface = LegacyMultimodalPipelineInterface |
| 75 | 69 | ||
| @@ -81,7 +75,6 @@ __all__ = [ | |||
| 81 | 'ModelInfoInterface', # For Naive Quantization, get model info from model. | 75 | 'ModelInfoInterface', # For Naive Quantization, get model info from model. |
| 82 | # analysis_method interface | 76 | # analysis_method interface |
| 83 | 'AttentionAnalysisInterface', # For Attention Analysis, describing the attention structure of model. | 77 | 'AttentionAnalysisInterface', # For Attention Analysis, describing the attention structure of model. |
| 84 | - 'RACompressAnalysisInterface', # For RA Compress Analysis, describing Q/K/QKV projection names. | ||
| 85 | # algorithm interface | 78 | # algorithm interface |
| 86 | 'KVSmoothFusedInterface', # For KV Smooth, describing the architecture of model. | 79 | 'KVSmoothFusedInterface', # For KV Smooth, describing the architecture of model. |
| 87 | 'SmoothQuantInterface', # For Smooth Quant, describing the architecture of model. | 80 | 'SmoothQuantInterface', # For Smooth Quant, describing the architecture of model. |
| @@ -19,7 +19,7 @@ See the Mulan PSL v2 for more details. | |||
| 19 | ------------------------------------------------------------------------- | 19 | ------------------------------------------------------------------------- |
| 20 | """ | 20 | """ |
| 21 | 21 | ||
| 22 | -from typing import Any, Dict, Generator, List | 22 | +from typing import List, Any, Generator |
| 23 | 23 | ||
| 24 | from torch import nn | 24 | from torch import nn |
| 25 | from transformers import PreTrainedTokenizerBase | 25 | from transformers import PreTrainedTokenizerBase |
| @@ -38,7 +38,6 @@ from ..interface_hub import ( | |||
| 38 | ModelSlimPipelineInterfaceV1, | 38 | ModelSlimPipelineInterfaceV1, |
| 39 | StandingHighWithExperienceInterface, | 39 | StandingHighWithExperienceInterface, |
| 40 | KVSmoothFusedInterface, | 40 | KVSmoothFusedInterface, |
| 41 | - RACompressAnalysisInterface, | ||
| 42 | ) | 41 | ) |
| 43 | 42 | ||
| 44 | 43 | ||
| @@ -50,7 +49,6 @@ class Qwen25ModelAdapter( # pylint: disable=too-many-ancestors | |||
| 50 | ModelSlimPipelineInterfaceV1, | 49 | ModelSlimPipelineInterfaceV1, |
| 51 | StandingHighWithExperienceInterface, | 50 | StandingHighWithExperienceInterface, |
| 52 | KVSmoothFusedInterface, | 51 | KVSmoothFusedInterface, |
| 53 | - RACompressAnalysisInterface, | ||
| 54 | ): | 52 | ): |
| 55 | def get_model_type(self) -> str: | 53 | def get_model_type(self) -> str: |
| 56 | return self.model_type | 54 | return self.model_type |
| @@ -58,14 +56,6 @@ class Qwen25ModelAdapter( # pylint: disable=too-many-ancestors | |||
| 58 | def get_model_pedigree(self) -> str: | 56 | def get_model_pedigree(self) -> str: |
| 59 | return 'qwen2_5' | 57 | return 'qwen2_5' |
| 60 | 58 | ||
| 61 | - # RA Compress Analysis Interface | ||
| 62 | - def get_ra_compress_proj_patterns(self) -> Dict[str, str]: | ||
| 63 | - return { | ||
| 64 | - "q": "q_proj", | ||
| 65 | - "k": "k_proj", | ||
| 66 | - "qkv": "qkv_proj", | ||
| 67 | - } | ||
| 68 | - | ||
| 69 | def load_model(self, device: DeviceType = DeviceType.NPU) -> nn.Module: | 59 | def load_model(self, device: DeviceType = DeviceType.NPU) -> nn.Module: |
| 70 | return self._load_model(device) | 60 | return self._load_model(device) |
| 71 | 61 | ||
| @@ -407,10 +407,7 @@ class Qwen3_5ModelAdapter( # pylint: disable=too-many-ancestors | |||
| 407 | # num_layers = self.config.text_config.num_hidden_layers | 407 | # num_layers = self.config.text_config.num_hidden_layers |
| 408 | has_mtp = self._has_mtp() | 408 | has_mtp = self._has_mtp() |
| 409 | hidden_states = inputs_embeds | 409 | hidden_states = inputs_embeds |
| 410 | - # 确保与模型权重的 dtype 一致（量化器 fake_quant 可能把激活值提升为 float32） | ||
| 411 | - model_dtype = next(model.parameters(), torch.empty(0)).dtype | ||
| 412 | for name, layer in self.generate_decoder_layer(model): | 410 | for name, layer in self.generate_decoder_layer(model): |
| 413 | - hidden_states = hidden_states.to(model_dtype) | ||
| 414 | is_mtp_layer = has_mtp and name.startswith("mtp") | 411 | is_mtp_layer = has_mtp and name.startswith("mtp") |
| 415 | 412 | ||
| 416 | if is_mtp_layer: | 413 | if is_mtp_layer: |
| @@ -18,13 +18,11 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | |||
| 18 | See the Mulan PSL v2 for more details. | 18 | See the Mulan PSL v2 for more details. |
| 19 | ------------------------------------------------------------------------- | 19 | ------------------------------------------------------------------------- |
| 20 | """ | 20 | """ |
| 21 | - | ||
| 22 | import fnmatch | 21 | import fnmatch |
| 23 | -import inspect | ||
| 24 | from abc import ABC, abstractmethod | 22 | from abc import ABC, abstractmethod |
| 25 | -from typing import Any, Callable, Dict, Generic, List, TypeVar | 23 | +from typing import Callable, Dict, Generic, List, TypeVar |
| 26 | 24 | ||
| 27 | -from torch import nn | 25 | +import torch.nn as nn |
| 28 | 26 | ||
| 29 | from msmodelslim.utils.exception import UnsupportedError | 27 | from msmodelslim.utils.exception import UnsupportedError |
| 30 | 28 | ||
| @@ -56,30 +54,21 @@ class AnalysisTargetMatcher(ABC): | |||
| 56 | 54 | ||
| 57 | def _matches(self, module: nn.Module) -> bool: | 55 | def _matches(self, module: nn.Module) -> bool: |
| 58 | """当前 module 是否算作目标层。""" | 56 | """当前 module 是否算作目标层。""" |
| 59 | - raise NotImplementedError | 57 | + ... |
| 60 | 58 | ||
| 61 | 59 | ||
| 62 | class LayerAnalysisMethod(ABC): | 60 | class LayerAnalysisMethod(ABC): |
| 61 | + | ||
| 63 | 62 | ||
| 64 | 63 | ||
| 65 | def name(self) -> str: | 64 | def name(self) -> str: |
| 66 | """Name of the analysis method""" | 65 | """Name of the analysis method""" |
| 67 | - raise NotImplementedError | 66 | + ... |
| 68 | 67 | ||
| 69 | 68 | ||
| 70 | def get_hook(self) -> Callable: | 69 | def get_hook(self) -> Callable: |
| 71 | """Get the hook function to collect data during model inference.""" | 70 | """Get the hook function to collect data during model inference.""" |
| 72 | - raise NotImplementedError | 71 | + ... |
| 73 | - | ||
| 74 | - def enrich_layer_scores(self, layer_scores: List[Dict[str, Any]]) -> None: | ||
| 75 | - """可选的后处理钩子，在 layer_scores 写入 context 之前对其进行 enrich。 | ||
| 76 | - | ||
| 77 | - 默认实现为 no-op；具体方法可覆盖此方法以添加方法特定的元数据 | ||
| 78 | - （如 ra_compress 会添加 induction_heads / echo_heads 信息）。 | ||
| 79 | - | ||
| 80 | - Args: | ||
| 81 | - layer_scores: 层分数字典列表，就地修改。 | ||
| 82 | - """ | ||
| 83 | 72 | ||
| 84 | 73 | ||
| 85 | TMethod = TypeVar("TMethod", bound=LayerAnalysisMethod) | 74 | TMethod = TypeVar("TMethod", bound=LayerAnalysisMethod) |
| @@ -102,21 +91,9 @@ class BaseMethodFactory(Generic[TMethod]): | |||
| 102 | methods = self._get_methods() | 91 | methods = self._get_methods() |
| 103 | if method_name not in methods: | 92 | if method_name not in methods: |
| 104 | supported = list(methods.keys()) | 93 | supported = list(methods.keys()) |
| 105 | - raise UnsupportedError( | 94 | + raise UnsupportedError(f"Selected analysis method '{method_name}' is not supported.", |
| 106 | - f"Selected analysis method '{method_name}' is not supported.", | 95 | + action=f"Please use a supported analysis method. Supported methods: {supported}") |
| 107 | - action=f"Please use a supported analysis method. Supported methods: {supported}", | 96 | + return methods[method_name](**kwargs) |
| 108 | - ) | ||
| 109 | - method_cls = methods[method_name] | ||
| 110 | - # 按方法 __init__ 签名过滤 kwargs：只透传方法实际接受的参数。 | ||
| 111 | - # 例如 RaCompressAnalysisMethod 接受 adapter，而 Kurtosis/Quantile/Std 不接受， | ||
| 112 | - # 这样调用方统一传 adapter=adapter 也不会误传给不支持的方法。 | ||
| 113 | - sig = inspect.signature(method_cls) | ||
| 114 | - has_var_keyword = any(p.kind == p.VAR_KEYWORD for p in sig.parameters.values()) | ||
| 115 | - if has_var_keyword: | ||
| 116 | - filtered_kwargs = kwargs | ||
| 117 | - else: | ||
| 118 | - filtered_kwargs = {k: v for k, v in kwargs.items() if k in sig.parameters} | ||
| 119 | - return method_cls(**filtered_kwargs) | ||
| 120 | 97 | ||
| 121 | def register_method(self, method_name: str, method_class: type[TMethod]) -> None: | 98 | def register_method(self, method_name: str, method_class: type[TMethod]) -> None: |
| 122 | if not issubclass(method_class, LayerAnalysisMethod): | 99 | if not issubclass(method_class, LayerAnalysisMethod): |
| @@ -125,3 +102,4 @@ class BaseMethodFactory(Generic[TMethod]): | |||
| 125 | 102 | ||
| 126 | def get_supported_methods(self) -> List[str]: | 103 | def get_supported_methods(self) -> List[str]: |
| 127 | return list(self._get_methods().keys()) | 104 | return list(self._get_methods().keys()) |
| 105 | + | ||
| @@ -25,10 +25,8 @@ from .base import UnaryAnalysisMethod | |||
| 25 | from .quantile import QuantileAnalysisMethod | 25 | from .quantile import QuantileAnalysisMethod |
| 26 | from .std import StdAnalysisMethod | 26 | from .std import StdAnalysisMethod |
| 27 | from .kurtosis import KurtosisAnalysisMethod | 27 | from .kurtosis import KurtosisAnalysisMethod |
| 28 | -from .ra_compress import RaCompressAnalysisMethod | ||
| 29 | 28 | ||
| 30 | UnaryAnalysisMethodFactory = BaseMethodFactory[UnaryAnalysisMethod]() | 29 | UnaryAnalysisMethodFactory = BaseMethodFactory[UnaryAnalysisMethod]() |
| 31 | UnaryAnalysisMethodFactory.register_method("quantile", QuantileAnalysisMethod) | 30 | UnaryAnalysisMethodFactory.register_method("quantile", QuantileAnalysisMethod) |
| 32 | UnaryAnalysisMethodFactory.register_method("std", StdAnalysisMethod) | 31 | UnaryAnalysisMethodFactory.register_method("std", StdAnalysisMethod) |
| 33 | UnaryAnalysisMethodFactory.register_method("kurtosis", KurtosisAnalysisMethod) | 32 | UnaryAnalysisMethodFactory.register_method("kurtosis", KurtosisAnalysisMethod) |
| 34 | -UnaryAnalysisMethodFactory.register_method("ra_compress", RaCompressAnalysisMethod) | ||
| @@ -1,30 +0,0 @@ | |||
| 1 | -#!/usr/bin/env python | ||
| 2 | -# -*- coding: UTF-8 -*- | ||
| 3 | - | ||
| 4 | -""" | ||
| 5 | -------------------------------------------------------------------------- | ||
| 6 | -This file is part of the MindStudio project. | ||
| 7 | -Copyright (c) 2026 Huawei Technologies Co.,Ltd. | ||
| 8 | - | ||
| 9 | -MindStudio is licensed under Mulan PSL v2. | ||
| 10 | -You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 11 | -You may obtain a copy of Mulan PSL v2 at: | ||
| 12 | - | ||
| 13 | - http://license.coscl.org.cn/MulanPSL2 | ||
| 14 | - | ||
| 15 | -THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 16 | -EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 17 | -MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 18 | -See the Mulan PSL v2 for more details. | ||
| 19 | -------------------------------------------------------------------------- | ||
| 20 | -""" | ||
| 21 | - | ||
| 22 | -__all__ = [ | ||
| 23 | - "DUMMY_INPUT_LENGTH", | ||
| 24 | - "REPET_TIMES", | ||
| 25 | - "RaCompressAnalysisInterface", | ||
| 26 | - "RaCompressAnalysisMethod", | ||
| 27 | -] | ||
| 28 | - | ||
| 29 | -from .impl import DUMMY_INPUT_LENGTH, REPET_TIMES, RaCompressAnalysisMethod | ||
| 30 | -from .interface import RaCompressAnalysisInterface | ||
| @@ -1,540 +0,0 @@ | |||
| 1 | -#!/usr/bin/env python | ||
| 2 | -# -*- coding: UTF-8 -*- | ||
| 3 | - | ||
| 4 | -""" | ||
| 5 | -------------------------------------------------------------------------- | ||
| 6 | -This file is part of the MindStudio project. | ||
| 7 | -Copyright (c) 2026 Huawei Technologies Co.,Ltd. | ||
| 8 | - | ||
| 9 | -MindStudio is licensed under Mulan PSL v2. | ||
| 10 | -You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 11 | -You may obtain a copy of Mulan PSL v2 at: | ||
| 12 | - | ||
| 13 | - http://license.coscl.org.cn/MulanPSL2 | ||
| 14 | - | ||
| 15 | -THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 16 | -EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 17 | -MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 18 | -See the Mulan PSL v2 for more details. | ||
| 19 | -------------------------------------------------------------------------- | ||
| 20 | -""" | ||
| 21 | - | ||
| 22 | -from typing import Any, Callable, Dict, List, Optional | ||
| 23 | - | ||
| 24 | -import torch | ||
| 25 | -from torch import nn | ||
| 26 | -import torch.nn.functional as F | ||
| 27 | - | ||
| 28 | -from msmodelslim.processor.analysis.methods_base import AnalysisTargetMatcher | ||
| 29 | -from msmodelslim.utils.exception import UnexpectedError | ||
| 30 | -from msmodelslim.utils.logging import get_logger | ||
| 31 | -from ..base import UnaryAnalysisMethod | ||
| 32 | -from .interface import RaCompressAnalysisInterface | ||
| 33 | - | ||
| 34 | -logger = get_logger() | ||
| 35 | - | ||
| 36 | -DUMMY_INPUT_LENGTH = 2500 | ||
| 37 | -REPET_TIMES = 4 | ||
| 38 | - | ||
| 39 | -# 默认比例（可通过 YAML 配置覆盖） | ||
| 40 | -_DEFAULT_INDUCTION_HEAD_RATIO = 0.14 | ||
| 41 | -_DEFAULT_ECHO_HEAD_RATIO = 0.01 | ||
| 42 | - | ||
| 43 | -# 默认名称模式（当 adapter 未实现 RaCompressAnalysisInterface 时使用） | ||
| 44 | -_DEFAULT_Q_NAME_PATTERN = "q_proj" | ||
| 45 | -_DEFAULT_K_NAME_PATTERN = "k_proj" | ||
| 46 | -_DEFAULT_QKV_NAME_PATTERN = "qkv_proj" | ||
| 47 | - | ||
| 48 | - | ||
| 49 | -class RaCompressAnalysisMethod(UnaryAnalysisMethod, AnalysisTargetMatcher): | ||
| 50 | - """RA Compress analysis method for attention head importance. | ||
| 51 | - | ||
| 52 | - Hooks Q/K/QKV projection Linear layers to capture Q and K | ||
| 53 | - outputs, then manually reconstructs the attention softmax matrix | ||
| 54 | - (Q @ K^T / sqrt(d)) to compute prefix matching and copying matching | ||
| 55 | - scores for each attention head. | ||
| 56 | - | ||
| 57 | - The scores identify induction heads (prefix matching) and echo heads | ||
| 58 | - (copying matching), which are critical for long-context repeat | ||
| 59 | - detection in RA compression. | ||
| 60 | - | ||
| 61 | - After all layers are processed, call :meth:`get_compress_heads` to | ||
| 62 | - select top heads by ratio and produce the head dict for saving. | ||
| 63 | - | ||
| 64 | - Q/K/QKV 名称模式通过 ``adapter`` 的 :class:`RaCompressAnalysisInterface` | ||
| 65 | - 获取，未实现时回退到 ``q_proj`` / ``k_proj`` / ``qkv_proj``。 | ||
| 66 | - """ | ||
| 67 | - | ||
| 68 | - def __init__(self, adapter: Optional[object] = None): | ||
| 69 | - self.adapter = adapter | ||
| 70 | - | ||
| 71 | - # 从 adapter 获取名称模式，或使用默认值 | ||
| 72 | - if adapter is not None and isinstance(adapter, RaCompressAnalysisInterface): | ||
| 73 | - patterns = adapter.get_ra_compress_proj_patterns() | ||
| 74 | - self._q_name_pattern = patterns.get("q", _DEFAULT_Q_NAME_PATTERN) | ||
| 75 | - self._k_name_pattern = patterns.get("k", _DEFAULT_K_NAME_PATTERN) | ||
| 76 | - self._qkv_name_pattern = patterns.get("qkv", _DEFAULT_QKV_NAME_PATTERN) | ||
| 77 | - else: | ||
| 78 | - self._q_name_pattern = _DEFAULT_Q_NAME_PATTERN | ||
| 79 | - self._k_name_pattern = _DEFAULT_K_NAME_PATTERN | ||
| 80 | - self._qkv_name_pattern = _DEFAULT_QKV_NAME_PATTERN | ||
| 81 | - | ||
| 82 | - self._num_attention_heads: int = 0 | ||
| 83 | - self._num_key_value_heads: int = 0 | ||
| 84 | - self._hidden_size: int = 0 | ||
| 85 | - self._head_dim: int = 0 | ||
| 86 | - self._config_extracted: bool = False | ||
| 87 | - | ||
| 88 | - # YAML 可配置的超参 | ||
| 89 | - self._induction_head_ratio: float = _DEFAULT_INDUCTION_HEAD_RATIO | ||
| 90 | - self._echo_head_ratio: float = _DEFAULT_ECHO_HEAD_RATIO | ||
| 91 | - | ||
| 92 | - # 存储 Q 和 K 的 Linear 输出 | ||
| 93 | - self._q_outputs: Dict[str, torch.Tensor] = {} | ||
| 94 | - self._k_outputs: Dict[str, torch.Tensor] = {} | ||
| 95 | - | ||
| 96 | - # 存储每层每个 head 的 prefix / copying 分数 | ||
| 97 | - # key = 层索引(int, 从0开始), value = list[float] (每个 head 一个分数) | ||
| 98 | - self._prefix_scores: Dict[int, List[float]] = {} | ||
| 99 | - self._copying_scores: Dict[int, List[float]] = {} | ||
| 100 | - | ||
| 101 | - # 层名 -> 层索引 的映射 | ||
| 102 | - self._layer_name_to_idx: Dict[str, int] = {} | ||
| 103 | - # 层索引 -> 层名 的映射（反向，用于 get_compress_heads 输出 name 作为 key） | ||
| 104 | - self._layer_idx_to_name: Dict[int, str] = {} | ||
| 105 | - self._next_layer_idx: int = 0 | ||
| 106 | - | ||
| 107 | - | ||
| 108 | - def name(self) -> str: | ||
| 109 | - return "ra_compress" | ||
| 110 | - | ||
| 111 | - def _matches(self, module: nn.Module) -> bool: | ||
| 112 | - return isinstance(module, nn.Linear) | ||
| 113 | - | ||
| 114 | - def get_target_layers(self, model: nn.Module, prefix: str = "") -> List[str]: | ||
| 115 | - """Return only Q, K, or QKV Linear layers as target layers.""" | ||
| 116 | - target_layers = [] | ||
| 117 | - for name, module in model.named_modules(prefix=prefix): | ||
| 118 | - if not self._matches(module): | ||
| 119 | - continue | ||
| 120 | - if self._is_target_layer(name): | ||
| 121 | - target_layers.append(name) | ||
| 122 | - if not self._config_extracted: | ||
| 123 | - self._extract_attention_config(model, name) | ||
| 124 | - return target_layers | ||
| 125 | - | ||
| 126 | - def _is_target_layer(self, name: str) -> bool: | ||
| 127 | - return self._q_name_pattern in name or self._k_name_pattern in name or self._qkv_name_pattern in name | ||
| 128 | - | ||
| 129 | - def _extract_attention_config(self, model: nn.Module, layer_name: str) -> None: | ||
| 130 | - """从模型 config 提取注意力配置。 | ||
| 131 | - | ||
| 132 | - LayerWiseRunner 传入的 model 是单个 decoder layer，其 self_attn 子模块 | ||
| 133 | - 持有 config 引用（Qwen2Attention.config）；完整模型则直接有 model.config。 | ||
| 134 | - 直接从 config 读取，不需要复杂的路径导航。 | ||
| 135 | - """ | ||
| 136 | - # 获取 config 对象：优先 attention 子模块的 config，其次 model 自身 | ||
| 137 | - cfg = None | ||
| 138 | - if hasattr(model, 'self_attn') and hasattr(model.self_attn, 'config'): | ||
| 139 | - cfg = model.self_attn.config | ||
| 140 | - elif hasattr(model, 'config'): | ||
| 141 | - cfg = model.config | ||
| 142 | - | ||
| 143 | - if cfg is not None: | ||
| 144 | - self._num_attention_heads = int( | ||
| 145 | - getattr(cfg, 'num_attention_heads', 0) or getattr(cfg, 'num_heads', 0) or getattr(cfg, 'n_head', 0) | ||
| 146 | - ) | ||
| 147 | - self._num_key_value_heads = int( | ||
| 148 | - getattr(cfg, 'num_key_value_heads', 0) | ||
| 149 | - or getattr(cfg, 'multi_query_group_num', 0) | ||
| 150 | - or getattr(cfg, 'num_kv_heads', 0) | ||
| 151 | - or self._num_attention_heads | ||
| 152 | - ) | ||
| 153 | - self._hidden_size = int(getattr(cfg, 'hidden_size', 0) or getattr(cfg, 'embed_dim', 0)) | ||
| 154 | - head_dim_cfg = getattr(cfg, 'head_dim', None) | ||
| 155 | - if head_dim_cfg: | ||
| 156 | - self._head_dim = int(head_dim_cfg) | ||
| 157 | - elif self._num_attention_heads > 0 and self._hidden_size > 0: | ||
| 158 | - self._head_dim = self._hidden_size // self._num_attention_heads | ||
| 159 | - | ||
| 160 | - if self._num_attention_heads == 0: | ||
| 161 | - logger.warning("RA compress: can not extract attention config from %s", layer_name) | ||
| 162 | - else: | ||
| 163 | - logger.info( | ||
| 164 | - "RA compress config: num_attention_heads=%d, num_key_value_heads=%d, hidden_size=%d, head_dim=%d", | ||
| 165 | - self._num_attention_heads, | ||
| 166 | - self._num_key_value_heads, | ||
| 167 | - self._hidden_size, | ||
| 168 | - self._head_dim, | ||
| 169 | - ) | ||
| 170 | - self._config_extracted = True | ||
| 171 | - | ||
| 172 | - def get_hook(self) -> Callable: | ||
| 173 | - """Hook 注册在 q_proj / k_proj / qkv_proj 的 nn.Linear 上。 | ||
| 174 | - | ||
| 175 | - 捕获 Linear 的输出（即 Q 或 K 的投影结果），保留在 device 上 | ||
| 176 | - （不搬到 CPU），供后续在 NPU/GPU 上直接重建 attention 矩阵。 | ||
| 177 | - """ | ||
| 178 | - | ||
| 179 | - def linear_output_hook( | ||
| 180 | - module: nn.Module, | ||
| 181 | - input_tensor: Any, | ||
| 182 | - output_tensor: Any, | ||
| 183 | - layer_name: str, | ||
| 184 | - stats_dict: Dict[str, Any], | ||
| 185 | - ): | ||
| 186 | - if isinstance(output_tensor, tuple): | ||
| 187 | - output_tensor = output_tensor[0] | ||
| 188 | - | ||
| 189 | - # 保留在原 device（NPU/GPU）上，不 .cpu() | ||
| 190 | - output_detached = output_tensor.detach() | ||
| 191 | - | ||
| 192 | - # 存入 stats_dict（processor 靠此判断是否有数据并触发 compute_score） | ||
| 193 | - if layer_name not in stats_dict: | ||
| 194 | - stats_dict[layer_name] = {'outputs': [], 'layer_name': layer_name} | ||
| 195 | - stats_dict[layer_name]['outputs'].append(output_detached) | ||
| 196 | - | ||
| 197 | - # 存入 method 自己的字典 | ||
| 198 | - if self._qkv_name_pattern in layer_name: | ||
| 199 | - self._q_outputs[layer_name] = output_detached | ||
| 200 | - self._k_outputs[layer_name] = output_detached | ||
| 201 | - elif self._q_name_pattern in layer_name: | ||
| 202 | - self._q_outputs[layer_name] = output_detached | ||
| 203 | - elif self._k_name_pattern in layer_name: | ||
| 204 | - self._k_outputs[layer_name] = output_detached | ||
| 205 | - | ||
| 206 | - return linear_output_hook | ||
| 207 | - | ||
| 208 | - def compute_score(self, layer_data: Dict[str, Any]) -> float: | ||
| 209 | - """用 Q 和 K 的 Linear 输出重建 attention softmax 矩阵，计算分数。 | ||
| 210 | - | ||
| 211 | - 对 Q/QKV 层：取 Q 输出和对应 K 输出，手动计算 | ||
| 212 | - softmax(Q @ K^T / sqrt(head_dim))，再算 prefix + copying 分数。 | ||
| 213 | - 同时记录每个 head 的单独分数，用于后续 head 筛选。 | ||
| 214 | - 对 K 层：返回 0（分数在 Q 层计算）。 | ||
| 215 | - """ | ||
| 216 | - layer_name = layer_data.get('layer_name', '') | ||
| 217 | - outputs = layer_data.get('outputs', []) | ||
| 218 | - | ||
| 219 | - if not outputs: | ||
| 220 | - return 0.0 | ||
| 221 | - | ||
| 222 | - # K 层不计算分数 | ||
| 223 | - if self._k_name_pattern in layer_name and self._q_name_pattern not in layer_name: | ||
| 224 | - return 0.0 | ||
| 225 | - | ||
| 226 | - if self._q_name_pattern not in layer_name and self._qkv_name_pattern not in layer_name: | ||
| 227 | - return 0.0 | ||
| 228 | - | ||
| 229 | - # 取 Q 输出 | ||
| 230 | - q_output = self._q_outputs.get(layer_name) | ||
| 231 | - if q_output is None: | ||
| 232 | - return 0.0 | ||
| 233 | - | ||
| 234 | - # 分配层索引 | ||
| 235 | - if layer_name not in self._layer_name_to_idx: | ||
| 236 | - self._layer_name_to_idx[layer_name] = self._next_layer_idx | ||
| 237 | - self._layer_idx_to_name[self._next_layer_idx] = layer_name | ||
| 238 | - self._next_layer_idx += 1 | ||
| 239 | - layer_idx = self._layer_name_to_idx[layer_name] | ||
| 240 | - | ||
| 241 | - # 取对应 K 输出并计算 | ||
| 242 | - if self._qkv_name_pattern in layer_name: | ||
| 243 | - prefix_per_head, copying_per_head = self._compute_qkv_scores(layer_name, q_output) | ||
| 244 | - else: | ||
| 245 | - k_name = layer_name.replace(self._q_name_pattern, self._k_name_pattern) | ||
| 246 | - k_output = self._k_outputs.get(k_name) | ||
| 247 | - if k_output is None: | ||
| 248 | - logger.warning( | ||
| 249 | - "RA compress: no K output for Q layer %s (expected K at %s)", | ||
| 250 | - layer_name, | ||
| 251 | - k_name, | ||
| 252 | - ) | ||
| 253 | - return 0.0 | ||
| 254 | - prefix_per_head, copying_per_head = self._compute_qk_scores(layer_name, q_output, k_output) | ||
| 255 | - | ||
| 256 | - # 存储每头分数 | ||
| 257 | - self._prefix_scores[layer_idx] = prefix_per_head | ||
| 258 | - self._copying_scores[layer_idx] = copying_per_head | ||
| 259 | - | ||
| 260 | - # 返回所有头的平均 combined 分数 | ||
| 261 | - combined_per_head = [p + c for p, c in zip(prefix_per_head, copying_per_head)] | ||
| 262 | - avg_combined = sum(combined_per_head) / len(combined_per_head) if combined_per_head else 0.0 | ||
| 263 | - | ||
| 264 | - logger.debug( | ||
| 265 | - "RA compress: layer %s (idx=%d) prefix=%s copying=%s avg_combined=%.6f", | ||
| 266 | - layer_name, | ||
| 267 | - layer_idx, | ||
| 268 | - prefix_per_head, | ||
| 269 | - copying_per_head, | ||
| 270 | - avg_combined, | ||
| 271 | - ) | ||
| 272 | - return float(avg_combined) | ||
| 273 | - | ||
| 274 | - def _compute_qkv_scores( | ||
| 275 | - self, | ||
| 276 | - layer_name: str, | ||
| 277 | - qkv_output: torch.Tensor, | ||
| 278 | - ) -> tuple: | ||
| 279 | - """从 qkv_proj 输出中切分 Q 和 K，计算每头分数。""" | ||
| 280 | - if self._num_attention_heads == 0 or self._head_dim == 0: | ||
| 281 | - return [], [] | ||
| 282 | - | ||
| 283 | - q_dim = self._num_attention_heads * self._head_dim | ||
| 284 | - k_dim = self._num_key_value_heads * self._head_dim | ||
| 285 | - | ||
| 286 | - total_dim = qkv_output.shape[-1] | ||
| 287 | - if total_dim < q_dim + k_dim: | ||
| 288 | - logger.warning( | ||
| 289 | - "RA compress: qkv output dim %d < q_dim(%d) + k_dim(%d) for %s", | ||
| 290 | - total_dim, | ||
| 291 | - q_dim, | ||
| 292 | - k_dim, | ||
| 293 | - layer_name, | ||
| 294 | - ) | ||
| 295 | - return [], [] | ||
| 296 | - | ||
| 297 | - q_output = qkv_output[..., :q_dim] | ||
| 298 | - k_output = qkv_output[..., q_dim : q_dim + k_dim] | ||
| 299 | - | ||
| 300 | - return self._compute_qk_scores(layer_name, q_output, k_output) | ||
| 301 | - | ||
| 302 | - def _compute_qk_scores( | ||
| 303 | - self, | ||
| 304 | - layer_name: str, | ||
| 305 | - q_output: torch.Tensor, | ||
| 306 | - k_output: torch.Tensor, | ||
| 307 | - ) -> tuple: | ||
| 308 | - """用 Q 和 K 的输出手动重建 attention softmax 矩阵并计算每头分数。 | ||
| 309 | - | ||
| 310 | - 逐 head 在 device 上完成 Q@K^T / causal mask / softmax， | ||
| 311 | - 配合向量化算分，避免 Python 双重循环。 | ||
| 312 | - """ | ||
| 313 | - if self._num_attention_heads == 0 or self._head_dim == 0: | ||
| 314 | - return [], [] | ||
| 315 | - | ||
| 316 | - try: | ||
| 317 | - q_flat = self._flatten_to_2d(q_output) | ||
| 318 | - k_flat = self._flatten_to_2d(k_output) | ||
| 319 | - | ||
| 320 | - total_tokens = q_flat.shape[0] | ||
| 321 | - required_tokens = DUMMY_INPUT_LENGTH * REPET_TIMES | ||
| 322 | - if total_tokens < required_tokens: | ||
| 323 | - raise UnexpectedError( | ||
| 324 | - f"RA compress requires at least {required_tokens} tokens " | ||
| 325 | - f"(DUMMY_INPUT_LENGTH={DUMMY_INPUT_LENGTH} * REPET_TIMES={REPET_TIMES}), " | ||
| 326 | - f"but got {total_tokens}. Please use calib_dummy.jsonl or ensure the " | ||
| 327 | - f"calibration data has sufficient token length." | ||
| 328 | - ) | ||
| 329 | - | ||
| 330 | - device = q_flat.device | ||
| 331 | - num_heads = self._num_attention_heads | ||
| 332 | - num_kv_heads = self._num_key_value_heads if self._num_key_value_heads > 0 else num_heads | ||
| 333 | - head_dim = self._head_dim | ||
| 334 | - scale = float(head_dim) ** 0.5 | ||
| 335 | - repeats = num_heads // num_kv_heads if num_kv_heads < num_heads else 1 | ||
| 336 | - | ||
| 337 | - # [total_tokens, num_heads, head_dim] | ||
| 338 | - q_reshaped = q_flat.reshape(total_tokens, num_heads, head_dim) | ||
| 339 | - # [total_tokens, num_kv_heads, head_dim] | ||
| 340 | - k_reshaped = k_flat.reshape(total_tokens, num_kv_heads, head_dim) | ||
| 341 | - | ||
| 342 | - # 预生成 causal mask: [total_tokens, total_tokens]，逐 head 使用 | ||
| 343 | - causal_mask = torch.triu( | ||
| 344 | - torch.ones(total_tokens, total_tokens, dtype=torch.bool, device=device), | ||
| 345 | - diagonal=1, | ||
| 346 | - ) | ||
| 347 | - | ||
| 348 | - prefix_per_head: List[float] = [] | ||
| 349 | - copying_per_head: List[float] = [] | ||
| 350 | - | ||
| 351 | - for h in range(num_heads): | ||
| 352 | - kv_idx = h // repeats | ||
| 353 | - q_h = q_reshaped[:, h, :] # [seq, head_dim] | ||
| 354 | - k_h = k_reshaped[:, kv_idx, :] # [seq, head_dim] | ||
| 355 | - | ||
| 356 | - # [seq, seq] — 在 device 上完成 | ||
| 357 | - attn = torch.matmul(q_h, k_h.transpose(0, 1)) / scale | ||
| 358 | - attn = attn.masked_fill(causal_mask, float('-inf')) | ||
| 359 | - attn = F.softmax(attn, dim=-1) | ||
| 360 | - | ||
| 361 | - # 向量化算分 | ||
| 362 | - prefix_per_head.append(self._prefix_score_for_matrix(attn)) | ||
| 363 | - copying_per_head.append(self._copying_score_for_matrix(attn)) | ||
| 364 | - | ||
| 365 | - del attn | ||
| 366 | - | ||
| 367 | - return prefix_per_head, copying_per_head | ||
| 368 | - | ||
| 369 | - except Exception as e: | ||
| 370 | - logger.warning("RA compress: error computing scores for %s: %s", layer_name, e) | ||
| 371 | - return [], [] | ||
| 372 | - | ||
| 373 | - | ||
| 374 | - def _flatten_to_2d(tensor: torch.Tensor) -> torch.Tensor: | ||
| 375 | - """将 [batch, seq, dim] 或 [batch*seq, dim] 统一展平为 [total_tokens, dim]。""" | ||
| 376 | - if tensor.dim() == 2: | ||
| 377 | - return tensor | ||
| 378 | - if tensor.dim() == 3: | ||
| 379 | - batch, seq, dim = tensor.shape | ||
| 380 | - return tensor.reshape(batch * seq, dim) | ||
| 381 | - if tensor.dim() == 4: | ||
| 382 | - return tensor.reshape(-1, tensor.shape[-1]) | ||
| 383 | - return tensor.reshape(-1, tensor.shape[-1]) | ||
| 384 | - | ||
| 385 | - | ||
| 386 | - def _prefix_score_for_matrix(attn: torch.Tensor) -> float: | ||
| 387 | - """向量化计算单个 head 的 prefix matching 分数。 | ||
| 388 | - | ||
| 389 | - attn: [seq_len, seq_len]，在 device 上。 | ||
| 390 | - 按 (段 k, 偏移 d) 分块 gather，完全避免 Python 双重循环。 | ||
| 391 | - """ | ||
| 392 | - if attn.dim() != 2: | ||
| 393 | - return 0.0 | ||
| 394 | - seq_len = attn.shape[0] | ||
| 395 | - device = attn.device | ||
| 396 | - total_sum = 0.0 | ||
| 397 | - num_rows = 0 | ||
| 398 | - | ||
| 399 | - # 段数（基于 seq_len 和 DUMMY_INPUT_LENGTH 动态计算） | ||
| 400 | - num_segments = (seq_len + DUMMY_INPUT_LENGTH - 1) // DUMMY_INPUT_LENGTH | ||
| 401 | - | ||
| 402 | - for k in range(1, num_segments): # 跳过第 0 段 (k=0) | ||
| 403 | - start = k * DUMMY_INPUT_LENGTH | ||
| 404 | - if start >= seq_len: | ||
| 405 | - break | ||
| 406 | - end = min((k + 1) * DUMMY_INPUT_LENGTH, seq_len) | ||
| 407 | - # 行索引 | ||
| 408 | - i_vals = torch.arange(start, end, device=device) | ||
| 409 | - # d 从 -k 到 -1 | ||
| 410 | - for d in range(-k, 0): | ||
| 411 | - col_vals = i_vals + d * DUMMY_INPUT_LENGTH + 1 # prefix: +1 | ||
| 412 | - # 有效 mask | ||
| 413 | - valid = (col_vals >= 0) & (col_vals < seq_len) | ||
| 414 | - if not valid.any(): | ||
| 415 | - continue | ||
| 416 | - total_sum += attn[i_vals[valid], col_vals[valid]].sum().item() | ||
| 417 | - num_rows += valid.sum().item() | ||
| 418 | - | ||
| 419 | - return total_sum / num_rows if num_rows > 0 else 0.0 | ||
| 420 | - | ||
| 421 | - | ||
| 422 | - def _copying_score_for_matrix(attn: torch.Tensor) -> float: | ||
| 423 | - """向量化计算单个 head 的 copying matching 分数。 | ||
| 424 | - | ||
| 425 | - attn: [seq_len, seq_len]，在 device 上。 | ||
| 426 | - """ | ||
| 427 | - if attn.dim() != 2: | ||
| 428 | - return 0.0 | ||
| 429 | - seq_len = attn.shape[0] | ||
| 430 | - device = attn.device | ||
| 431 | - total_sum = 0.0 | ||
| 432 | - num_rows = 0 | ||
| 433 | - | ||
| 434 | - num_segments = (seq_len + DUMMY_INPUT_LENGTH - 1) // DUMMY_INPUT_LENGTH | ||
| 435 | - | ||
| 436 | - for k in range(1, num_segments): | ||
| 437 | - start = k * DUMMY_INPUT_LENGTH | ||
| 438 | - if start >= seq_len: | ||
| 439 | - break | ||
| 440 | - end = min((k + 1) * DUMMY_INPUT_LENGTH, seq_len) | ||
| 441 | - i_vals = torch.arange(start, end, device=device) | ||
| 442 | - for d in range(-k, 0): | ||
| 443 | - col_vals = i_vals + d * DUMMY_INPUT_LENGTH # copying: 不加 1 | ||
| 444 | - valid = (col_vals >= 0) & (col_vals < seq_len) | ||
| 445 | - if not valid.any(): | ||
| 446 | - continue | ||
| 447 | - total_sum += attn[i_vals[valid], col_vals[valid]].sum().item() | ||
| 448 | - num_rows += valid.sum().item() | ||
| 449 | - | ||
| 450 | - return total_sum / num_rows if num_rows > 0 else 0.0 | ||
| 451 | - | ||
| 452 | - def enrich_layer_scores(self, layer_scores: List[Dict[str, Any]]) -> None: | ||
| 453 | - """将 head 选择信息合并进 layer_scores 条目。 | ||
| 454 | - | ||
| 455 | - 遍历 get_compress_heads() 的结果，将 induction_heads / echo_heads | ||
| 456 | - 按层名写入对应的 layer_scores 条目。 | ||
| 457 | - """ | ||
| 458 | - head_dict = self.get_compress_heads() | ||
| 459 | - prefix_map = head_dict.get('prefix_matching', {}) | ||
| 460 | - copying_map = head_dict.get('copying', {}) | ||
| 461 | - | ||
| 462 | - name_to_induction = {} | ||
| 463 | - name_to_echo = {} | ||
| 464 | - for layer_idx, heads in prefix_map.items(): | ||
| 465 | - name = self._layer_idx_to_name.get(int(layer_idx), str(layer_idx)) | ||
| 466 | - name_to_induction[name] = heads | ||
| 467 | - for layer_idx, heads in copying_map.items(): | ||
| 468 | - name = self._layer_idx_to_name.get(int(layer_idx), str(layer_idx)) | ||
| 469 | - name_to_echo[name] = heads | ||
| 470 | - | ||
| 471 | - for entry in layer_scores: | ||
| 472 | - name = entry['name'] | ||
| 473 | - entry['induction_heads'] = name_to_induction.get(name, []) | ||
| 474 | - entry['echo_heads'] = name_to_echo.get(name, []) | ||
| 475 | - | ||
| 476 | - # ========== Head 选择逻辑（与 ra_rope_tools.py 对齐）========== | ||
| 477 | - | ||
| 478 | - def get_compress_heads(self) -> Dict[str, Dict[int, List[int]]]: | ||
| 479 | - """选择 top heads 并返回 head_dict。 | ||
| 480 | - | ||
| 481 | - - induction head: prefix matching 分数前 14% | ||
| 482 | - - echo head: copying matching 分数前 1% | ||
| 483 | - | ||
| 484 | - key 为层索引（int），value 为需要保留的 KV 头索引列表。 | ||
| 485 | - | ||
| 486 | - Returns: | ||
| 487 | - { | ||
| 488 | - 'prefix_matching': {layer_idx: [kv_head_idx, ...]}, | ||
| 489 | - 'copying': {layer_idx: [kv_head_idx, ...]}, | ||
| 490 | - } | ||
| 491 | - """ | ||
| 492 | - # GQA: 每个 kv group 内取 max，分组后索引即为 KV 头索引 | ||
| 493 | - num_kv_per_group = ( | ||
| 494 | - int(self._num_attention_heads // self._num_key_value_heads) if self._num_key_value_heads > 0 else 1 | ||
| 495 | - ) | ||
| 496 | - | ||
| 497 | - prefix_grouped = self._max_every_group(self._prefix_scores, num_kv_per_group) | ||
| 498 | - copying_grouped = self._max_every_group(self._copying_scores, num_kv_per_group) | ||
| 499 | - | ||
| 500 | - selected_prefix = self._select_top_heads(prefix_grouped, self._induction_head_ratio) | ||
| 501 | - selected_copying = self._select_top_heads(copying_grouped, self._echo_head_ratio) | ||
| 502 | - | ||
| 503 | - # key 直接使用 layer_idx（int），value 为 KV 头索引列表 | ||
| 504 | - head_dict = { | ||
| 505 | - 'prefix_matching': self._remove_empty_list_keys(selected_prefix), | ||
| 506 | - 'copying': self._remove_empty_list_keys(selected_copying), | ||
| 507 | - } | ||
| 508 | - return head_dict | ||
| 509 | - | ||
| 510 | - | ||
| 511 | - def _max_every_group(data: Dict[int, List[float]], n: int) -> Dict[int, List[float]]: | ||
| 512 | - """每个 n 个 head 一组，取组内 max（GQA 分组）。""" | ||
| 513 | - if n <= 1: | ||
| 514 | - return data | ||
| 515 | - result = {} | ||
| 516 | - for key, values in data.items(): | ||
| 517 | - max_values = [max(values[i : i + n]) for i in range(0, len(values), n)] | ||
| 518 | - result[key] = max_values | ||
| 519 | - return result | ||
| 520 | - | ||
| 521 | - | ||
| 522 | - def _select_top_heads(data: Dict[int, List[float]], ratio: float) -> Dict[int, List[int]]: | ||
| 523 | - """选择分数前 ratio 比例的 head，返回每层的 head 索引列表。""" | ||
| 524 | - all_values = [value for key in data for value in data[key]] | ||
| 525 | - if not all_values: | ||
| 526 | - return {} | ||
| 527 | - | ||
| 528 | - sorted_values = sorted(all_values, reverse=True) | ||
| 529 | - percent_index = round(len(sorted_values) * ratio) | ||
| 530 | - percent_values = sorted_values[:percent_index] | ||
| 531 | - | ||
| 532 | - result = {} | ||
| 533 | - for key in data: | ||
| 534 | - indices = [i for i, value in enumerate(data[key]) if value in percent_values] | ||
| 535 | - result[key] = indices | ||
| 536 | - return result | ||
| 537 | - | ||
| 538 | - | ||
| 539 | - def _remove_empty_list_keys(dictionary: Dict) -> Dict: | ||
| 540 | - return {k: v for k, v in dictionary.items() if v != []} | ||
| @@ -1,46 +0,0 @@ | |||
| 1 | -#!/usr/bin/env python | ||
| 2 | -# -*- coding: UTF-8 -*- | ||
| 3 | - | ||
| 4 | -""" | ||
| 5 | -------------------------------------------------------------------------- | ||
| 6 | -This file is part of the MindStudio project. | ||
| 7 | -Copyright (c) 2026 Huawei Technologies Co.,Ltd. | ||
| 8 | - | ||
| 9 | -MindStudio is licensed under Mulan PSL v2. | ||
| 10 | -You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 11 | -You may obtain a copy of Mulan PSL v2 at: | ||
| 12 | - | ||
| 13 | - http://license.coscl.org.cn/MulanPSL2 | ||
| 14 | - | ||
| 15 | -THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 16 | -EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 17 | -MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 18 | -See the Mulan PSL v2 for more details. | ||
| 19 | -------------------------------------------------------------------------- | ||
| 20 | -""" | ||
| 21 | - | ||
| 22 | -from abc import ABC, abstractmethod | ||
| 23 | -from typing import Dict | ||
| 24 | - | ||
| 25 | - | ||
| 26 | -class RaCompressAnalysisInterface(ABC): | ||
| 27 | - """RA Compress 分析需要在模型适配器中实现的接口。 | ||
| 28 | - | ||
| 29 | - 提供 Q、K、QKV 投影层的名称模式，用于定位目标层。 | ||
| 30 | - """ | ||
| 31 | - | ||
| 32 | - | ||
| 33 | - def get_ra_compress_proj_patterns(self) -> Dict[str, str]: | ||
| 34 | - """返回 Q/K/QKV 投影层名称模式字典。 | ||
| 35 | - | ||
| 36 | - 返回值格式:: | ||
| 37 | - | ||
| 38 | - { | ||
| 39 | - "q": "q_proj", # Q 投影层名称模式 | ||
| 40 | - "k": "k_proj", # K 投影层名称模式 | ||
| 41 | - "qkv": "qkv_proj", # QKV 融合投影层名称模式 | ||
| 42 | - } | ||
| 43 | - | ||
| 44 | - 未使用的模式可以留空字符串（如无 QKV 融合时 qkv=""）。 | ||
| 45 | - """ | ||
| 46 | - raise NotImplementedError | ||
| @@ -71,7 +71,7 @@ class UnaryAnalysisProcessor(AutoSessionProcessor): | |||
| 71 | ): | 71 | ): |
| 72 | super().__init__(model) | 72 | super().__init__(model) |
| 73 | self.config = config | 73 | self.config = config |
| 74 | - self._analysis_method = UnaryAnalysisMethodFactory.create_method(config.metrics, adapter=adapter) | 74 | + self._analysis_method = UnaryAnalysisMethodFactory.create_method(config.metrics) |
| 75 | self._target_layers: List[str] = [] | 75 | self._target_layers: List[str] = [] |
| 76 | self._layer_stats: Dict[str, Any] = {} | 76 | self._layer_stats: Dict[str, Any] = {} |
| 77 | self._layer_scores: List[Dict[str, Any]] = [] | 77 | self._layer_scores: List[Dict[str, Any]] = [] |
| @@ -135,10 +135,6 @@ class UnaryAnalysisProcessor(AutoSessionProcessor): | |||
| 135 | if ctx is None: | 135 | if ctx is None: |
| 136 | return | 136 | return |
| 137 | layer_analysis = ctx["layer_analysis"] # pylint: disable=unsubscriptable-object | 137 | layer_analysis = ctx["layer_analysis"] # pylint: disable=unsubscriptable-object |
| 138 | - | ||
| 139 | - # 让分析方法 enrich layer_scores（如 ra_compress 会添加 induction_heads/echo_heads） | ||
| 140 | - self._analysis_method.enrich_layer_scores(self._layer_scores) | ||
| 141 | - | ||
| 142 | layer_analysis.debug["layer_scores"] = self._layer_scores | 138 | layer_analysis.debug["layer_scores"] = self._layer_scores |
| 143 | layer_analysis.debug["method"] = self._analysis_method.name | 139 | layer_analysis.debug["method"] = self._analysis_method.name |
| 144 | layer_analysis.debug["patterns"] = self.config.patterns | 140 | layer_analysis.debug["patterns"] = self.config.patterns |
| @@ -1,6 +1,6 @@ | |||
| 1 | [files] | 1 | [files] |
| 2 | # these files may be written in non english words | 2 | # these files may be written in non english words |
| 3 | - extend-exclude = ["lab_calib/calib_dummy.jsonl"] | 3 | + extend-exclude = [] |
| 4 | ignore-hidden = true | 4 | ignore-hidden = true |
| 5 | ignore-files = true | 5 | ignore-files = true |
| 6 | ignore-dot = true | 6 | ignore-dot = true |
| @@ -254,8 +254,6 @@ class TestAppAnalysisModule(TestComprehensiveAnalysisCoverage): | |||
| 254 | mock_result, | 254 | mock_result, |
| 255 | 15, | 255 | 15, |
| 256 | AnalysisScope.LINEAR, | 256 | AnalysisScope.LINEAR, |
| 257 | - save_path=None, | ||
| 258 | - model_type="Qwen2.5-7B-Instruct", | ||
| 259 | ) | 257 | ) |
| 260 | 258 | ||
| 261 | 259 | ||
| @@ -56,37 +56,37 @@ class TestAnalysisMain: | |||
| 56 | return Namespace(**defaults) | 56 | return Namespace(**defaults) |
| 57 | 57 | ||
| 58 | 58 | ||
| 59 | - @patch("msmodelslim.cli.analysis.__main__.AnalysisResultDisplayerFactory") | 59 | + @patch("msmodelslim.cli.analysis.__main__.LoggingAnalysisResultDisplayer") |
| 60 | 60 | ||
| 61 | - def test_main_dispatches_to_linear_scope_when_scope_is_linear(self, mock_pl_cls, mock_factory, mock_app_cls): | 61 | + def test_main_dispatches_to_linear_scope_when_scope_is_linear(self, mock_pl_cls, mock_disp_cls, mock_app_cls): |
| 62 | """主路径：scope=linear 时应构造 LinearArgs 并调用 analyze。""" | 62 | """主路径：scope=linear 时应构造 LinearArgs 并调用 analyze。""" |
| 63 | from unittest.mock import MagicMock | 63 | from unittest.mock import MagicMock |
| 64 | 64 | ||
| 65 | mock_pl_instance = MagicMock() | 65 | mock_pl_instance = MagicMock() |
| 66 | mock_pl_cls.return_value = mock_pl_instance | 66 | mock_pl_cls.return_value = mock_pl_instance |
| 67 | - mock_displayer = MagicMock() | 67 | + mock_disp_instance = MagicMock() |
| 68 | - mock_factory.create.return_value = mock_displayer | 68 | + mock_disp_cls.return_value = mock_disp_instance |
| 69 | mock_app_instance = MagicMock() | 69 | mock_app_instance = MagicMock() |
| 70 | mock_app_cls.return_value = mock_app_instance | 70 | mock_app_cls.return_value = mock_app_instance |
| 71 | 71 | ||
| 72 | args = self._make_args(scope="linear", metrics="kurtosis") | 72 | args = self._make_args(scope="linear", metrics="kurtosis") |
| 73 | main(args) | 73 | main(args) |
| 74 | 74 | ||
| 75 | - mock_factory.create.assert_called_once_with("kurtosis") | 75 | + # analyze 应被调用 |
| 76 | mock_app_instance.analyze.assert_called_once() | 76 | mock_app_instance.analyze.assert_called_once() |
| 77 | call_kwargs = mock_app_instance.analyze.call_args.kwargs | 77 | call_kwargs = mock_app_instance.analyze.call_args.kwargs |
| 78 | assert call_kwargs["model_type"] == "qwen3" | 78 | assert call_kwargs["model_type"] == "qwen3" |
| 79 | assert call_kwargs["topk"] == 15 | 79 | assert call_kwargs["topk"] == 15 |
| 80 | 80 | ||
| 81 | 81 | ||
| 82 | - @patch("msmodelslim.cli.analysis.__main__.AnalysisResultDisplayerFactory") | 82 | + @patch("msmodelslim.cli.analysis.__main__.LoggingAnalysisResultDisplayer") |
| 83 | 83 | ||
| 84 | - def test_main_dispatches_to_layer_scope_when_scope_is_layer(self, mock_pl_cls, mock_factory, mock_app_cls): | 84 | + def test_main_dispatches_to_layer_scope_when_scope_is_layer(self, mock_pl_cls, mock_disp_cls, mock_app_cls): |
| 85 | """主路径：scope=layer 时应构造 LayerArgs。""" | 85 | """主路径：scope=layer 时应构造 LayerArgs。""" |
| 86 | from unittest.mock import MagicMock | 86 | from unittest.mock import MagicMock |
| 87 | 87 | ||
| 88 | mock_pl_cls.return_value = MagicMock() | 88 | mock_pl_cls.return_value = MagicMock() |
| 89 | - mock_factory.create.return_value = MagicMock() | 89 | + mock_disp_cls.return_value = MagicMock() |
| 90 | mock_app_instance = MagicMock() | 90 | mock_app_instance = MagicMock() |
| 91 | mock_app_cls.return_value = mock_app_instance | 91 | mock_app_cls.return_value = mock_app_instance |
| 92 | 92 | ||
| @@ -96,14 +96,14 @@ class TestAnalysisMain: | |||
| 96 | mock_app_instance.analyze.assert_called_once() | 96 | mock_app_instance.analyze.assert_called_once() |
| 97 | 97 | ||
| 98 | 98 | ||
| 99 | - @patch("msmodelslim.cli.analysis.__main__.AnalysisResultDisplayerFactory") | 99 | + @patch("msmodelslim.cli.analysis.__main__.LoggingAnalysisResultDisplayer") |
| 100 | 100 | ||
| 101 | - def test_main_dispatches_to_attn_scope_when_scope_is_attn(self, mock_pl_cls, mock_factory, mock_app_cls): | 101 | + def test_main_dispatches_to_attn_scope_when_scope_is_attn(self, mock_pl_cls, mock_disp_cls, mock_app_cls): |
| 102 | """主路径：scope=attn 时应构造 AttnArgs。""" | 102 | """主路径：scope=attn 时应构造 AttnArgs。""" |
| 103 | from unittest.mock import MagicMock | 103 | from unittest.mock import MagicMock |
| 104 | 104 | ||
| 105 | mock_pl_cls.return_value = MagicMock() | 105 | mock_pl_cls.return_value = MagicMock() |
| 106 | - mock_factory.create.return_value = MagicMock() | 106 | + mock_disp_cls.return_value = MagicMock() |
| 107 | mock_app_instance = MagicMock() | 107 | mock_app_instance = MagicMock() |
| 108 | mock_app_cls.return_value = mock_app_instance | 108 | mock_app_cls.return_value = mock_app_instance |
| 109 | 109 | ||
| @@ -1,598 +0,0 @@ | |||
| 1 | -#!/usr/bin/env python | ||
| 2 | -# -*- coding: UTF-8 -*- | ||
| 3 | - | ||
| 4 | -""" | ||
| 5 | -------------------------------------------------------------------------- | ||
| 6 | -This file is part of the MindStudio project. | ||
| 7 | -Copyright (c) 2026 Huawei Technologies Co.,Ltd. | ||
| 8 | - | ||
| 9 | -MindStudio is licensed under Mulan PSL v2. | ||
| 10 | -You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 11 | -You may obtain a copy of Mulan PSL v2 at: | ||
| 12 | - | ||
| 13 | - http://license.coscl.org.cn/MulanPSL2 | ||
| 14 | - | ||
| 15 | -THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 16 | -EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 17 | -MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 18 | -See the Mulan PSL v2 for more details. | ||
| 19 | -------------------------------------------------------------------------- | ||
| 20 | -""" | ||
| 21 | - | ||
| 22 | -import unittest | ||
| 23 | -from typing import Dict | ||
| 24 | - | ||
| 25 | -import torch | ||
| 26 | - | ||
| 27 | -from msmodelslim.processor.analysis.unary_operator.metrics.ra_compress import ( | ||
| 28 | - DUMMY_INPUT_LENGTH, | ||
| 29 | - REPET_TIMES, | ||
| 30 | - RaCompressAnalysisMethod, | ||
| 31 | -) | ||
| 32 | -from msmodelslim.processor.analysis.unary_operator.metrics.ra_compress.interface import ( | ||
| 33 | - RaCompressAnalysisInterface, | ||
| 34 | -) | ||
| 35 | - | ||
| 36 | - | ||
| 37 | -class FakeAdapter(RaCompressAnalysisInterface): | ||
| 38 | - """实现 RaCompressAnalysisInterface 的测试用 adapter。""" | ||
| 39 | - | ||
| 40 | - def get_ra_compress_proj_patterns(self) -> Dict[str, str]: | ||
| 41 | - return {"q": "q_proj", "k": "k_proj", "qkv": "qkv_proj"} | ||
| 42 | - | ||
| 43 | - | ||
| 44 | -class TestRaCompressPrefixScore(unittest.TestCase): | ||
| 45 | - """测试 _prefix_score_for_matrix — prefix matching 分数计算。""" | ||
| 46 | - | ||
| 47 | - def test_prefix_score_returns_zero_when_attn_not_2d(self): | ||
| 48 | - """非 2D tensor 返回 0。""" | ||
| 49 | - attn_1d = torch.ones(10) | ||
| 50 | - self.assertEqual(RaCompressAnalysisMethod._prefix_score_for_matrix(attn_1d), 0.0) | ||
| 51 | - | ||
| 52 | - def test_prefix_score_returns_zero_when_seq_len_below_dummy_length(self): | ||
| 53 | - """seq_len < DUMMY_INPUT_LENGTH 时没有完整段，返回 0。""" | ||
| 54 | - attn = torch.ones(DUMMY_INPUT_LENGTH - 1, DUMMY_INPUT_LENGTH - 1) | ||
| 55 | - attn = attn / attn.sum(dim=-1, keepdim=True) | ||
| 56 | - self.assertEqual(RaCompressAnalysisMethod._prefix_score_for_matrix(attn), 0.0) | ||
| 57 | - | ||
| 58 | - def test_prefix_score_returns_positive_when_attention_is_uniform(self): | ||
| 59 | - """均匀分布的注意力矩阵，prefix 分数应接近 1/seq_len（每个位置均匀分配）。""" | ||
| 60 | - seq_len = DUMMY_INPUT_LENGTH * 2 | ||
| 61 | - attn = torch.ones(seq_len, seq_len) | ||
| 62 | - # 下三角（含对角线）为有效，上三角为 0 | ||
| 63 | - causal_mask = torch.triu(torch.ones(seq_len, seq_len, dtype=torch.bool), diagonal=1) | ||
| 64 | - attn = attn.masked_fill(causal_mask, 0.0) | ||
| 65 | - attn = attn / attn.sum(dim=-1, keepdim=True) | ||
| 66 | - | ||
| 67 | - score = RaCompressAnalysisMethod._prefix_score_for_matrix(attn) | ||
| 68 | - # 对于均匀分布，prefix 偏移 +1 的位置应接近 1/seq_len | ||
| 69 | - self.assertGreater(score, 0.0) | ||
| 70 | - self.assertLess(score, 0.1) | ||
| 71 | - | ||
| 72 | - def test_prefix_score_returns_high_when_attention_on_prefix_offset(self): | ||
| 73 | - """对角线集中（主对角线+1 偏移）的注意力矩阵应有较高的 prefix 分数。""" | ||
| 74 | - seq_len = DUMMY_INPUT_LENGTH * 2 | ||
| 75 | - attn = torch.zeros(seq_len, seq_len) | ||
| 76 | - # 在 prefix matching 的位置（i + d*SEG + 1）设置高值 | ||
| 77 | - for k in range(1, 2): | ||
| 78 | - start = k * DUMMY_INPUT_LENGTH | ||
| 79 | - end = min((k + 1) * DUMMY_INPUT_LENGTH, seq_len) | ||
| 80 | - for i in range(start, end): | ||
| 81 | - col = i + (-k) * DUMMY_INPUT_LENGTH + 1 | ||
| 82 | - if 0 <= col < seq_len: | ||
| 83 | - attn[i, col] = 1.0 | ||
| 84 | - # 因果 mask | ||
| 85 | - causal_mask = torch.triu(torch.ones(seq_len, seq_len, dtype=torch.bool), diagonal=1) | ||
| 86 | - attn = attn.masked_fill(causal_mask, 0.0) | ||
| 87 | - row_sums = attn.sum(dim=-1, keepdim=True) | ||
| 88 | - row_sums[row_sums == 0] = 1.0 | ||
| 89 | - attn = attn / row_sums | ||
| 90 | - | ||
| 91 | - score = RaCompressAnalysisMethod._prefix_score_for_matrix(attn) | ||
| 92 | - self.assertGreater(score, 0.5) | ||
| 93 | - | ||
| 94 | - | ||
| 95 | -class TestRaCompressCopyingScore(unittest.TestCase): | ||
| 96 | - """测试 _copying_score_for_matrix — copying matching 分数计算。""" | ||
| 97 | - | ||
| 98 | - def test_copying_score_returns_zero_when_attn_not_2d(self): | ||
| 99 | - """非 2D tensor 返回 0。""" | ||
| 100 | - self.assertEqual(RaCompressAnalysisMethod._copying_score_for_matrix(torch.ones(5)), 0.0) | ||
| 101 | - | ||
| 102 | - def test_copying_score_differs_from_prefix_when_uniform_attention(self): | ||
| 103 | - """copying 分数与 prefix 分数不同（偏移差 1）。""" | ||
| 104 | - seq_len = DUMMY_INPUT_LENGTH * 2 | ||
| 105 | - attn = torch.ones(seq_len, seq_len) | ||
| 106 | - causal_mask = torch.triu(torch.ones(seq_len, seq_len, dtype=torch.bool), diagonal=1) | ||
| 107 | - attn = attn.masked_fill(causal_mask, 0.0) | ||
| 108 | - attn = attn / attn.sum(dim=-1, keepdim=True) | ||
| 109 | - | ||
| 110 | - prefix = RaCompressAnalysisMethod._prefix_score_for_matrix(attn) | ||
| 111 | - copying = RaCompressAnalysisMethod._copying_score_for_matrix(attn) | ||
| 112 | - # 均匀分布下两者应该很接近但不完全相等（偏移差 1） | ||
| 113 | - self.assertGreater(prefix, 0.0) | ||
| 114 | - self.assertGreater(copying, 0.0) | ||
| 115 | - | ||
| 116 | - def test_copying_score_returns_zero_when_seq_len_below_dummy_length(self): | ||
| 117 | - """seq_len < DUMMY_INPUT_LENGTH 时没有完整段，copying 返回 0。""" | ||
| 118 | - attn = torch.ones(DUMMY_INPUT_LENGTH - 1, DUMMY_INPUT_LENGTH - 1) | ||
| 119 | - attn = attn / attn.sum(dim=-1, keepdim=True) | ||
| 120 | - self.assertEqual(RaCompressAnalysisMethod._copying_score_for_matrix(attn), 0.0) | ||
| 121 | - | ||
| 122 | - | ||
| 123 | -class TestRaCompressMaxEveryGroup(unittest.TestCase): | ||
| 124 | - """测试 _max_every_group — GQA 分组取 max。""" | ||
| 125 | - | ||
| 126 | - def test_max_every_group_returns_data_unchanged_when_n_is_1(self): | ||
| 127 | - """n=1（无分组）时原样返回。""" | ||
| 128 | - data = {0: [1.0, 2.0, 3.0], 1: [4.0, 5.0]} | ||
| 129 | - result = RaCompressAnalysisMethod._max_every_group(data, 1) | ||
| 130 | - self.assertEqual(result, data) | ||
| 131 | - | ||
| 132 | - def test_max_every_group_returns_data_unchanged_when_n_is_zero(self): | ||
| 133 | - """n=0 时按 n<=1 分支原样返回（边界保护）。""" | ||
| 134 | - data = {0: [1.0, 2.0]} | ||
| 135 | - result = RaCompressAnalysisMethod._max_every_group(data, 0) | ||
| 136 | - self.assertEqual(result, data) | ||
| 137 | - | ||
| 138 | - def test_max_every_group_returns_group_max_when_n_is_2(self): | ||
| 139 | - """n=2 时每 2 个 head 取 max。""" | ||
| 140 | - data = {0: [1.0, 3.0, 2.0, 4.0]} | ||
| 141 | - result = RaCompressAnalysisMethod._max_every_group(data, 2) | ||
| 142 | - self.assertEqual(result[0], [3.0, 4.0]) | ||
| 143 | - | ||
| 144 | - def test_max_every_group_returns_remainder_as_group_when_heads_not_divisible(self): | ||
| 145 | - """head 数不整除 n 时余项单独成组。""" | ||
| 146 | - data = {0: [1.0, 5.0, 3.0]} | ||
| 147 | - result = RaCompressAnalysisMethod._max_every_group(data, 2) | ||
| 148 | - self.assertEqual(result[0], [5.0, 3.0]) | ||
| 149 | - | ||
| 150 | - | ||
| 151 | -class TestRaCompressSelectTopHeads(unittest.TestCase): | ||
| 152 | - """测试 _select_top_heads — 按 ratio 选 top heads。""" | ||
| 153 | - | ||
| 154 | - def test_select_top_heads_returns_empty_when_data_empty(self): | ||
| 155 | - """空数据返回空 dict。""" | ||
| 156 | - self.assertEqual(RaCompressAnalysisMethod._select_top_heads({}, 0.14), {}) | ||
| 157 | - | ||
| 158 | - def test_select_top_heads_returns_empty_when_ratio_is_zero(self): | ||
| 159 | - """ratio=0 时 percent_index=0，所有层返回空索引列表（边界）。""" | ||
| 160 | - data = {0: [0.1, 0.9]} | ||
| 161 | - result = RaCompressAnalysisMethod._select_top_heads(data, 0.0) | ||
| 162 | - self.assertEqual(result[0], []) | ||
| 163 | - | ||
| 164 | - def test_select_top_heads_returns_count_when_ratio_quarter(self): | ||
| 165 | - """25% ratio 正确选择 top heads。""" | ||
| 166 | - data = {0: [0.1, 0.9, 0.5, 0.3], 1: [0.8, 0.2, 0.7, 0.4]} | ||
| 167 | - result = RaCompressAnalysisMethod._select_top_heads(data, 0.25) | ||
| 168 | - # 8 个值，25% = 2 个，top 2 = [0.9, 0.8] | ||
| 169 | - all_selected = [] | ||
| 170 | - for heads in result.values(): | ||
| 171 | - all_selected.extend(heads) | ||
| 172 | - self.assertEqual(len(all_selected), 2) | ||
| 173 | - | ||
| 174 | - def test_select_top_heads_returns_one_when_ratio_one_percent(self): | ||
| 175 | - """1% ratio 在大数据集上只选极少 head。""" | ||
| 176 | - data = {0: list(range(100))} | ||
| 177 | - result = RaCompressAnalysisMethod._select_top_heads(data, 0.01) | ||
| 178 | - # 100 个值，1% = 1 个，top 1 = [99] | ||
| 179 | - self.assertEqual(len(result[0]), 1) | ||
| 180 | - | ||
| 181 | - def test_select_top_heads_returns_all_indices_when_ratio_is_one(self): | ||
| 182 | - """ratio=1.0 选择所有 head。""" | ||
| 183 | - data = {0: [0.1, 0.2, 0.3]} | ||
| 184 | - result = RaCompressAnalysisMethod._select_top_heads(data, 1.0) | ||
| 185 | - self.assertEqual(result[0], [0, 1, 2]) | ||
| 186 | - | ||
| 187 | - | ||
| 188 | -class TestRaCompressGetCompressHeads(unittest.TestCase): | ||
| 189 | - """测试 get_compress_heads — 完整 head 选择流水线。""" | ||
| 190 | - | ||
| 191 | - def setUp(self): | ||
| 192 | - self.method = RaCompressAnalysisMethod(adapter=FakeAdapter()) | ||
| 193 | - self.method._num_attention_heads = 4 | ||
| 194 | - self.method._num_key_value_heads = 2 | ||
| 195 | - self.method._head_dim = 8 | ||
| 196 | - | ||
| 197 | - def test_get_compress_heads_returns_empty_when_no_scores(self): | ||
| 198 | - """无分数数据时返回空 head_dict。""" | ||
| 199 | - result = self.method.get_compress_heads() | ||
| 200 | - self.assertEqual(result, {"prefix_matching": {}, "copying": {}}) | ||
| 201 | - | ||
| 202 | - def test_get_compress_heads_selects_induction_when_prefix_scores_high(self): | ||
| 203 | - """induction heads 按 prefix 分数选择。""" | ||
| 204 | - self.method._prefix_scores = { | ||
| 205 | - 0: [0.9, 0.1, 0.8, 0.2], | ||
| 206 | - 1: [0.1, 0.9, 0.2, 0.8], | ||
| 207 | - } | ||
| 208 | - self.method._copying_scores = { | ||
| 209 | - 0: [0.0, 0.0, 0.0, 0.0], | ||
| 210 | - 1: [0.0, 0.0, 0.0, 0.0], | ||
| 211 | - } | ||
| 212 | - result = self.method.get_compress_heads() | ||
| 213 | - prefix_map = result["prefix_matching"] | ||
| 214 | - # GQA: num_heads=4, kv_heads=2, n=2, 分组后 [max(0.9,0.1), max(0.8,0.2)] = [0.9, 0.8] | ||
| 215 | - # 14% of 4 values (2 layers * 2 kv heads) = round(4 * 0.14) = 1 → top 1 = [0.9] | ||
| 216 | - # 0.9 在 layer 0, group 0 → kv head 0 | ||
| 217 | - self.assertIn(0, prefix_map) | ||
| 218 | - | ||
| 219 | - def test_get_compress_heads_selects_echo_when_copying_scores_high(self): | ||
| 220 | - """echo heads 按 copying 分数选择。""" | ||
| 221 | - self.method._prefix_scores = { | ||
| 222 | - 0: [0.0, 0.0, 0.0, 0.0], | ||
| 223 | - 1: [0.0, 0.0, 0.0, 0.0], | ||
| 224 | - } | ||
| 225 | - self.method._copying_scores = { | ||
| 226 | - 0: [0.1, 0.9, 0.2, 0.8], | ||
| 227 | - 1: [0.3, 0.7, 0.4, 0.6], | ||
| 228 | - } | ||
| 229 | - result = self.method.get_compress_heads() | ||
| 230 | - copying_map = result["copying"] | ||
| 231 | - # GQA: num_heads=4, kv_heads=2, n=2 | ||
| 232 | - # grouped: layer0=[max(0.1,0.9), max(0.2,0.8)]=[0.9,0.8], layer1=[max(0.3,0.7), max(0.4,0.6)]=[0.7,0.6] | ||
| 233 | - # 4 values, 1% = round(4*0.01)=0 → 0 selected, 用更大 ratio 测试 | ||
| 234 | - # 改用 induction_head_ratio=0.5 模拟 | ||
| 235 | - self.method._induction_head_ratio = 0.5 | ||
| 236 | - self.method._echo_head_ratio = 0.5 | ||
| 237 | - result = self.method.get_compress_heads() | ||
| 238 | - copying_map = result["copying"] | ||
| 239 | - self.assertIn(0, copying_map) | ||
| 240 | - | ||
| 241 | - def test_get_compress_heads_removes_empty_lists_when_layer_scores_zero(self): | ||
| 242 | - """空列表的层被移除。""" | ||
| 243 | - self.method._prefix_scores = { | ||
| 244 | - 0: [0.0, 0.0, 0.0, 0.0], | ||
| 245 | - 1: [0.9, 0.1, 0.8, 0.2], | ||
| 246 | - } | ||
| 247 | - self.method._copying_scores = { | ||
| 248 | - 0: [0.0, 0.0, 0.0, 0.0], | ||
| 249 | - 1: [0.0, 0.0, 0.0, 0.0], | ||
| 250 | - } | ||
| 251 | - result = self.method.get_compress_heads() | ||
| 252 | - # layer 0 的所有分数都是 0，不应出现在 prefix_matching 中 | ||
| 253 | - self.assertNotIn(0, result["prefix_matching"]) | ||
| 254 | - | ||
| 255 | - def test_get_compress_heads_returns_heads_when_no_gqa(self): | ||
| 256 | - """num_kv_heads == num_attention_heads（无 GQA）时分组 n=1 原样选择（边界）。""" | ||
| 257 | - method = RaCompressAnalysisMethod(adapter=FakeAdapter()) | ||
| 258 | - method._num_attention_heads = 2 | ||
| 259 | - method._num_key_value_heads = 2 # 无 GQA，分组 n=1 | ||
| 260 | - method._head_dim = 4 | ||
| 261 | - method._induction_head_ratio = 0.5 | ||
| 262 | - method._prefix_scores = {0: [0.9, 0.1]} | ||
| 263 | - method._copying_scores = {0: [0.0, 0.0]} | ||
| 264 | - result = method.get_compress_heads() | ||
| 265 | - self.assertIsInstance(result["prefix_matching"], dict) | ||
| 266 | - | ||
| 267 | - | ||
| 268 | -class TestRaCompressEnrichLayerScores(unittest.TestCase): | ||
| 269 | - """测试 enrich_layer_scores — 将 head 信息写入 layer_scores。""" | ||
| 270 | - | ||
| 271 | - def setUp(self): | ||
| 272 | - self.method = RaCompressAnalysisMethod(adapter=FakeAdapter()) | ||
| 273 | - self.method._num_attention_heads = 4 | ||
| 274 | - self.method._num_key_value_heads = 2 | ||
| 275 | - self.method._head_dim = 8 | ||
| 276 | - | ||
| 277 | - def test_enrich_layer_scores_populates_heads_when_scores_exist(self): | ||
| 278 | - """正确填充 induction_heads / echo_heads。""" | ||
| 279 | - self.method._prefix_scores = {0: [0.9, 0.1, 0.8, 0.2]} | ||
| 280 | - self.method._copying_scores = {0: [0.1, 0.9, 0.2, 0.8]} | ||
| 281 | - self.method._layer_idx_to_name = {0: "model.layers.0.self_attn.q_proj"} | ||
| 282 | - | ||
| 283 | - layer_scores = [{"name": "model.layers.0.self_attn.q_proj", "score": 0.5}] | ||
| 284 | - self.method.enrich_layer_scores(layer_scores) | ||
| 285 | - | ||
| 286 | - self.assertIn("induction_heads", layer_scores[0]) | ||
| 287 | - self.assertIn("echo_heads", layer_scores[0]) | ||
| 288 | - self.assertIsInstance(layer_scores[0]["induction_heads"], list) | ||
| 289 | - self.assertIsInstance(layer_scores[0]["echo_heads"], list) | ||
| 290 | - | ||
| 291 | - def test_enrich_layer_scores_sets_empty_lists_when_layer_missing(self): | ||
| 292 | - """layer_scores 中的层没有对应分数时，heads 为空列表。""" | ||
| 293 | - self.method._prefix_scores = {} | ||
| 294 | - self.method._copying_scores = {} | ||
| 295 | - self.method._layer_idx_to_name = {} | ||
| 296 | - | ||
| 297 | - layer_scores = [{"name": "model.layers.0.self_attn.q_proj", "score": 0.5}] | ||
| 298 | - self.method.enrich_layer_scores(layer_scores) | ||
| 299 | - | ||
| 300 | - self.assertEqual(layer_scores[0]["induction_heads"], []) | ||
| 301 | - self.assertEqual(layer_scores[0]["echo_heads"], []) | ||
| 302 | - | ||
| 303 | - def test_enrich_layer_scores_keeps_empty_when_layer_scores_empty(self): | ||
| 304 | - """layer_scores 为空列表时不报错、保持空（异常/边界）。""" | ||
| 305 | - layer_scores = [] | ||
| 306 | - self.method.enrich_layer_scores(layer_scores) | ||
| 307 | - self.assertEqual(layer_scores, []) | ||
| 308 | - | ||
| 309 | - | ||
| 310 | -class TestRaCompressFlattenTo2D(unittest.TestCase): | ||
| 311 | - """测试 _flatten_to_2d — tensor 展平。""" | ||
| 312 | - | ||
| 313 | - def test_flatten_to_2d_returns_unchanged_when_input_2d(self): | ||
| 314 | - """2D tensor 原样返回。""" | ||
| 315 | - t = torch.randn(10, 8) | ||
| 316 | - result = RaCompressAnalysisMethod._flatten_to_2d(t) | ||
| 317 | - self.assertEqual(result.shape, (10, 8)) | ||
| 318 | - self.assertTrue(torch.equal(result, t)) | ||
| 319 | - | ||
| 320 | - def test_flatten_to_2d_returns_flattened_when_input_3d(self): | ||
| 321 | - """3D tensor [batch, seq, dim] → [batch*seq, dim]。""" | ||
| 322 | - t = torch.randn(2, 5, 8) | ||
| 323 | - result = RaCompressAnalysisMethod._flatten_to_2d(t) | ||
| 324 | - self.assertEqual(result.shape, (10, 8)) | ||
| 325 | - | ||
| 326 | - def test_flatten_to_2d_returns_flattened_when_input_4d(self): | ||
| 327 | - """4D tensor → [*, dim]。""" | ||
| 328 | - t = torch.randn(2, 3, 4, 8) | ||
| 329 | - result = RaCompressAnalysisMethod._flatten_to_2d(t) | ||
| 330 | - self.assertEqual(result.shape, (24, 8)) | ||
| 331 | - | ||
| 332 | - | ||
| 333 | -class TestRaCompressIsTargetLayer(unittest.TestCase): | ||
| 334 | - """测试 _is_target_layer — 层名匹配。""" | ||
| 335 | - | ||
| 336 | - def setUp(self): | ||
| 337 | - self.method = RaCompressAnalysisMethod(adapter=FakeAdapter()) | ||
| 338 | - | ||
| 339 | - def test_is_target_layer_returns_true_when_q_proj(self): | ||
| 340 | - self.assertTrue(self.method._is_target_layer("model.layers.0.self_attn.q_proj")) | ||
| 341 | - | ||
| 342 | - def test_is_target_layer_returns_true_when_k_proj(self): | ||
| 343 | - self.assertTrue(self.method._is_target_layer("model.layers.0.self_attn.k_proj")) | ||
| 344 | - | ||
| 345 | - def test_is_target_layer_returns_true_when_qkv_proj(self): | ||
| 346 | - self.assertTrue(self.method._is_target_layer("model.layers.0.self_attn.qkv_proj")) | ||
| 347 | - | ||
| 348 | - def test_is_target_layer_returns_false_when_non_proj_layer(self): | ||
| 349 | - self.assertFalse(self.method._is_target_layer("model.layers.0.mlp.gate_proj")) | ||
| 350 | - self.assertFalse(self.method._is_target_layer("model.layers.0.self_attn.o_proj")) | ||
| 351 | - | ||
| 352 | - | ||
| 353 | -class TestRaCompressComputeScore(unittest.TestCase): | ||
| 354 | - """测试 compute_score — 分数计算分派。""" | ||
| 355 | - | ||
| 356 | - def setUp(self): | ||
| 357 | - self.method = RaCompressAnalysisMethod(adapter=FakeAdapter()) | ||
| 358 | - self.method._num_attention_heads = 2 | ||
| 359 | - self.method._num_key_value_heads = 2 | ||
| 360 | - self.method._head_dim = 4 | ||
| 361 | - | ||
| 362 | - def test_compute_score_returns_zero_when_no_outputs(self): | ||
| 363 | - """无 output 数据返回 0。""" | ||
| 364 | - result = self.method.compute_score({"layer_name": "q_proj", "outputs": []}) | ||
| 365 | - self.assertEqual(result, 0.0) | ||
| 366 | - | ||
| 367 | - def test_compute_score_returns_zero_when_k_layer(self): | ||
| 368 | - """K 层不计算分数，返回 0。""" | ||
| 369 | - result = self.method.compute_score({"layer_name": "k_proj", "outputs": [torch.randn(1, 4)]}) | ||
| 370 | - self.assertEqual(result, 0.0) | ||
| 371 | - | ||
| 372 | - def test_compute_score_returns_zero_when_non_target_layer(self): | ||
| 373 | - """非 Q/K/QKV 层返回 0。""" | ||
| 374 | - result = self.method.compute_score({"layer_name": "o_proj", "outputs": [torch.randn(1, 4)]}) | ||
| 375 | - self.assertEqual(result, 0.0) | ||
| 376 | - | ||
| 377 | - def test_compute_score_returns_zero_when_q_layer_has_no_k_output(self): | ||
| 378 | - """Q 层但没有对应 K 输出时返回 0。""" | ||
| 379 | - self.method._q_outputs = {"model.layers.0.self_attn.q_proj": torch.randn(10, 8)} | ||
| 380 | - result = self.method.compute_score( | ||
| 381 | - { | ||
| 382 | - "layer_name": "model.layers.0.self_attn.q_proj", | ||
| 383 | - "outputs": [torch.randn(10, 8)], | ||
| 384 | - } | ||
| 385 | - ) | ||
| 386 | - self.assertEqual(result, 0.0) | ||
| 387 | - | ||
| 388 | - def test_compute_score_returns_zero_when_layer_name_empty(self): | ||
| 389 | - """layer_name 为空字符串时不匹配任何模式，返回 0（异常/边界）。""" | ||
| 390 | - result = self.method.compute_score({"layer_name": "", "outputs": [torch.randn(1, 4)]}) | ||
| 391 | - self.assertEqual(result, 0.0) | ||
| 392 | - | ||
| 393 | - | ||
| 394 | -class TestRaCompressComputeQkScores(unittest.TestCase): | ||
| 395 | - """测试 _compute_qk_scores — Q@K^T 分数计算。""" | ||
| 396 | - | ||
| 397 | - def test_compute_qk_scores_returns_empty_when_attention_config_not_set(self): | ||
| 398 | - """注意力配置未提取时返回空列表。""" | ||
| 399 | - method = RaCompressAnalysisMethod(adapter=FakeAdapter()) | ||
| 400 | - method._num_attention_heads = 0 | ||
| 401 | - method._head_dim = 0 | ||
| 402 | - q = torch.randn(10, 8) | ||
| 403 | - k = torch.randn(10, 8) | ||
| 404 | - prefix, copying = method._compute_qk_scores("q_proj", q, k) | ||
| 405 | - self.assertEqual(prefix, []) | ||
| 406 | - self.assertEqual(copying, []) | ||
| 407 | - | ||
| 408 | - def test_compute_qk_scores_returns_empty_when_token_count_insufficient(self): | ||
| 409 | - """token 数量不足时返回空列表（异常被 catch 并 log warning）。""" | ||
| 410 | - method = RaCompressAnalysisMethod(adapter=FakeAdapter()) | ||
| 411 | - method._num_attention_heads = 2 | ||
| 412 | - method._num_key_value_heads = 2 | ||
| 413 | - method._head_dim = 4 | ||
| 414 | - | ||
| 415 | - # 只有 10 个 token，远小于 2500*4=10000 | ||
| 416 | - q = torch.randn(10, 8) | ||
| 417 | - k = torch.randn(10, 8) | ||
| 418 | - | ||
| 419 | - prefix, copying = method._compute_qk_scores("q_proj", q, k) | ||
| 420 | - self.assertEqual(prefix, []) | ||
| 421 | - self.assertEqual(copying, []) | ||
| 422 | - | ||
| 423 | - def test_compute_qk_scores_returns_per_head_scores_when_tokens_sufficient(self): | ||
| 424 | - """满足条件时返回每头分数列表。""" | ||
| 425 | - method = RaCompressAnalysisMethod(adapter=FakeAdapter()) | ||
| 426 | - method._num_attention_heads = 2 | ||
| 427 | - method._num_key_value_heads = 2 | ||
| 428 | - method._head_dim = 4 | ||
| 429 | - method._induction_head_ratio = 0.5 | ||
| 430 | - method._echo_head_ratio = 0.5 | ||
| 431 | - | ||
| 432 | - total_tokens = DUMMY_INPUT_LENGTH * REPET_TIMES | ||
| 433 | - q = torch.randn(total_tokens, 8) | ||
| 434 | - k = torch.randn(total_tokens, 8) | ||
| 435 | - | ||
| 436 | - prefix, copying = method._compute_qk_scores("q_proj", q, k) | ||
| 437 | - self.assertEqual(len(prefix), 2) | ||
| 438 | - self.assertEqual(len(copying), 2) | ||
| 439 | - # 分数在 [0, 1] 范围内（softmax 概率） | ||
| 440 | - for score in prefix: | ||
| 441 | - self.assertGreaterEqual(score, 0.0) | ||
| 442 | - self.assertLessEqual(score, 1.0) | ||
| 443 | - for score in copying: | ||
| 444 | - self.assertGreaterEqual(score, 0.0) | ||
| 445 | - self.assertLessEqual(score, 1.0) | ||
| 446 | - | ||
| 447 | - | ||
| 448 | -class TestRaCompressInterfaceIntegration(unittest.TestCase): | ||
| 449 | - """测试 RaCompressAnalysisInterface 接口集成。""" | ||
| 450 | - | ||
| 451 | - def test_init_loads_patterns_when_adapter_provided(self): | ||
| 452 | - """adapter 提供的名称模式被正确使用。""" | ||
| 453 | - method = RaCompressAnalysisMethod(adapter=FakeAdapter()) | ||
| 454 | - self.assertEqual(method._q_name_pattern, "q_proj") | ||
| 455 | - self.assertEqual(method._k_name_pattern, "k_proj") | ||
| 456 | - self.assertEqual(method._qkv_name_pattern, "qkv_proj") | ||
| 457 | - | ||
| 458 | - def test_init_uses_default_patterns_when_adapter_none(self): | ||
| 459 | - """无 adapter 时使用默认名称模式。""" | ||
| 460 | - method = RaCompressAnalysisMethod(adapter=None) | ||
| 461 | - self.assertEqual(method._q_name_pattern, "q_proj") | ||
| 462 | - self.assertEqual(method._k_name_pattern, "k_proj") | ||
| 463 | - self.assertEqual(method._qkv_name_pattern, "qkv_proj") | ||
| 464 | - | ||
| 465 | - def test_init_uses_custom_patterns_when_adapter_overrides(self): | ||
| 466 | - """adapter 提供自定义名称模式。""" | ||
| 467 | - | ||
| 468 | - class CustomAdapter(RaCompressAnalysisInterface): | ||
| 469 | - def get_ra_compress_proj_patterns(self) -> Dict[str, str]: | ||
| 470 | - return {"q": "query", "k": "key", "qkv": "qkv_fused"} | ||
| 471 | - | ||
| 472 | - method = RaCompressAnalysisMethod(adapter=CustomAdapter()) | ||
| 473 | - self.assertEqual(method._q_name_pattern, "query") | ||
| 474 | - self.assertEqual(method._k_name_pattern, "key") | ||
| 475 | - self.assertEqual(method._qkv_name_pattern, "qkv_fused") | ||
| 476 | - self.assertTrue(method._is_target_layer("model.layers.0.query")) | ||
| 477 | - self.assertFalse(method._is_target_layer("model.layers.0.q_proj")) | ||
| 478 | - | ||
| 479 | - | ||
| 480 | -class TestRaCompressHookBehavior(unittest.TestCase): | ||
| 481 | - """测试 get_hook 注册的 hook 行为。""" | ||
| 482 | - | ||
| 483 | - def setUp(self): | ||
| 484 | - self.method = RaCompressAnalysisMethod(adapter=FakeAdapter()) | ||
| 485 | - self.hook = self.method.get_hook() | ||
| 486 | - | ||
| 487 | - def test_hook_stores_q_output_when_q_proj_layer(self): | ||
| 488 | - """hook 正确存储 Q 输出。""" | ||
| 489 | - stats_dict = {} | ||
| 490 | - q_output = torch.randn(5, 8) | ||
| 491 | - self.hook(None, None, q_output, "model.layers.0.q_proj", stats_dict) | ||
| 492 | - self.assertIn("model.layers.0.q_proj", self.method._q_outputs) | ||
| 493 | - self.assertTrue(torch.equal(self.method._q_outputs["model.layers.0.q_proj"], q_output)) | ||
| 494 | - | ||
| 495 | - def test_hook_stores_k_output_when_k_proj_layer(self): | ||
| 496 | - """hook 正确存储 K 输出。""" | ||
| 497 | - stats_dict = {} | ||
| 498 | - k_output = torch.randn(5, 8) | ||
| 499 | - self.hook(None, None, k_output, "model.layers.0.k_proj", stats_dict) | ||
| 500 | - self.assertIn("model.layers.0.k_proj", self.method._k_outputs) | ||
| 501 | - | ||
| 502 | - def test_hook_stores_qkv_to_both_when_qkv_proj_layer(self): | ||
| 503 | - """QKV 融合层输出同时存入 Q 和 K。""" | ||
| 504 | - stats_dict = {} | ||
| 505 | - qkv_output = torch.randn(5, 24) | ||
| 506 | - self.hook(None, None, qkv_output, "model.layers.0.qkv_proj", stats_dict) | ||
| 507 | - self.assertIn("model.layers.0.qkv_proj", self.method._q_outputs) | ||
| 508 | - self.assertIn("model.layers.0.qkv_proj", self.method._k_outputs) | ||
| 509 | - | ||
| 510 | - def test_hook_populates_stats_dict_when_called(self): | ||
| 511 | - """hook 正确填充 stats_dict。""" | ||
| 512 | - stats_dict = {} | ||
| 513 | - output = torch.randn(5, 8) | ||
| 514 | - self.hook(None, None, output, "model.layers.0.q_proj", stats_dict) | ||
| 515 | - self.assertIn("model.layers.0.q_proj", stats_dict) | ||
| 516 | - self.assertIn("outputs", stats_dict["model.layers.0.q_proj"]) | ||
| 517 | - self.assertEqual(len(stats_dict["model.layers.0.q_proj"]["outputs"]), 1) | ||
| 518 | - | ||
| 519 | - def test_hook_stores_first_element_when_output_is_tuple(self): | ||
| 520 | - """hook 处理 tuple 输出（取第 0 项）。""" | ||
| 521 | - stats_dict = {} | ||
| 522 | - output = (torch.randn(5, 8), torch.randn(5, 8)) | ||
| 523 | - self.hook(None, None, output, "model.layers.0.q_proj", stats_dict) | ||
| 524 | - stored = self.method._q_outputs["model.layers.0.q_proj"] | ||
| 525 | - self.assertTrue(torch.equal(stored, output[0])) | ||
| 526 | - | ||
| 527 | - def test_hook_appends_multiple_outputs_when_called_repeatedly(self): | ||
| 528 | - """hook 多次调用时 outputs 列表累积（边界：多次前向）。""" | ||
| 529 | - stats_dict = {} | ||
| 530 | - self.hook(None, None, torch.randn(5, 8), "model.layers.0.q_proj", stats_dict) | ||
| 531 | - self.hook(None, None, torch.randn(5, 8), "model.layers.0.q_proj", stats_dict) | ||
| 532 | - self.assertEqual(len(stats_dict["model.layers.0.q_proj"]["outputs"]), 2) | ||
| 533 | - | ||
| 534 | - | ||
| 535 | -class TestRaCompressEndToEnd(unittest.TestCase): | ||
| 536 | - """端到端测试：从 hook 到 head 选择的完整流程。""" | ||
| 537 | - | ||
| 538 | - def test_pipeline_selects_heads_when_qk_separate_projection(self): | ||
| 539 | - """Q/K 分离投影的完整流程：hook → compute_score → get_compress_heads。""" | ||
| 540 | - method = RaCompressAnalysisMethod(adapter=FakeAdapter()) | ||
| 541 | - method._num_attention_heads = 2 | ||
| 542 | - method._num_key_value_heads = 2 | ||
| 543 | - method._head_dim = 4 | ||
| 544 | - method._induction_head_ratio = 0.5 | ||
| 545 | - method._echo_head_ratio = 0.5 | ||
| 546 | - | ||
| 547 | - total_tokens = DUMMY_INPUT_LENGTH * REPET_TIMES | ||
| 548 | - q_output = torch.randn(total_tokens, 8) | ||
| 549 | - k_output = torch.randn(total_tokens, 8) | ||
| 550 | - | ||
| 551 | - hook = method.get_hook() | ||
| 552 | - stats_dict_q = {} | ||
| 553 | - stats_dict_k = {} | ||
| 554 | - hook(None, None, q_output, "model.layers.0.q_proj", stats_dict_q) | ||
| 555 | - hook(None, None, k_output, "model.layers.0.k_proj", stats_dict_k) | ||
| 556 | - | ||
| 557 | - # compute_score 对 Q 层计算 | ||
| 558 | - score = method.compute_score(stats_dict_q["model.layers.0.q_proj"]) | ||
| 559 | - self.assertGreaterEqual(score, 0.0) | ||
| 560 | - | ||
| 561 | - # get_compress_heads | ||
| 562 | - head_dict = method.get_compress_heads() | ||
| 563 | - self.assertIn("prefix_matching", head_dict) | ||
| 564 | - self.assertIn("copying", head_dict) | ||
| 565 | - | ||
| 566 | - # enrich_layer_scores | ||
| 567 | - layer_scores = [{"name": "model.layers.0.q_proj", "score": score}] | ||
| 568 | - method.enrich_layer_scores(layer_scores) | ||
| 569 | - self.assertIn("induction_heads", layer_scores[0]) | ||
| 570 | - self.assertIn("echo_heads", layer_scores[0]) | ||
| 571 | - | ||
| 572 | - def test_pipeline_selects_heads_when_qkv_fused_projection(self): | ||
| 573 | - """QKV 融合投影的完整流程：hook → compute_score → get_compress_heads（正常场景补充）。""" | ||
| 574 | - method = RaCompressAnalysisMethod(adapter=FakeAdapter()) | ||
| 575 | - method._num_attention_heads = 2 | ||
| 576 | - method._num_key_value_heads = 2 | ||
| 577 | - method._head_dim = 4 | ||
| 578 | - method._induction_head_ratio = 0.5 | ||
| 579 | - method._echo_head_ratio = 0.5 | ||
| 580 | - | ||
| 581 | - total_tokens = DUMMY_INPUT_LENGTH * REPET_TIMES | ||
| 582 | - # q_dim(num_heads*head_dim=2*4=8) + k_dim(num_kv*head_dim=2*4=8) = 16 | ||
| 583 | - qkv_output = torch.randn(total_tokens, 16) | ||
| 584 | - | ||
| 585 | - hook = method.get_hook() | ||
| 586 | - stats_dict = {} | ||
| 587 | - hook(None, None, qkv_output, "model.layers.0.qkv_proj", stats_dict) | ||
| 588 | - | ||
| 589 | - score = method.compute_score(stats_dict["model.layers.0.qkv_proj"]) | ||
| 590 | - self.assertGreaterEqual(score, 0.0) | ||
| 591 | - | ||
| 592 | - head_dict = method.get_compress_heads() | ||
| 593 | - self.assertIn("prefix_matching", head_dict) | ||
| 594 | - self.assertIn("copying", head_dict) | ||
| 595 | - | ||
| 596 | - | ||
| 597 | -if __name__ == "__main__": | ||
| 598 | - unittest.main() | ||
| @@ -23,14 +23,13 @@ import unittest | |||
| 23 | from types import SimpleNamespace | 23 | from types import SimpleNamespace |
| 24 | from unittest.mock import MagicMock, patch | 24 | from unittest.mock import MagicMock, patch |
| 25 | 25 | ||
| 26 | -from torch import nn | 26 | +import torch.nn as nn |
| 27 | 27 | ||
| 28 | from msmodelslim.core.base.protocol import BatchProcessRequest | 28 | from msmodelslim.core.base.protocol import BatchProcessRequest |
| 29 | from msmodelslim.processor.analysis.unary_operator.processor import ( | 29 | from msmodelslim.processor.analysis.unary_operator.processor import ( |
| 30 | UnaryAnalysisProcessor, | 30 | UnaryAnalysisProcessor, |
| 31 | UnaryAnalysisProcessorConfig, | 31 | UnaryAnalysisProcessorConfig, |
| 32 | ) | 32 | ) |
| 33 | -from msmodelslim.utils.exception import UnexpectedError | ||
| 34 | 33 | ||
| 35 | 34 | ||
| 36 | class TinyBlock(nn.Module): | 35 | class TinyBlock(nn.Module): |
| @@ -62,7 +61,7 @@ class TestUnaryAnalysisProcessor(unittest.TestCase): | |||
| 62 | 61 | ||
| 63 | processor = UnaryAnalysisProcessor(self.model, self.config) | 62 | processor = UnaryAnalysisProcessor(self.model, self.config) |
| 64 | 63 | ||
| 65 | - mock_create_method.assert_called_once_with("std", adapter=None) | 64 | + mock_create_method.assert_called_once_with("std") |
| 66 | self.assertEqual(processor.config, self.config) | 65 | self.assertEqual(processor.config, self.config) |
| 67 | self.assertIs(processor._analysis_method, fake_method) | 66 | self.assertIs(processor._analysis_method, fake_method) |
| 68 | self.assertEqual(processor._target_layers, []) | 67 | self.assertEqual(processor._target_layers, []) |
| @@ -163,29 +162,6 @@ class TestUnaryAnalysisProcessor(unittest.TestCase): | |||
| 163 | self.assertEqual(fake_ctx["layer_analysis"].debug["method"], "std") | 162 | self.assertEqual(fake_ctx["layer_analysis"].debug["method"], "std") |
| 164 | self.assertEqual(fake_ctx["layer_analysis"].debug["patterns"], self.config.patterns) | 163 | self.assertEqual(fake_ctx["layer_analysis"].debug["patterns"], self.config.patterns) |
| 165 | 164 | ||
| 166 | - | ||
| 167 | - | ||
| 168 | - def test_pre_run_raises_unexpected_error_when_context_is_none(self, mock_create_method, mock_get_current_context): | ||
| 169 | - """上下文缺失时 pre_run 抛 UnexpectedError（异常场景）。""" | ||
| 170 | - fake_method = self._build_fake_method() | ||
| 171 | - mock_create_method.return_value = fake_method | ||
| 172 | - mock_get_current_context.return_value = None | ||
| 173 | - | ||
| 174 | - processor = UnaryAnalysisProcessor(self.model, self.config) | ||
| 175 | - with self.assertRaises(UnexpectedError): | ||
| 176 | - processor.pre_run() | ||
| 177 | - | ||
| 178 | - | ||
| 179 | - | ||
| 180 | - def test_post_run_returns_silently_when_context_is_none(self, mock_create_method, mock_get_current_context): | ||
| 181 | - """上下文缺失时 post_run 静默返回，不抛异常（边界场景）。""" | ||
| 182 | - fake_method = self._build_fake_method() | ||
| 183 | - mock_create_method.return_value = fake_method | ||
| 184 | - mock_get_current_context.return_value = None | ||
| 185 | - | ||
| 186 | - processor = UnaryAnalysisProcessor(self.model, self.config) | ||
| 187 | - processor.post_run() # 不抛异常即通过 | ||
| 188 | - | ||
| 189 | 165 | ||
| 190 | def test_get_layer_scores_return_scores_when_called(self, mock_create_method): | 166 | def test_get_layer_scores_return_scores_when_called(self, mock_create_method): |
| 191 | fake_method = self._build_fake_method() | 167 | fake_method = self._build_fake_method() |