已合并
feat(trust_ai): add document #930
Wu,Qiang-Roy创建于 3月25日
feat(trust_ai): add document #930
已合并
共 5 个文件变更+757-7
| @@ -1,10 +1,251 @@ | |||
| 1 | # ai_asset_obfuscate | 1 | # ai_asset_obfuscate |
| 2 | +# 介绍 | ||
| 2 | 3 | ||
| 3 | -华为昇腾ai_asset_obfuscate混淆SDK, 提供模型、数据的预混淆处理能力 | 4 | +ai_asset_obfuscate 是一个Python工程项目,用于注册混淆因子并进行混淆模型权重的工具。 |
| 4 | 5 | ||
| 5 | -## 构建 whl 包 | 6 | +## 📄 安装依赖 |
| 6 | 7 | ||
| 7 | -### 方法一:使用构建脚本 | 8 | +请参阅下文了解快速安装和使用示例。 |
| 9 | + | ||
| 10 | +确保环境安装 [**Python>=3.8**](https://www.python.org/) | ||
| 11 | + | ||
| 12 | +``` | ||
| 13 | +pip install -r requirements.txt | ||
| 14 | +``` | ||
| 15 | + | ||
| 16 | +## 功能特性 | ||
| 17 | + | ||
| 18 | +ai_asset_obfuscate 提供以下核心功能: | ||
| 19 | + | ||
| 20 | +* **模型资产混淆**:对模型权重进行混淆保护,支持多种模型类型和自定义配置 | ||
| 21 | +* **数据资产混淆**:对推理数据进行混淆保护,支持一维和二维数据 | ||
| 22 | +* **图片数据混淆**:对图片数据进行混淆保护,支持Base64和字节数组格式 | ||
| 23 | +* **视频数据混淆**:对视频数据进行混淆保护,支持Base64和字节数组格式 | ||
| 24 | +* **口令管理**:提供口令加密功能,保护敏感信息 | ||
| 25 | +* **混淆因子管理**:支持混淆因子的创建、下发和本地保存 | ||
| 26 | + | ||
| 27 | +## 支持的模型列表 | ||
| 28 | + | ||
| 29 | +✅支持,测试验收 | ||
| 30 | +⭕️理论支持,测试无需验收 | ||
| 31 | +❌不支持 | ||
| 32 | + | ||
| 33 | +| 模型类型 | 模型名称 | 混淆态推理 | 推荐卡数 | 推理服务 | 推理性能测试 | 混淆态微调 | LoRA | 推荐卡数 | 微调后精度测试 | 备注 | | ||
| 34 | +| ------------- | -------------------- | --------------- | ------------------ | ---------- | -------------- | ------------ | -------- | ---------- | ---------------- | ------ | | ||
| 35 | +| DeepSeek | DeepSeek-V3 | ✅(A2,A3) | A2:16卡 A3:8卡 | MindIE | ✅(A2,A3) | ⭕️ | ⭕️ | - | - | | | ||
| 36 | +| DeepSeek | DeepSeek-R1 | ✅(A2,A3) | A2:16卡 A3:8卡 | MindIE | - | ⭕️ | ⭕️ | - | - | | | ||
| 37 | +| Qwen2.5 | Qwen2.5-0.5B | ⭕️ | - | MindIE | - | ⭕️ | ⭕️ | - | - | | | ||
| 38 | +| Qwen2.5 | Qwen2.5-1.5B | ✅(310,A2,A3) | A2:2卡 A3:1卡 | MindIE | - | ⭕️ | ⭕️ | - | - | | | ||
| 39 | +| Qwen2.5 | Qwen2.5-3B | ⭕️ | | MindIE | - | ⭕️ | ⭕️ | - | - | | | ||
| 40 | +| Qwen2.5 | Qwen2.5-7B | ✅(310,A2,A3) | A2:2卡 A3:1卡 | MindIE | - | ⭕️ | ⭕️ | - | - | | | ||
| 41 | +| Qwen2.5 | Qwen2.5-14B | ✅(310,A2,A3) | A2:8卡 A3:4卡 | MindIE | - | ⭕️ | ⭕️ | - | - | | | ||
| 42 | +| Qwen2.5 | Qwen2.5-32B | ✅(310,A2,A3) | A2:8卡 A3:4卡 | MindIE | ✅(A3) | ✅(A3) | ✅(A2) | A3:8卡 | ✅(A3) | | | ||
| 43 | +| Qwen2.5 | Qwen2.5-72B | ⭕️ | - | MindIE | - | ⭕️ | ⭕️ | - | - | | | ||
| 44 | +| Qwen3 | Qwen3-0.6B | ⭕️ | - | MindIE | - | ⭕️ | ⭕️ | - | - | | | ||
| 45 | +| Qwen3 | Qwen3-1.7B | ✅(A2) | A2:1卡 | MindIE | - | ✅(A2) | ✅(A2) | A2:8卡 | - | | | ||
| 46 | +| Qwen3 | Qwen3-4B | ⭕️ | - | MindIE | - | ⭕️ | ⭕️ | - | | - | | ||
| 47 | +| Qwen3 | Qwen3-8B | ✅(A2) | A2:1卡 | MindIE | - | ✅(A2) | ✅(A2) | A2:8卡 | - | | | ||
| 48 | +| Qwen3 | Qwen3-14B | ✅(310) | 310:8卡 | MindIE | ✅(A2) | ⭕️ | ⭕️ | - | - | | | ||
| 49 | +| Qwen3 | Qwen3-32B | ⭕️ | - | MindIE | - | ✅(A3) | ✅(A3) | A3:8卡 | ✅(A3) | | | ||
| 50 | +| Qwen3 | Qwen3-30B-A3B | ✅(A2) | A2:4卡 | vLLM | - | ✅(A2) | ✅(A2) | A2:16卡 | - | | | ||
| 51 | +| Qwen3 | Qwen3-235B-A22B | ✅(A3) | A3:8卡 | vLLM | - | ⭕️ | ⭕️ | - | - | | | ||
| 52 | +| Qwen3-VL | Qwen3-VL-2B | ⭕️ | - | vLLM | - | ⭕️ | ❌ | - | - | | | ||
| 53 | +| Qwen3-VL | Qwen3-VL-4B | ⭕️ | - | vLLM | - | ⭕️ | ❌ | - | - | | | ||
| 54 | +| Qwen3-VL | Qwen3-VL-8B | ✅(310) | 310:2卡 | vLLM | - | ✅(A2) | ❌ | A2:8卡 | - | | | ||
| 55 | +| Qwen3-VL | Qwen3-VL-32B | ✅(A3) | A3:4卡 | vLLM | ✅(A3) | ✅(A3) | ❌ | A3:8卡 | ✅(A3) | | | ||
| 56 | +| Qwen3-VL | Qwen3-VL-30B-A3B | ✅(A2) | A2:2卡 | vLLM | ✅(A2) | ✅(A3) | ❌ | A3:8卡 | - | | | ||
| 57 | +| Qwen3-VL | Qwen3-VL-235B-A22B | ⭕️ | - | vLLM | - | ⭕️ | ❌ | - | - | | | ||
| 58 | +| Qwen3-Omni | - | ❌ | - | vLLM | - | ❌ | ❌ | - | - | | | ||
| 59 | +| Qwen3-Audio | - | ❌ | - | vLLM | - | ❌ | ❌ | - | - | | | ||
| 60 | + | ||
| 61 | +## 快速开始 | ||
| 62 | + | ||
| 63 | +### 模型混淆示例 | ||
| 64 | + | ||
| 65 | +```python | ||
| 66 | +from ai_asset_obfuscate import ModelAssetObfuscation, ErrorCode, ModelType | ||
| 67 | + | ||
| 68 | +# 创建模型混淆实例 | ||
| 69 | +model = ModelAssetObfuscation.create_model_obfuscation( | ||
| 70 | + model_path="/path/to/model", | ||
| 71 | + model_type=ModelType.Qwen3, | ||
| 72 | + tp_num=4 | ||
| 73 | +) | ||
| 74 | + | ||
| 75 | +# 设置混淆因子 | ||
| 76 | +result = model.set_seed_content( | ||
| 77 | + seed_type=1, # 模型混淆因子 | ||
| 78 | + seed_content="your_seed_content_32_chars_min" | ||
| 79 | +) | ||
| 80 | +if result != ErrorCode.SUCCESS.value: | ||
| 81 | + print(f"设置混淆因子失败: {result}") | ||
| 82 | + | ||
| 83 | +# 执行模型混淆 | ||
| 84 | +result = model.model_weight_obf( | ||
| 85 | + obf_type=1, # 0:使用所有混淆因子, 1:仅使用模型混淆因子, 2:仅使用数据混淆因子 | ||
| 86 | + model_save_path="/path/to/model_save_path", | ||
| 87 | + device_type = "cpu", | ||
| 88 | + device_id=[0, 1, 2, 3] | ||
| 89 | +) | ||
| 90 | +if result == ErrorCode.SUCCESS.value: | ||
| 91 | + print("模型混淆成功") | ||
| 92 | +``` | ||
| 93 | + | ||
| 94 | +### 数据混淆示例 | ||
| 95 | + | ||
| 96 | +```python | ||
| 97 | +from ai_asset_obfuscate import DataAssetObfuscation, ErrorCode, ModelType | ||
| 98 | + | ||
| 99 | +# 创建数据混淆实例 | ||
| 100 | +data_obf = DataAssetObfuscation( | ||
| 101 | + vocab_size=32000, | ||
| 102 | + token_white_list=[0, 1] # 不做混淆的token白名单 | ||
| 103 | +) | ||
| 104 | + | ||
| 105 | +# 设置混淆因子 | ||
| 106 | +result = data_obf.set_seed_content( | ||
| 107 | + seed_content="your_seed_content_32_chars_min", | ||
| 108 | + is_local_save=False, | ||
| 109 | + seed_ciphertext_dir=None | ||
| 110 | +) | ||
| 111 | + | ||
| 112 | +# 混淆二维数据 | ||
| 113 | +tokens = [[1, 2, 3], [4, 5, 6]] | ||
| 114 | +obf_tokens = data_obf.data_2d_obf(tokens) | ||
| 115 | + | ||
| 116 | +# 解混淆二维数据 | ||
| 117 | +deobf_tokens = data_obf.data_2d_deobf(obf_tokens) | ||
| 118 | +``` | ||
| 119 | + | ||
| 120 | +### 图片混淆示例 | ||
| 121 | + | ||
| 122 | +```python | ||
| 123 | +from ai_asset_obfuscate.vision_api import ImageDataAssetObfuscation | ||
| 124 | + | ||
| 125 | +# 创建图片混淆实例 | ||
| 126 | +image_obf = ImageDataAssetObfuscation() | ||
| 127 | + | ||
| 128 | +# 设置混淆因子 | ||
| 129 | +image_obf.set_seed_content("your_seed_content_32_chars_min") | ||
| 130 | + | ||
| 131 | +# 混淆Base64格式图片 | ||
| 132 | +image_base64 = "iVBORw0KGgoAAAANSUhEUgAA..." | ||
| 133 | +obf_image = image_obf.image_base64_obf(image_base64) | ||
| 134 | +``` | ||
| 135 | + | ||
| 136 | +### 视频混淆示例 | ||
| 137 | + | ||
| 138 | +```python | ||
| 139 | +from ai_asset_obfuscate.vision_api import VideoDataAssetObfuscation | ||
| 140 | + | ||
| 141 | +# 创建视频混淆实例 | ||
| 142 | +image_obf = VideoDataAssetObfuscation() | ||
| 143 | + | ||
| 144 | +# 设置混淆因子 | ||
| 145 | +image_obf.set_seed_content("your_seed_content_32_chars_min") | ||
| 146 | + | ||
| 147 | +# 混淆Base64格式视频 | ||
| 148 | +video_base64 = "iVBORw0KGgoAAAANSUhEUgAA..." | ||
| 149 | +obf_video = image_obf.video_base64_obf(video_base64 ) | ||
| 150 | +``` | ||
| 151 | + | ||
| 152 | +## API接口文档 | ||
| 153 | + | ||
| 154 | +[**接口文档**](https://gitcode.com/Ascend/mindsdk-referenceapps/blob/master/TrustAiSDK/README_INTERFACE.md) | ||
| 155 | + | ||
| 156 | +## 混淆因子管理 | ||
| 157 | + | ||
| 158 | +### 混淆因子类型 | ||
| 159 | + | ||
| 160 | +* **模型混淆因子(类型1)**:用于模型权重混淆 | ||
| 161 | +* **数据混淆因子(类型2)**:用于推理数据混淆 | ||
| 162 | + | ||
| 163 | +### 混淆因子生命周期 | ||
| 164 | + | ||
| 165 | +1. **创建**:通过`distribute_obf_seed`或`local_save_obf_seed`创建混淆因子 | ||
| 166 | +2. **下发**:通过TLS/PSK安全通道下发到NPU设备 | ||
| 167 | +3. **使用**:在混淆/解混淆操作中使用混淆因子 | ||
| 168 | +4. **管理**:支持本地保存和远程下发两种方式 | ||
| 169 | + | ||
| 170 | +### 混淆因子设置方式 | ||
| 171 | + | ||
| 172 | +* **直接设置**:通过`set_seed_content`方法直接设置混淆因子明文 | ||
| 173 | +* **安全通道设置**:通过`set_seed_safer`方法通过TLS/PSK安全通道设置 | ||
| 174 | +* **本地加载**:通过`set_seed_content`方法的`is_local_save`参数从本地加载 | ||
| 175 | + | ||
| 176 | +## 安全特性 | ||
| 177 | + | ||
| 178 | +### TLS/PSK安全机制 | ||
| 179 | + | ||
| 180 | +obf_sdk 采用双重安全机制保护混淆因子: | ||
| 181 | + | ||
| 182 | +* **TLS(Transport Layer Security)**:提供安全的通信通道,确保混淆因子在传输过程中的安全性 | ||
| 183 | +* **PSK(Pre-Shared Key)**:使用预共享密钥进行额外的安全验证 | ||
| 184 | + | ||
| 185 | +### 口令加密要求 | ||
| 186 | + | ||
| 187 | +加密口令需满足以下要求: | ||
| 188 | + | ||
| 189 | +* 长度:40-64字符 | ||
| 190 | +* 必须包含以下至少2种字符类型: | ||
| 191 | + * 大写字母(A-Z) | ||
| 192 | + * 小写字母(a-z) | ||
| 193 | + * 数字(0-9) | ||
| 194 | + * 特殊字符(如!@#$%^&*等) | ||
| 195 | + | ||
| 196 | +### 安全建议 | ||
| 197 | + | ||
| 198 | +1. 定期更换混淆因子和口令 | ||
| 199 | +2. 使用强密码生成器创建混淆因子 | ||
| 200 | +3. 妥善保管TLS证书和PSK密钥 | ||
| 201 | +4. 限制混淆因子的访问权限 | ||
| 202 | +5. 定期审计混淆因子的使用记录 | ||
| 203 | + | ||
| 204 | +## 错误码 | ||
| 205 | + | ||
| 206 | +### 常见错误码 | ||
| 207 | + | ||
| 208 | +| 错误码 | 错误信息 | 说明 | | ||
| 209 | +| -------- | ----------------------------------------------------------------------------- | ----------------------------------- | | ||
| 210 | +| 0 | Success | 操作成功 | | ||
| 211 | +| 1 | Failed | 操作失败 | | ||
| 212 | +| 2 | Failed To Load Library | 加载库失败 | | ||
| 213 | +| 1001 | Parameter validation failed | 参数验证失败 | | ||
| 214 | +| 1002 | Path validation failed | 路径验证失败 | | ||
| 215 | +| 1003 | Obfuscate seed content validation failed | 混淆因子内容验证失败 | | ||
| 216 | +| 1004 | The config param tp number validation failed | TP数量验证失败 | | ||
| 217 | +| 1005 | The obfuscate config path validation failed | 配置路径验证失败 | | ||
| 218 | +| 1006 | The obfuscate config file is incorrect json format | 配置文件JSON格式错误 | | ||
| 219 | +| 1007 | The model type validation failed | 模型类型验证失败 | | ||
| 220 | +| 1015 | There is insufficient disk space | 磁盘空间不足 | | ||
| 221 | +| 1016 | Unknown I/O error | 未知I/O错误 | | ||
| 222 | +| 2001 | Encryption passwd failed | 口令加密失败 | | ||
| 223 | +| 2002 | Decryption passwd failed | 口令解密失败 | | ||
| 224 | +| 2003 | Create obfuscate seed failed | 创建混淆因子失败 | | ||
| 225 | +| 2004 | Query obfuscate seed failed | 查询混淆因子失败 | | ||
| 226 | +| 3001 | Generated random list failed | 生成随机列表失败 | | ||
| 227 | +| 3002 | The size of vocab must be greater than zero | 词汇表大小必须大于0 | | ||
| 228 | +| 3003 | Item type must be int and item must less than vocab_size | 元素类型必须为int且小于词汇表大小 | | ||
| 229 | +| 4001 | The op_type is not supported on the model shape | 操作类型不支持该模型形状 | | ||
| 230 | +| 5001 | The flag validation failed | 标志验证失败 | | ||
| 231 | +| 5002 | Model weight obfuscation for model protection has already been completed | 模型保护混淆已完成 | | ||
| 232 | +| 5003 | Model weight obfuscation for data protection has already been completed | 数据保护混淆已完成 | | ||
| 233 | +| 5004 | Model and data protection via weight obfuscation has already been completed | 模型和数据保护混淆已完成 | | ||
| 234 | +| 5005 | Failed to read configuration file | 读取配置文件失败 | | ||
| 235 | +| 5006 | Failed to update configuration file | 更新配置文件失败 | | ||
| 236 | +| 5007 | Model weights do not support current de-obfuscation operation | 模型权重不支持当前解混淆操作 | | ||
| 237 | +| 5008 | Failed to decode base64 string | Base64解码失败 | | ||
| 238 | +| 5009 | Failed to open image | 打开图片失败 | | ||
| 239 | +| 5010 | Failed to open video | 打开视频失败 | | ||
| 240 | +| 5011 | The seed content is not set | 混淆因子内容未设置 | | ||
| 241 | +| 5016 | Failed to create weight obfuscator | 创建权重混淆器失败 | | ||
| 242 | +| 5017 | Failed to apply weight obfuscation | 应用权重混淆失败 | | ||
| 243 | +| 5018 | Obfuscator is not initialized | 混淆器未初始化 | | ||
| 244 | + | ||
| 245 | + | ||
| 246 | +# 构建 whl 包 | ||
| 247 | + | ||
| 248 | +## 方法一:使用构建脚本 | ||
| 8 | 249 | ||
| 9 | 1. 进入 TrustAiSDK 目录: | 250 | 1. 进入 TrustAiSDK 目录: |
| 10 | ```bash | 251 | ```bash |
| @@ -22,7 +263,7 @@ | |||
| 22 | - 下载对应架构的 so 文件并复制到 ai_asset_obfuscate/libs 目录 | 263 | - 下载对应架构的 so 文件并复制到 ai_asset_obfuscate/libs 目录 |
| 23 | - 将生成的 whl 文件复制到 `output` 目录 | 264 | - 将生成的 whl 文件复制到 `output` 目录 |
| 24 | 265 | ||
| 25 | -### 指定架构构建 | 266 | +## 指定架构构建 |
| 26 | 267 | ||
| 27 | 如果需要为特定架构构建 whl 包,可以使用 `--arch` 参数: | 268 | 如果需要为特定架构构建 whl 包,可以使用 `--arch` 参数: |
| 28 | 269 | ||
| @@ -34,6 +275,6 @@ bash build.sh --arch=x86_64 | |||
| 34 | bash build.sh --arch=aarch64 | 275 | bash build.sh --arch=aarch64 |
| 35 | ``` | 276 | ``` |
| 36 | 277 | ||
| 37 | -### 构建结果 | 278 | +## 构建结果 |
| 38 | 279 | ||
| 39 | 构建完成后,生成的 whl 包会位于 `TrustAiSDK/output` 目录中,文件名为 `ai_asset_obfuscate-1.0.0-py3-none-linux_{arch}.whl`,其中 `{arch}` 为 `x86_64` 或 `aarch64`。 | 280 | 构建完成后,生成的 whl 包会位于 `TrustAiSDK/output` 目录中,文件名为 `ai_asset_obfuscate-1.0.0-py3-none-linux_{arch}.whl`,其中 `{arch}` 为 `x86_64` 或 `aarch64`。 |
| @@ -0,0 +1,500 @@ | |||
| 1 | +## API接口文档 | ||
| 2 | + | ||
| 3 | +### ModelAssetObfuscation 类 | ||
| 4 | + | ||
| 5 | +模型资产混淆核心类,提供模型权重混淆功能。 | ||
| 6 | + | ||
| 7 | +#### create_model_obfuscation | ||
| 8 | + | ||
| 9 | +创建标准模型混淆实例。 | ||
| 10 | + | ||
| 11 | +```python | ||
| 12 | +@staticmethod | ||
| 13 | +def create_model_obfuscation( | ||
| 14 | + model_path: str, | ||
| 15 | + model_type: ModelType, | ||
| 16 | + tp_num: int = None, | ||
| 17 | + token_white_list: list = None, | ||
| 18 | + obf_coefficient: float = None, | ||
| 19 | + is_obfuscation: bool = True | ||
| 20 | +) -> 'ModelAssetObfuscation' | ||
| 21 | +``` | ||
| 22 | + | ||
| 23 | +**参数:** | ||
| 24 | + | ||
| 25 | +* `model_path`: 模型文件路径 | ||
| 26 | +* `model_type`: 模型类型(ModelType枚举) | ||
| 27 | +* `tp_num`: 张量并行数量 | ||
| 28 | +* `token_white_list`: token白名单,该token不做混淆 | ||
| 29 | +* `obf_coefficient`: 混淆系数 | ||
| 30 | +* `is_obfuscation`: 是否混淆(True混淆,False不混淆) | ||
| 31 | + | ||
| 32 | +**返回值:** ModelAssetObfuscation实例 | ||
| 33 | + | ||
| 34 | +#### create_custom_model_obfuscation | ||
| 35 | + | ||
| 36 | +创建自定义模型混淆实例。 | ||
| 37 | + | ||
| 38 | +```python | ||
| 39 | +@staticmethod | ||
| 40 | +def create_custom_model_obfuscation( | ||
| 41 | + model_path: str, | ||
| 42 | + custom_obf_config_path: str, | ||
| 43 | + tp_num: int = None, | ||
| 44 | + token_white_list: list = None, | ||
| 45 | + obf_coefficient: float = None, | ||
| 46 | + is_obfuscation: bool = True | ||
| 47 | +) -> 'ModelAssetObfuscation' | ||
| 48 | +``` | ||
| 49 | + | ||
| 50 | +**参数:** | ||
| 51 | + | ||
| 52 | +* `model_path`: 模型文件路径 | ||
| 53 | +* `custom_obf_config_path`: 配置文件路径 | ||
| 54 | +* `tp_num`: 张量并行数量 | ||
| 55 | +* `token_white_list`: token白名单,该token不做混淆 | ||
| 56 | +* `obf_coefficient`: 混淆系数 | ||
| 57 | +* `is_obfuscation`: 是否混淆(True混淆,False不混淆) | ||
| 58 | + | ||
| 59 | +**返回值:** ModelAssetObfuscation实例 | ||
| 60 | + | ||
| 61 | +#### set_seed_content | ||
| 62 | + | ||
| 63 | +设置混淆因子内容。 | ||
| 64 | + | ||
| 65 | +```python | ||
| 66 | +def set_seed_content( | ||
| 67 | + seed_type: int = Constant.MODEL_SEED_TYPE, | ||
| 68 | + seed_content: str = None, | ||
| 69 | + is_local_save: bool = False, | ||
| 70 | + seed_ciphertext_dir: str = None | ||
| 71 | +) -> (int, str) | ||
| 72 | +``` | ||
| 73 | + | ||
| 74 | +**参数:** | ||
| 75 | + | ||
| 76 | +* `seed_type`: 混淆因子类型(1:模型混淆因子, 2:数据混淆因子) | ||
| 77 | +* `seed_content`: 混淆因子明文内容(32-112字符) | ||
| 78 | +* `is_local_save`: 是否从本地获取混淆因子 | ||
| 79 | +* `seed_ciphertext_dir`: 密文保存路径(is_local_save为True时需要) | ||
| 80 | + | ||
| 81 | +**返回值:** (错误码, 错误信息) | ||
| 82 | + | ||
| 83 | +#### model_weight_obf | ||
| 84 | + | ||
| 85 | +执行模型权重混淆。 | ||
| 86 | + | ||
| 87 | +```python | ||
| 88 | +def model_weight_obf( | ||
| 89 | + obf_type: int, | ||
| 90 | + precision_mode: int = None, | ||
| 91 | + model_save_path: str = None, | ||
| 92 | + device_type: str = 'cpu', | ||
| 93 | + device_id: List[int] = None | ||
| 94 | +) -> int | ||
| 95 | +``` | ||
| 96 | + | ||
| 97 | +**参数:** | ||
| 98 | + | ||
| 99 | +* `obf_type`: 混淆类型(0:使用所有混淆因子, 1:仅使用模型混淆因子, 2:仅使用数据混淆因子) | ||
| 100 | +* `precision_mode`: 精度选择(可选0,1) 0为浮点计算模式 1为量化计算模式 | ||
| 101 | +* `model_save_path`: 混淆后模型存储路径 | ||
| 102 | +* `device_type`: 使用cpu加速 | ||
| 103 | +* `device_id`:当device_type是cpu时,可不传; | ||
| 104 | + | ||
| 105 | +**返回值:** (错误码, 错误信息) | ||
| 106 | + | ||
| 107 | +### DataAssetObfuscation 类 | ||
| 108 | + | ||
| 109 | +数据资产混淆核心类,提供推理数据混淆和解混淆功能。 | ||
| 110 | + | ||
| 111 | +#### set_seed_content | ||
| 112 | + | ||
| 113 | +设置混淆因子内容。 | ||
| 114 | + | ||
| 115 | +```python | ||
| 116 | +def set_seed_content( | ||
| 117 | + seed_content: str = None, | ||
| 118 | + is_local_save: bool = False, | ||
| 119 | + seed_ciphertext_dir: str = None | ||
| 120 | +) -> (int, str) | ||
| 121 | +``` | ||
| 122 | + | ||
| 123 | +**参数:** | ||
| 124 | + | ||
| 125 | +* `seed_content`: 混淆因子明文内容(32-112字符) | ||
| 126 | +* `is_local_save`: 是否从本地获取混淆因子 | ||
| 127 | +* `seed_ciphertext_dir`: 密文保存路径(is_local_save为True时需要) | ||
| 128 | + | ||
| 129 | +**返回值:** (错误码, 错误信息) | ||
| 130 | + | ||
| 131 | +#### set_seed_safer | ||
| 132 | + | ||
| 133 | +通过TLS/PSK安全通道设置混淆因子。 | ||
| 134 | + | ||
| 135 | +```python | ||
| 136 | +def set_seed_safer( | ||
| 137 | + tls_info: tuple, | ||
| 138 | + psk_info: tuple | ||
| 139 | +) -> (int, str) | ||
| 140 | +``` | ||
| 141 | + | ||
| 142 | +**参数:** | ||
| 143 | + | ||
| 144 | +* `tls_info`: TLS配置元组 (ca_file, cert_file, pri_keyfile, port, ks_path, ciphertext_path) | ||
| 145 | +* `psk_info`: PSK配置元组 (psk_path, ks_path_psk, ciphertext_path_psk) | ||
| 146 | + | ||
| 147 | +**返回值:** (错误码, 错误信息) | ||
| 148 | + | ||
| 149 | +#### data_2d_obf | ||
| 150 | + | ||
| 151 | +混淆二维数据。 | ||
| 152 | + | ||
| 153 | +```python | ||
| 154 | +def data_2d_obf(tokens: List[List[int]]) -> List[List[int]] | ||
| 155 | +``` | ||
| 156 | + | ||
| 157 | +**参数:** | ||
| 158 | + | ||
| 159 | +* `tokens`: 待混淆的tokens(二维列表,内层元素为int) | ||
| 160 | + | ||
| 161 | +**返回值:** 混淆后的tokens | ||
| 162 | + | ||
| 163 | +#### data_1d_obf | ||
| 164 | + | ||
| 165 | +混淆一维数据。 | ||
| 166 | + | ||
| 167 | +```python | ||
| 168 | +def data_1d_obf(tokens: List[int]) -> List[int] | ||
| 169 | +``` | ||
| 170 | + | ||
| 171 | +**参数:** | ||
| 172 | + | ||
| 173 | +* `tokens`: 待混淆的tokens(一维列表,元素为int) | ||
| 174 | + | ||
| 175 | +**返回值:** 混淆后的tokens | ||
| 176 | + | ||
| 177 | +#### data_2d_deobf | ||
| 178 | + | ||
| 179 | +解混淆二维数据。 | ||
| 180 | + | ||
| 181 | +```python | ||
| 182 | +def data_2d_deobf(tokens: List[List[int]]) -> List[List[int]] | ||
| 183 | +``` | ||
| 184 | + | ||
| 185 | +**参数:** | ||
| 186 | + | ||
| 187 | +* `tokens`: 待解混淆的tokens(二维列表,内层元素为int) | ||
| 188 | + | ||
| 189 | +**返回值:** 解混淆后的tokens | ||
| 190 | + | ||
| 191 | +#### data_1d_deobf | ||
| 192 | + | ||
| 193 | +解混淆一维数据。 | ||
| 194 | + | ||
| 195 | +```python | ||
| 196 | +def data_1d_deobf(tokens: List[int]) -> List[int] | ||
| 197 | +``` | ||
| 198 | + | ||
| 199 | +**参数:** | ||
| 200 | + | ||
| 201 | +* `tokens`: 待解混淆的tokens(一维列表,元素为int) | ||
| 202 | + | ||
| 203 | +**返回值:** 解混淆后的tokens | ||
| 204 | + | ||
| 205 | +#### token_obf | ||
| 206 | + | ||
| 207 | +混淆单个token。 | ||
| 208 | + | ||
| 209 | +```python | ||
| 210 | +def token_obf(token: int) -> int | ||
| 211 | +``` | ||
| 212 | + | ||
| 213 | +**参数:** | ||
| 214 | + | ||
| 215 | +* `token`: 待混淆的token | ||
| 216 | + | ||
| 217 | +**返回值:** 混淆后的token | ||
| 218 | + | ||
| 219 | +#### token_deobf | ||
| 220 | + | ||
| 221 | +解混淆单个token。 | ||
| 222 | + | ||
| 223 | +```python | ||
| 224 | +def token_deobf(token: int) -> int | ||
| 225 | +``` | ||
| 226 | + | ||
| 227 | +**参数:** | ||
| 228 | + | ||
| 229 | +* `token`: 待解混淆的token | ||
| 230 | + | ||
| 231 | +**返回值:** 解混淆后的token | ||
| 232 | + | ||
| 233 | +### ImageDataAssetObfuscation 类 | ||
| 234 | + | ||
| 235 | +图片数据混淆类,提供图片数据的混淆功能。 | ||
| 236 | + | ||
| 237 | +#### create_by_config | ||
| 238 | + | ||
| 239 | +通过配置创建图片混淆实例。 | ||
| 240 | + | ||
| 241 | +```python | ||
| 242 | +@staticmethod | ||
| 243 | +def create_by_config(config_path: str) -> 'ImageDataAssetObfuscation' | ||
| 244 | +``` | ||
| 245 | + | ||
| 246 | +**参数:** | ||
| 247 | + | ||
| 248 | +* `config_path`: 配置文件路径 | ||
| 249 | + | ||
| 250 | +**返回值:** ImageDataAssetObfuscation实例 | ||
| 251 | + | ||
| 252 | +#### set_seed_content | ||
| 253 | + | ||
| 254 | +设置混淆因子内容。 | ||
| 255 | + | ||
| 256 | +```python | ||
| 257 | +def set_seed_content( | ||
| 258 | + seed_content: str = None, | ||
| 259 | + is_local_save: bool = False, | ||
| 260 | + seed_ciphertext_dir: str = None | ||
| 261 | +) -> (int, str) | ||
| 262 | +``` | ||
| 263 | + | ||
| 264 | +**参数:** | ||
| 265 | + | ||
| 266 | +* `seed_content`: 混淆因子明文内容(32-112字符) | ||
| 267 | +* `is_local_save`: 是否从本地获取混淆因子 | ||
| 268 | +* `seed_ciphertext_dir`: 密文保存路径(is_local_save为True时需要) | ||
| 269 | + | ||
| 270 | +**返回值:** (错误码, 错误信息) | ||
| 271 | + | ||
| 272 | +#### image_base64_obf | ||
| 273 | + | ||
| 274 | +混淆Base64格式图片。 | ||
| 275 | + | ||
| 276 | +```python | ||
| 277 | +def image_base64_obf(image_base64: str) -> str | ||
| 278 | +``` | ||
| 279 | + | ||
| 280 | +**参数:** | ||
| 281 | + | ||
| 282 | +* `image_base64`: Base64格式的图片数据 | ||
| 283 | + | ||
| 284 | +**返回值:** 混淆后的Base64图片数据 | ||
| 285 | + | ||
| 286 | +#### image_bytearray_obf | ||
| 287 | + | ||
| 288 | +混淆字节数组格式图片。 | ||
| 289 | + | ||
| 290 | +```python | ||
| 291 | +def image_bytearray_obf(image_bytearray: bytearray) -> bytearray | ||
| 292 | +``` | ||
| 293 | + | ||
| 294 | +**参数:** | ||
| 295 | + | ||
| 296 | +* `image_bytearray`: 字节数组格式的图片数据 | ||
| 297 | + | ||
| 298 | +**返回值:** 混淆后的字节数组图片数据 | ||
| 299 | + | ||
| 300 | +### VideoDataAssetObfuscation 类 | ||
| 301 | + | ||
| 302 | +视频数据混淆类,提供视频数据的混淆功能。 | ||
| 303 | + | ||
| 304 | +#### create_by_config | ||
| 305 | + | ||
| 306 | +通过配置创建视频混淆实例。 | ||
| 307 | + | ||
| 308 | +```python | ||
| 309 | +@staticmethod | ||
| 310 | +def create_by_config(config_path: str) -> 'VideoDataAssetObfuscation' | ||
| 311 | +``` | ||
| 312 | + | ||
| 313 | +**参数:** | ||
| 314 | + | ||
| 315 | +* `config_path`: 配置文件路径 | ||
| 316 | + | ||
| 317 | +**返回值:** VideoDataAssetObfuscation实例 | ||
| 318 | + | ||
| 319 | +#### set_seed_content | ||
| 320 | + | ||
| 321 | +设置混淆因子内容。 | ||
| 322 | + | ||
| 323 | +```python | ||
| 324 | +def set_seed_content( | ||
| 325 | + seed_content: str = None, | ||
| 326 | + is_local_save: bool = False, | ||
| 327 | + seed_ciphertext_dir: str = None | ||
| 328 | +) -> (int, str) | ||
| 329 | +``` | ||
| 330 | + | ||
| 331 | +**参数:** | ||
| 332 | + | ||
| 333 | +* `seed_content`: 混淆因子明文内容(32-112字符) | ||
| 334 | +* `is_local_save`: 是否从本地获取混淆因子 | ||
| 335 | +* `seed_ciphertext_dir`: 密文保存路径(is_local_save为True时需要) | ||
| 336 | + | ||
| 337 | +**返回值:** (错误码, 错误信息) | ||
| 338 | + | ||
| 339 | +#### video_base64_obf | ||
| 340 | + | ||
| 341 | +混淆Base64格式视频。 | ||
| 342 | + | ||
| 343 | +```python | ||
| 344 | +def video_base64_obf(video_base64: str) -> str | ||
| 345 | +``` | ||
| 346 | + | ||
| 347 | +**参数:** | ||
| 348 | + | ||
| 349 | +* `video_base64`: Base64格式的视频数据 | ||
| 350 | + | ||
| 351 | +**返回值:** 混淆后的Base64视频数据 | ||
| 352 | + | ||
| 353 | +#### video_bytearray_obf | ||
| 354 | + | ||
| 355 | +混淆字节数组格式视频。 | ||
| 356 | + | ||
| 357 | +```python | ||
| 358 | +def video_bytearray_obf(video_bytearray: bytearray) -> bytearray | ||
| 359 | +``` | ||
| 360 | + | ||
| 361 | +**参数:** | ||
| 362 | + | ||
| 363 | +* `video_bytearray`: 字节数组格式的视频数据 | ||
| 364 | + | ||
| 365 | +**返回值:** 混淆后的字节数组视频数据 | ||
| 366 | + | ||
| 367 | +### passwd_enc 函数 | ||
| 368 | + | ||
| 369 | +对口令进行加密保护。 | ||
| 370 | + | ||
| 371 | +```python | ||
| 372 | +def passwd_enc( | ||
| 373 | + ks_path: str, | ||
| 374 | + passwd: str, | ||
| 375 | + ciphertext_path: str | ||
| 376 | +) -> (int, str) | ||
| 377 | +``` | ||
| 378 | + | ||
| 379 | +**参数:** | ||
| 380 | + | ||
| 381 | +* `ks_path`: 加密工具路径 | ||
| 382 | +* `passwd`: 待加密口令(40-64字符,需包含大小写字母、数字、特殊字符中的至少2种) | ||
| 383 | +* `ciphertext_path`: 加密口令保存路径 | ||
| 384 | + | ||
| 385 | +**返回值:** (错误码, 错误信息) | ||
| 386 | + | ||
| 387 | +**示例:** | ||
| 388 | + | ||
| 389 | +```python | ||
| 390 | +from ai_asset_obfuscate import passwd_enc, ErrorCode | ||
| 391 | + | ||
| 392 | +result = passwd_enc( | ||
| 393 | + ks_path="/path/to/ks_tool", | ||
| 394 | + passwd="YourSecurePassword123!@", | ||
| 395 | + ciphertext_path="/path/to/ciphertext" | ||
| 396 | +) | ||
| 397 | +if result == ErrorCode.SUCCESS.value: | ||
| 398 | + print("口令加密成功") | ||
| 399 | +``` | ||
| 400 | + | ||
| 401 | +### distribute_obf_seed 函数 | ||
| 402 | + | ||
| 403 | +下发混淆因子到NPU设备。 | ||
| 404 | + | ||
| 405 | +```python | ||
| 406 | +def distribute_obf_seed( | ||
| 407 | + seed_type: int, | ||
| 408 | + tls_conf: TLSConfig, | ||
| 409 | + psk_conf: PskConfig, | ||
| 410 | + seed_content: str, | ||
| 411 | + device_id: List[int] = None | ||
| 412 | +) -> (int, str) | ||
| 413 | +``` | ||
| 414 | + | ||
| 415 | +**参数:** | ||
| 416 | + | ||
| 417 | +* `seed_type`: 混淆因子类型(1:模型混淆因子, 2:数据混淆因子) | ||
| 418 | +* `tls_conf`: TLS通信配置对象 | ||
| 419 | +* `psk_conf`: PSK私钥配置对象 | ||
| 420 | +* `seed_content`: 混淆因子明文(32-112字符) | ||
| 421 | +* `device_id`: 需要下发的设备ID列表(0-15),可选 | ||
| 422 | + | ||
| 423 | +**返回值:** (错误码, 错误信息) | ||
| 424 | + | ||
| 425 | +**示例:** | ||
| 426 | + | ||
| 427 | +```python | ||
| 428 | +from ai_asset_obfuscate import distribute_obf_seed, ErrorCode | ||
| 429 | +from ai_asset_obfuscate.model import TLSConfig, PskConfig | ||
| 430 | + | ||
| 431 | +# 创建TLS配置 | ||
| 432 | +tls_conf = TLSConfig( | ||
| 433 | + ca_file="/path/to/ca.pem", | ||
| 434 | + cert_file="/path/to/cert.pem", | ||
| 435 | + pri_keyfile="/path/to/key.pem", | ||
| 436 | + ks_path="/path/to/ks_tool", | ||
| 437 | + ciphertext_path="/path/to/ciphertext" | ||
| 438 | +) | ||
| 439 | +tls_conf.set_port(1024) | ||
| 440 | + | ||
| 441 | +# 创建PSK配置 | ||
| 442 | +psk_conf = PskConfig( | ||
| 443 | + psk_path="/path/to/psk", | ||
| 444 | + ks_path_psk="/path/to/ks_tool", | ||
| 445 | + ciphertext_path_psk="/path/to/psk_ciphertext" | ||
| 446 | +) | ||
| 447 | + | ||
| 448 | +# 下发混淆因子 | ||
| 449 | +result = distribute_obf_seed( | ||
| 450 | + seed_type=1, | ||
| 451 | + tls_conf=tls_conf, | ||
| 452 | + psk_conf=psk_conf, | ||
| 453 | + seed_content="your_seed_content_32_chars_min", | ||
| 454 | + device_id=[0, 1] | ||
| 455 | +) | ||
| 456 | +if result == ErrorCode.SUCCESS.value: | ||
| 457 | + print("混淆因子下发成功") | ||
| 458 | +``` | ||
| 459 | + | ||
| 460 | +### local_save_obf_seed 函数 | ||
| 461 | + | ||
| 462 | +本地保存混淆因子。 | ||
| 463 | + | ||
| 464 | +```python | ||
| 465 | +def local_save_obf_seed( | ||
| 466 | + seed_type: int, | ||
| 467 | + seed_ciphertext_dir: str, | ||
| 468 | + seed_content: str = None | ||
| 469 | +) -> (int, str) | ||
| 470 | +``` | ||
| 471 | + | ||
| 472 | +**参数:** | ||
| 473 | + | ||
| 474 | +* `seed_type`: 混淆因子类型(1:模型混淆因子, 2:数据混淆因子) | ||
| 475 | +* `seed_ciphertext_dir`: 密文保存目录路径 | ||
| 476 | +* `seed_content`: 混淆因子明文(32-112字符),如果为None则自动生成随机混淆因子 | ||
| 477 | + | ||
| 478 | +**返回值:** (错误码, 错误信息) | ||
| 479 | + | ||
| 480 | +**示例:** | ||
| 481 | + | ||
| 482 | +```python | ||
| 483 | +from ai_asset_obfuscate import local_save_obf_seed, ErrorCode | ||
| 484 | + | ||
| 485 | +# 本地保存混淆因子(自动生成) | ||
| 486 | +result = local_save_obf_seed( | ||
| 487 | + seed_type=1, | ||
| 488 | + seed_ciphertext_dir="/path/to/save_dir" | ||
| 489 | +) | ||
| 490 | + | ||
| 491 | +# 本地保存混淆因子(指定内容) | ||
| 492 | +result = local_save_obf_seed( | ||
| 493 | + seed_type=1, | ||
| 494 | + seed_ciphertext_dir="/path/to/save_dir", | ||
| 495 | + seed_content="your_seed_content_32_chars_min" | ||
| 496 | +) | ||
| 497 | +if result == ErrorCode.SUCCESS.value: | ||
| 498 | + print("混淆因子保存成功") | ||
| 499 | +``` | ||
| 500 | + | ||
| @@ -527,7 +527,7 @@ class ModelAssetObfuscation(AssetObfuscation): | |||
| 527 | # 找到所有需要加载的模型文件 | 527 | # 找到所有需要加载的模型文件 |
| 528 | all_model_name = self._get_models() | 528 | all_model_name = self._get_models() |
| 529 | from concurrent.futures import ThreadPoolExecutor, as_completed | 529 | from concurrent.futures import ThreadPoolExecutor, as_completed |
| 530 | - max_workers = min(len(all_model_name), os.cpu_count()) | 530 | + max_workers = min(len(all_model_name), min(os.cpu_count(), Constant.MAX_THREADS)) # 最大 32 线程并发 |
| 531 | futures = [] | 531 | futures = [] |
| 532 | with ThreadPoolExecutor(max_workers) as executor: | 532 | with ThreadPoolExecutor(max_workers) as executor: |
| 533 | for model_name in all_model_name: | 533 | for model_name in all_model_name: |
| @@ -47,4 +47,5 @@ class Constant: | |||
| 47 | MAX_BYTES_IMAGE_LENGTH = 20 | 47 | MAX_BYTES_IMAGE_LENGTH = 20 |
| 48 | MAX_BASE64_VIDEO_LENGTH = 681 | 48 | MAX_BASE64_VIDEO_LENGTH = 681 |
| 49 | MAX_BYTES_VIDEO_LENGTH = 512 | 49 | MAX_BYTES_VIDEO_LENGTH = 512 |
| 50 | - VISION_DATA_LEN = 256 | 50 | + VISION_DATA_LEN = 256 |
| 51 | + MAX_THREADS = 32 | ||
| @@ -0,0 +1,8 @@ | |||
| 1 | +torch>=2.0.0 | ||
| 2 | +torchvision>=0.15.0 | ||
| 3 | +numpy>=1.24.0 | ||
| 4 | +Pillow>=9.0.0 | ||
| 5 | +opencv-python>=4.8.0 | ||
| 6 | +av>=10.0.0 | ||
| 7 | +transformers>=4.30.0 | ||
| 8 | +safetensors>=0.3.0 | ||