Star29
14
代码
代码
Issues32
Pull Requests20
项目讨论
Wiki
分析
Star29
14
草稿
[WIP] feat/llamacpp-backend #146
hb创建于 19 天前
[WIP] feat/llamacpp-backend #146
草稿
hb创建于 19 天前
从bhu9530/smartservefeat/llamacpp-backend合入到openharmony-robot/smartservemain
共 9 个文件变更+176-16
Mplugin/include/session/adapter.h+6-0
@@ -42,6 +42,12 @@ public:
42 42 
43 virtual const std::string& Name() const = 0;43 virtual const std::string& Name() const = 0;
44 44 
45+ // 返回当前会话实际使用的推理后端；未知时返回空字符串，由服务层保留兼容回退行为。
46+ virtual std::string Backend() const
47+ {
48+ return {};
49+ }
50+ 
45 // 要求资源需求从大到小排列51 // 要求资源需求从大到小排列
46 virtual const std::vector<SessionResConfig>& InitialResConfigs() const = 0;52 virtual const std::vector<SessionResConfig>& InitialResConfigs() const = 0;
47 53 
Mplugin/include/task.h+15-0
@@ -269,6 +269,18 @@ public:
269 return tokenUsage_;269 return tokenUsage_;
270 }270 }
271 271 
272+ void SetActualBackend(std::string backend)
273+ {
274+ std::lock_guard<std::mutex> lock(mutex_);
275+ actualBackend_ = std::move(backend);
276+ }
277+ 
278+ std::string GetActualBackend()
279+ {
280+ std::lock_guard<std::mutex> lock(mutex_);
281+ return actualBackend_;
282+ }
283+ 
272 pid_t pid_;284 pid_t pid_;
273 std::string modelName_;285 std::string modelName_;
274 /** OpenAI chat_completions request JSON (model, messages, stream, sampling params, …). */286 /** OpenAI chat_completions request JSON (model, messages, stream, sampling params, …). */
@@ -289,6 +301,9 @@ public:
289 nlohmann::json tool_call_messages_;301 nlohmann::json tool_call_messages_;
290 302 
291private:303private:
304+ // 由实际执行任务的 SessionAdapter 填写，与请求中的 preferredBackend 分离。
305+ std::string actualBackend_;
306+ 
292 std::unique_ptr<EvalLLMToolCallState, EvalLLMToolCallStateDeleter> toolCallState_;307 std::unique_ptr<EvalLLMToolCallState, EvalLLMToolCallStateDeleter> toolCallState_;
293 308 
294 StreamCallback streamCb_;309 StreamCallback streamCb_;
Mplugin/llamacpp/plugin.cpp+70-0
@@ -15,14 +15,17 @@
15 15 
16#include <api.h>16#include <api.h>
17 17 
18+#include <cctype>
18#include <cstdio>19#include <cstdio>
19#include <cstdlib>20#include <cstdlib>
20#include <cstring>21#include <cstring>
21 22 
23+#include "ggml-backend.h"
22#include "llama.h"24#include "llama.h"
23#include "smart_serve_log.h"25#include "smart_serve_log.h"
24 26 
25#include "./engine.h"27#include "./engine.h"
28+#include "./plugin.h"
26 29 
27namespace OHOS {30namespace OHOS {
28namespace llamacpp {31namespace llamacpp {
@@ -33,6 +36,27 @@ static bool g_isInited = false;
33 36 
34namespace {37namespace {
35 38 
39+std::string NormalizeBackendName(ggml_backend_dev_t device)
40+{
41+ if (device == nullptr) {
42+ return {};
43+ }
44+ const auto reg = ggml_backend_dev_backend_reg(device);
45+ const char* name = reg == nullptr ? nullptr : ggml_backend_reg_name(reg);
46+ if (name == nullptr || name[0] == '\0') {
47+ name = ggml_backend_dev_name(device);
48+ }
49+ if (name == nullptr) {
50+ return {};
51+ }
52+ 
53+ std::string normalized(name);
54+ for (char& c : normalized) {
55+ c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
56+ }
57+ return normalized;
58+}
59+ 
36bool IsLlamaVerboseEnabled()60bool IsLlamaVerboseEnabled()
37{61{
38 const char* env = std::getenv("SMART_SERVE_LLAMA_VERBOSE");62 const char* env = std::getenv("SMART_SERVE_LLAMA_VERBOSE");
@@ -53,6 +77,52 @@ void LlamaLogCallback(ggml_log_level level, const char* text, void* user_data)
53 77 
54} // namespace78} // namespace
55 79 
80+std::string DetectLlamaCppBackend()
81+{
82+ if (!llama_supports_gpu_offload()) {
83+ return "cpu";
84+ }
85+ 
86+ std::string rpcBackend;
87+ std::string gpuBackend;
88+ std::string integratedGpuBackend;
89+ for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
90+ const auto device = ggml_backend_dev_get(i);
91+ if (device == nullptr) {
92+ continue;
93+ }
94+ 
95+ const auto type = ggml_backend_dev_type(device);
96+ if (type != GGML_BACKEND_DEVICE_TYPE_GPU && type != GGML_BACKEND_DEVICE_TYPE_IGPU) {
97+ continue;
98+ }
99+ 
100+ const std::string backend = NormalizeBackendName(device);
101+ if (backend.empty()) {
102+ continue;
103+ }
104+ if (backend == "rpc") {
105+ if (rpcBackend.empty()) {
106+ rpcBackend = backend;
107+ }
108+ } else if (type == GGML_BACKEND_DEVICE_TYPE_GPU) {
109+ if (gpuBackend.empty()) {
110+ gpuBackend = backend;
111+ }
112+ } else if (integratedGpuBackend.empty()) {
113+ integratedGpuBackend = backend;
114+ }
115+ }
116+ 
117+ if (!rpcBackend.empty()) {
118+ return rpcBackend;
119+ }
120+ if (!gpuBackend.empty()) {
121+ return gpuBackend;
122+ }
123+ return integratedGpuBackend.empty() ? "cpu" : integratedGpuBackend;
124+}
125+ 
56bool Init()126bool Init()
57{127{
58 if (g_isInited) {128 if (g_isInited) {
Mplugin/llamacpp/plugin.h+5-0
@@ -16,9 +16,14 @@
16#ifndef SMART_SERVE_PLUGIN_LLAMACPP_PLUGIN_H16#ifndef SMART_SERVE_PLUGIN_LLAMACPP_PLUGIN_H
17#define SMART_SERVE_PLUGIN_LLAMACPP_PLUGIN_H17#define SMART_SERVE_PLUGIN_LLAMACPP_PLUGIN_H
18 18 
19+#include <string>
20+ 
19namespace OHOS {21namespace OHOS {
20namespace llamacpp {22namespace llamacpp {
21 23 
24+// 按 llama.cpp 默认设备选择规则返回当前主推理 backend。
25+std::string DetectLlamaCppBackend();
26+ 
22bool Init();27bool Init();
23 28 
24void Finalize();29void Finalize();
Mplugin/llamacpp/qwen/session.h+6-0
@@ -29,6 +29,7 @@
29#include "model/adapter.h"29#include "model/adapter.h"
30#include "model/model_config.h"30#include "model/model_config.h"
31#include "session/adapter.h"31#include "session/adapter.h"
32+#include "../plugin.h"
32#include "./model.h"33#include "./model.h"
33 34 
34namespace OHOS {35namespace OHOS {
@@ -58,6 +59,11 @@ public:
58 return name_;59 return name_;
59 }60 }
60 61 
62+ std::string Backend() const override
63+ {
64+ return DetectLlamaCppBackend();
65+ }
66+ 
61 const std::vector<SmartServe::SessionResConfig>& InitialResConfigs() const override67 const std::vector<SmartServe::SessionResConfig>& InitialResConfigs() const override
62 {68 {
63 return resConfigs_;69 return resConfigs_;
Mplugin/llamacpp/qwen_vl/session.h+6-0
@@ -28,6 +28,7 @@
28#include "model/adapter.h"28#include "model/adapter.h"
29#include "model/model_config.h"29#include "model/model_config.h"
30#include "session/adapter.h"30#include "session/adapter.h"
31+#include "../plugin.h"
31#include "./model.h"32#include "./model.h"
32#include <mtmd.h>33#include <mtmd.h>
33#include <mtmd-helper.h>34#include <mtmd-helper.h>
@@ -60,6 +61,11 @@ public:
60 return name_;61 return name_;
61 }62 }
62 63 
64+ std::string Backend() const override
65+ {
66+ return DetectLlamaCppBackend();
67+ }
68+ 
63 const std::vector<SmartServe::SessionResConfig>& InitialResConfigs() const override69 const std::vector<SmartServe::SessionResConfig>& InitialResConfigs() const override
64 {70 {
65 return resConfigs_;71 return resConfigs_;
Mplugin/register_engine.md+3-1
@@ -46,7 +46,7 @@ SmartServe 当前支持两种接入方式：
46| `Service::LoadModels()` | 解析 `models.json`，校验模型路径，把模型配置注册给模型管理器 |46| `Service::LoadModels()` | 解析 `models.json`，校验模型路径，把模型配置注册给模型管理器 |
47| `EngineCreator` | 根据 `ModelConfig` 创建 `ModelAdapter` 和 `SessionAdapter` |47| `EngineCreator` | 根据 `ModelConfig` 创建 `ModelAdapter` 和 `SessionAdapter` |
48| `ModelAdapter` | 加载模型权重或系统模型资源 |48| `ModelAdapter` | 加载模型权重或系统模型资源 |
49-| `SessionAdapter` | 执行推理请求，写入完整或流式结果 |49+| `SessionAdapter` | 执行推理请求，写入完整或流式结果，并通过 `Backend()` 报告实际推理后端 |
50 50 
51参考代码：51参考代码：
52 52 
@@ -380,6 +380,8 @@ extern "C" GewuSmartServeEngineError MyEngineGetInterface(
380 380 
381## 编写 `SessionAdapter::Eval()`381## 编写 `SessionAdapter::Eval()`
382 382 
383+`SessionAdapter::Backend()` 用于填写 Chat Completions 响应中的 `backend` 字段。插件能够确定实际后端时应重写该方法，并返回规范化的小写名称（例如 `cpu`、`metal`、`cuda` 或 `vulkan`）；无法确定时可保留默认空字符串，服务层会兼容回退到请求偏好或 `cpu`。
384+ 
383`EvalLLMTask::inputs_` 是 OpenAI Chat Completions 请求 JSON，包含 `messages`、采样参数、`stream` 等字段。引擎需要根据自身 runtime 解析这些字段。385`EvalLLMTask::inputs_` 是 OpenAI Chat Completions 请求 JSON，包含 `messages`、采样参数、`stream` 等字段。引擎需要根据自身 runtime 解析这些字段。
384 386 
385可参考：387可参考：
Mservices/src/server/service/client_api.cpp+28-15
@@ -124,6 +124,15 @@ bool IsTerminalTaskState(EvalTask::TaskState state)
124 state == EvalTask::TaskState::CANCELLED;124 state == EvalTask::TaskState::CANCELLED;
125}125}
126 126 
127+std::string ResponseBackend(EvalLLMTask& task)
128+{
129+ const std::string actualBackend = task.GetActualBackend();
130+ if (!actualBackend.empty()) {
131+ return actualBackend;
132+ }
133+ return task.preferredBackend.empty() ? "cpu" : task.preferredBackend;
134+}
135+ 
127// Chat Completions 请求中与任务执行相关的可选参数集合。136// Chat Completions 请求中与任务执行相关的可选参数集合。
128struct ChatCompletionsRequestOptions {137struct ChatCompletionsRequestOptions {
129 SamplingParams samplingParams;138 SamplingParams samplingParams;
@@ -741,8 +750,7 @@ std::string ClientAPIHandler::HandleChatCompletionsRequest(const Request& reques
741 }750 }
742 751 
743 const std::string engineName = task->engineName;752 const std::string engineName = task->engineName;
744- const std::string backendName =753+ const std::string backendName = ResponseBackend(*task);
745- task->preferredBackend.empty() ? "cpu" : task->preferredBackend;
746 754 
747 ChatCompletionsResponse response;755 ChatCompletionsResponse response;
748 response.id = "chatcmpl-" + std::to_string(tid);756 response.id = "chatcmpl-" + std::to_string(tid);
@@ -827,7 +835,7 @@ std::string ClientAPIHandler::HandleGetChatCompletionRequest(const Request& requ
827 response.id = "chatcmpl-" + std::to_string(tid);835 response.id = "chatcmpl-" + std::to_string(tid);
828 response.model = task->modelName_;836 response.model = task->modelName_;
829 response.engine = task->engineName;837 response.engine = task->engineName;
830- response.backend = task->preferredBackend.empty() ? "cpu" : task->preferredBackend;838+ response.backend = ResponseBackend(*task);
831 response.created = std::chrono::duration_cast<std::chrono::seconds>(839 response.created = std::chrono::duration_cast<std::chrono::seconds>(
832 std::chrono::system_clock::now().time_since_epoch()840 std::chrono::system_clock::now().time_since_epoch()
833 ).count();841 ).count();
@@ -856,6 +864,10 @@ bool ClientAPIHandler::EvalTask(std::shared_ptr<EvalLLMTask> task)
856 task->modelName_.c_str(), task->engineName.c_str(), task->preferredBackend.c_str());864 task->modelName_.c_str(), task->engineName.c_str(), task->preferredBackend.c_str());
857 return false;865 return false;
858 }866 }
867+ const std::string actualBackend = session->Adapter()->Backend();
868+ if (!actualBackend.empty()) {
869+ task->SetActualBackend(actualBackend);
870+ }
859 session->AddTask(std::move(task));871 session->AddTask(std::move(task));
860 872 
861 service->scheduler_->Schedule(std::make_unique<ScheduleEvent>());873 service->scheduler_->Schedule(std::make_unique<ScheduleEvent>());
@@ -939,16 +951,24 @@ std::string ClientAPIHandler::HandleStreamChatCompletionsRequest(const Request&
939 task->InitReasoningSplitter(951 task->InitReasoningSplitter(
940 ShouldExtractReasoning(prepared.reasoningMode), startsInThinking);952 ShouldExtractReasoning(prepared.reasoningMode), startsInThinking);
941 953 
942- const std::string engineName = task->engineName;954+ auto service = Service::Instance();
943- const std::string backendName =955+ auto session = service->sessionManager_.GetOrCreateSession(task.get());
944- task->preferredBackend.empty() ? "cpu" : task->preferredBackend;956+ if (!session) {
957+ CleanupTask(tid);
958+ return MakeErrorResponse(ErrorType::NOT_FOUND_ERROR);
959+ }
960+ const std::string actualBackend = session->Adapter()->Backend();
961+ if (!actualBackend.empty()) {
962+ task->SetActualBackend(actualBackend);
963+ }
964+ 
945 StreamChunkMeta meta{965 StreamChunkMeta meta{
946 .taskId = "chatcmpl-" + std::to_string(tid),966 .taskId = "chatcmpl-" + std::to_string(tid),
947 .created = std::chrono::duration_cast<std::chrono::seconds>(967 .created = std::chrono::duration_cast<std::chrono::seconds>(
948 std::chrono::system_clock::now().time_since_epoch()).count(),968 std::chrono::system_clock::now().time_since_epoch()).count(),
949 .model = prepared.model,969 .model = prepared.model,
950- .engine = engineName,970+ .engine = task->engineName,
951- .backend = backendName,971+ .backend = ResponseBackend(*task),
952 };972 };
953 const bool includeUsage = task->includeUsage;973 const bool includeUsage = task->includeUsage;
954 974 
@@ -963,13 +983,6 @@ std::string ClientAPIHandler::HandleStreamChatCompletionsRequest(const Request&
963 chunkCallback(MakeStreamChunkJson(meta, deltaJson, std::nullopt, includeUsage));983 chunkCallback(MakeStreamChunkJson(meta, deltaJson, std::nullopt, includeUsage));
964 });984 });
965 985 
966- auto service = Service::Instance();
967- auto session = service->sessionManager_.GetOrCreateSession(task.get());
968- if (!session) {
969- task->SetStreamCallback(nullptr);
970- CleanupTask(tid);
971- return MakeErrorResponse(ErrorType::NOT_FOUND_ERROR);
972- }
973 session->AddTask(task);986 session->AddTask(task);
974 service->scheduler_->Schedule(std::make_unique<ScheduleEvent>());987 service->scheduler_->Schedule(std::make_unique<ScheduleEvent>());
975 988 
Mtest/interface/client_api_test.cpp+37-0
@@ -76,6 +76,7 @@ struct CapturedEval {
76 EvalTask::Type type = EvalTask::Type::LLM;76 EvalTask::Type type = EvalTask::Type::LLM;
77 std::string inputs;77 std::string inputs;
78 SamplingParams samplingParams;78 SamplingParams samplingParams;
79+ std::string backend;
79};80};
80 81 
81class CapturingModelAdapter final : public ModelAdapter {82class CapturingModelAdapter final : public ModelAdapter {
@@ -122,6 +123,11 @@ public:
122 return name_;123 return name_;
123 }124 }
124 125 
126+ std::string Backend() const override
127+ {
128+ return capture_->backend;
129+ }
130+ 
125 const std::vector<SessionResConfig>& InitialResConfigs() const override131 const std::vector<SessionResConfig>& InitialResConfigs() const override
126 {132 {
127 return resConfigs_;133 return resConfigs_;
@@ -364,6 +370,37 @@ TEST_F(ClientApiServiceTest, ListEnginesReturnsArray)
364 EXPECT_TRUE(response["engines"].is_array());370 EXPECT_TRUE(response["engines"].is_array());
365}371}
366 372 
373+// 验证响应优先返回 SessionAdapter 报告的实际 backend，而不是请求偏好或 CPU 默认值。
374+TEST_F(ClientApiServiceTest, ChatCompletionsReturnsActualSessionBackend)
375+{
376+ auto* service = Service::Instance();
377+ ASSERT_NE(service, nullptr);
378+ auto capture = std::make_shared<CapturedEval>();
379+ capture->backend = "metal";
380+ {
381+ std::unique_lock<std::mutex> lock(service->msMgmtMutex_);
382+ ASSERT_TRUE(service->engineManager_.Register(
383+ std::make_unique<CapturingEngineCreator>("capture-backend", capture)));
384+ 
385+ ModelConfig config;
386+ config.id = "capture-backend-model";
387+ config.name = config.id;
388+ config.engine = "capture-backend";
389+ ASSERT_TRUE(service->modelManager_.RegisterModel(config));
390+ }
391+ 
392+ ClientAPIHandler handler;
393+ json body = {
394+ {"model", "capture-backend-model"},
395+ {"messages", json::array({{{"role", "user"}, {"content", "hello"}}})},
396+ {"extra_body", {{"preferred_backend", "cpu"}}},
397+ };
398+ 
399+ const auto response = json::parse(handler.HandleRequest(WrapRequest(Request::CHAT_COMPLETIONS, body)));
400+ 
401+ EXPECT_EQ(response["backend"], "metal");
402+}
403+ 
367TEST_F(ClientApiServiceTest, UnloadMissingModelReturnsNotFound)404TEST_F(ClientApiServiceTest, UnloadMissingModelReturnsNotFound)
368{405{
369 ClientAPIHandler handler;406 ClientAPIHandler handler;