Thanks to visit codestin.com
Credit goes to github.com

Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

fix: auto-reload model for remote engine #1971

Merged
merged 1 commit into from
Feb 14, 2025
Merged

fix: auto-reload model for remote engine #1971

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion engine/controllers/engines.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -379,19 +379,21 @@ void Engines::UpdateEngine(
metadata = (*exist_engine).metadata;
}

(void) engine_service_->UnloadEngine(engine);
(void)engine_service_->UnloadEngine(engine);

auto upd_res =
engine_service_->UpsertEngine(engine, type, api_key, url, version,
"all-platforms", status, metadata);
if (upd_res.has_error()) {
Json::Value res;
res["message"] = upd_res.error();
CTL_WRN("Error: " << upd_res.error());
auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
resp->setStatusCode(k400BadRequest);
callback(resp);
} else {
Json::Value res;
CTL_INF("Remote Engine update successfully!");
res["message"] = "Remote Engine update successfully!";
auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
resp->setStatusCode(k200OK);
Expand All @@ -400,6 +402,7 @@ void Engines::UpdateEngine(
} else {
Json::Value res;
res["message"] = "Request body is empty!";
CTL_WRN("Error: Request body is empty!");
auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
resp->setStatusCode(k400BadRequest);
callback(resp);
Expand Down
10 changes: 4 additions & 6 deletions engine/services/inference_service.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ cpp::result<void, InferResult> InferenceService::HandleChatCompletion(
LOG_WARN << "Engine is not loaded yet";
return cpp::fail(std::make_pair(stt, res));
}

if (!model_id.empty()) {
if (auto model_service = model_service_.lock()) {
auto metadata_ptr = model_service->GetCachedModelMetadata(model_id);
Expand Down Expand Up @@ -72,7 +72,6 @@ cpp::result<void, InferResult> InferenceService::HandleChatCompletion(
}
}


CTL_DBG("Json body inference: " + json_body->toStyledString());

auto cb = [q, tool_choice](Json::Value status, Json::Value res) {
Expand Down Expand Up @@ -217,10 +216,9 @@ InferResult InferenceService::LoadModel(
std::get<RemoteEngineI*>(engine_result.value())
->LoadModel(json_body, std::move(cb));
}
if (!engine_service_->IsRemoteEngine(engine_type)) {
auto model_id = json_body->get("model", "").asString();
saved_models_[model_id] = json_body;
}
// Save model config to reload if needed
auto model_id = json_body->get("model", "").asString();
saved_models_[model_id] = json_body;
return std::make_pair(stt, r);
}

Expand Down
Loading