#include "transfer_task.h"
#include <glog/logging.h>
#include <algorithm>
#include <cstdlib>
namespace mooncake {
constexpr int kDefaultFilereadWorkers = 10;
FilereadWorkerPool::FilereadWorkerPool(std::shared_ptr<StorageBackend>& backend)
: shutdown_(false) {
VLOG(1) << "Creating FilereadWorkerPool with " << kDefaultFilereadWorkers
<< " workers";
workers_.reserve(kDefaultFilereadWorkers);
for (int i = 0; i < kDefaultFilereadWorkers; ++i) {
workers_.emplace_back(&FilereadWorkerPool::workerThread, this);
}
backend_ = backend;
}
FilereadWorkerPool::~FilereadWorkerPool() {
{
std::lock_guard<std::mutex> lock(queue_mutex_);
shutdown_.store(true);
}
queue_cv_.notify_all();
for (auto& worker : workers_) {
if (worker.joinable()) {
worker.join();
}
}
VLOG(1) << "FilereadWorkerPool destroyed";
}
void FilereadWorkerPool::submitTask(FilereadTask task) {
{
std::lock_guard<std::mutex> lock(queue_mutex_);
if (shutdown_.load()) {
LOG(WARNING)
<< "Attempting to submit task to shutdown FilereadWorkerPool";
task.state->set_completed(ErrorCode::TRANSFER_FAIL);
return;
}
task_queue_.push(std::move(task));
}
queue_cv_.notify_one();
}
void FilereadWorkerPool::workerThread() {
VLOG(2) << "FilereadWorkerPool worker thread started";
while (true) {
FilereadTask task("", 0, {}, nullptr);
{
std::unique_lock<std::mutex> lock(queue_mutex_);
queue_cv_.wait(lock, [this] {
return shutdown_.load() || !task_queue_.empty();
});
if (shutdown_.load() && task_queue_.empty()) {
break;
}
if (!task_queue_.empty()) {
task = std::move(task_queue_.front());
task_queue_.pop();
}
}
if (task.state) {
try {
if (!backend_) {
LOG(ERROR)
<< "Backend is not initialized, cannot load object";
task.state->set_completed(ErrorCode::TRANSFER_FAIL);
continue;
}
auto load_result = backend_->LoadObject(
task.file_path, task.slices, task.object_size);
if (load_result) {
VLOG(2) << "Fileread task completed successfully with "
<< task.file_path;
task.state->set_completed(ErrorCode::OK);
} else {
LOG(ERROR)
<< "Fileread task failed for file: " << task.file_path
<< " with error: " << toString(load_result.error());
task.state->set_completed(ErrorCode::TRANSFER_FAIL);
}
} catch (const std::exception& e) {
LOG(ERROR) << "Exception during async fileread: " << e.what();
task.state->set_completed(ErrorCode::TRANSFER_FAIL);
}
}
}
VLOG(2) << "FilereadWorkerPool worker thread exiting";
}
constexpr int kDefaultMemcpyWorkers = 1;
MemcpyWorkerPool::MemcpyWorkerPool() : shutdown_(false) {
VLOG(1) << "Creating MemcpyWorkerPool with " << kDefaultMemcpyWorkers
<< " workers";
workers_.reserve(kDefaultMemcpyWorkers);
for (int i = 0; i < kDefaultMemcpyWorkers; ++i) {
workers_.emplace_back(&MemcpyWorkerPool::workerThread, this);
}
}
MemcpyWorkerPool::~MemcpyWorkerPool() {
{
std::lock_guard<std::mutex> lock(queue_mutex_);
shutdown_.store(true);
}
queue_cv_.notify_all();
for (auto& worker : workers_) {
if (worker.joinable()) {
worker.join();
}
}
VLOG(1) << "MemcpyWorkerPool destroyed";
}
void MemcpyWorkerPool::submitTask(MemcpyTask task) {
{
std::lock_guard<std::mutex> lock(queue_mutex_);
if (shutdown_.load()) {
LOG(WARNING)
<< "Attempting to submit task to shutdown MemcpyWorkerPool";
task.state->set_completed(ErrorCode::TRANSFER_FAIL);
return;
}
task_queue_.push(std::move(task));
}
queue_cv_.notify_one();
}
void MemcpyWorkerPool::workerThread() {
VLOG(2) << "MemcpyWorkerPool worker thread started";
while (true) {
MemcpyTask task({}, nullptr);
{
std::unique_lock<std::mutex> lock(queue_mutex_);
queue_cv_.wait(lock, [this] {
return shutdown_.load() || !task_queue_.empty();
});
if (shutdown_.load() && task_queue_.empty()) {
break;
}
if (!task_queue_.empty()) {
task = std::move(task_queue_.front());
task_queue_.pop();
}
}
if (task.state) {
try {
for (const auto& op : task.operations) {
std::memcpy(op.dest, op.src, op.size);
}
VLOG(2) << "Memcpy task completed successfully with "
<< task.operations.size() << " operations";
task.state->set_completed(ErrorCode::OK);
} catch (const std::exception& e) {
LOG(ERROR) << "Exception during async memcpy: " << e.what();
task.state->set_completed(ErrorCode::TRANSFER_FAIL);
}
}
}
VLOG(2) << "MemcpyWorkerPool worker thread exiting";
}
bool TransferEngineOperationState::is_completed() {
std::lock_guard<std::mutex> lock(mutex_);
if (result_.has_value()) {
return true;
}
check_task_status();
return result_.has_value();
}
void TransferEngineOperationState::check_task_status() {
bool all_completed = true;
bool has_failure = false;
for (size_t i = 0; i < batch_size_; ++i) {
TransferStatus status;
Status s = engine_.getTransferStatus(batch_id_, i, status);
if (!s.ok()) {
LOG(ERROR) << "Failed to get transfer status for batch "
<< batch_id_ << " task " << i << " with error "
<< s.message();
set_result_internal(ErrorCode::TRANSFER_FAIL);
return;
}
switch (status.s) {
case TransferStatusEnum::COMPLETED:
break;
case TransferStatusEnum::FAILED:
case TransferStatusEnum::CANCELED:
case TransferStatusEnum::INVALID:
LOG(ERROR) << "Transfer failed for batch " << batch_id_
<< " task " << i << " with status "
<< static_cast<int>(status.s);
has_failure = true;
break;
default:
all_completed = false;
break;
}
}
if (has_failure) {
VLOG(1) << "Setting batch " << batch_id_
<< " result to TRANSFER_FAIL due to task failures";
set_result_internal(ErrorCode::TRANSFER_FAIL);
return;
}
if (all_completed) {
set_result_internal(ErrorCode::OK);
return;
}
return;
}
void TransferEngineOperationState::set_result_internal(ErrorCode error_code) {
if (result_.has_value()) {
LOG(ERROR) << "Attempting to set result multiple times for batch "
<< batch_id_
<< ". Previous result: " << static_cast<int>(result_.value())
<< ", attempted new result: " << static_cast<int>(error_code)
<< ". This indicates a race condition or logic error.";
return;
}
VLOG(1) << "Setting transfer result for batch " << batch_id_ << " to "
<< static_cast<int>(error_code);
result_.emplace(error_code);
cv_.notify_all();
}
void TransferEngineOperationState::wait_for_completion() {
if (is_completed()) {
return;
}
VLOG(1) << "Starting transfer engine polling for batch " << batch_id_;
constexpr int64_t timeout_seconds = 60;
constexpr int64_t kOneSecondInNano = 1000 * 1000 * 1000;
const int64_t start_ts = getCurrentTimeInNano();
while (true) {
if (getCurrentTimeInNano() - start_ts >
timeout_seconds * kOneSecondInNano) {
LOG(ERROR) << "Failed to complete transfers after "
<< timeout_seconds << " seconds for batch " << batch_id_;
set_result_internal(ErrorCode::TRANSFER_FAIL);
return;
}
std::unique_lock<std::mutex> lock(mutex_);
check_task_status();
if (result_.has_value()) {
VLOG(1) << "Transfer engine operation completed for batch "
<< batch_id_
<< " with result: " << static_cast<int>(result_.value());
break;
}
VLOG(1) << "Transfer engine operation still pending for batch "
<< batch_id_;
}
}
TransferFuture::TransferFuture(std::shared_ptr<OperationState> state)
: state_(std::move(state)) {
if (!state_) {
LOG(ERROR) << "TransferFuture requires valid state";
throw std::invalid_argument("TransferFuture requires valid state");
}
}
bool TransferFuture::isReady() const { return state_->is_completed(); }
ErrorCode TransferFuture::wait() {
if (!isReady()) {
state_->wait_for_completion();
}
return state_->get_result();
}
ErrorCode TransferFuture::get() { return wait(); }
TransferStrategy TransferFuture::strategy() const {
return state_->get_strategy();
}
TransferSubmitter::TransferSubmitter(TransferEngine& engine,
std::shared_ptr<StorageBackend>& backend,
TransferMetric* transfer_metric)
: engine_(engine),
memcpy_pool_(std::make_unique<MemcpyWorkerPool>()),
fileread_pool_(std::make_unique<FilereadWorkerPool>(backend)),
transfer_metric_(transfer_metric) {
const char* env_value = std::getenv("MC_STORE_MEMCPY");
if (env_value == nullptr) {
memcpy_enabled_ = false;
} else {
std::string env_str(env_value);
std::transform(env_str.begin(), env_str.end(), env_str.begin(),
::tolower);
if (env_str == "false" || env_str == "0" || env_str == "no" ||
env_str == "off") {
memcpy_enabled_ = false;
} else if (env_str == "true" || env_str == "1" || env_str == "yes" ||
env_str == "on") {
memcpy_enabled_ = true;
} else {
LOG(WARNING) << "Invalid value for MC_STORE_MEMCPY: " << env_str
<< ", defaulting to enabled";
memcpy_enabled_ = true;
}
}
VLOG(1) << "TransferSubmitter initialized with memcpy_enabled="
<< memcpy_enabled_;
}
std::optional<TransferFuture> TransferSubmitter::submit(
const Replica::Descriptor& replica, std::vector<Slice>& slices,
Transport::TransferRequest::OpCode op_code) {
std::optional<TransferFuture> future;
if (replica.is_memory_replica()) {
std::vector<AllocatedBuffer::Descriptor> handles;
auto& mem_desc = replica.get_memory_descriptor();
handles = mem_desc.buffer_descriptors;
if (!validateTransferParams(handles, slices)) {
return std::nullopt;
}
TransferStrategy strategy = selectStrategy(handles, slices);
switch (strategy) {
case TransferStrategy::LOCAL_MEMCPY:
future = submitMemcpyOperation(handles, slices, op_code);
break;
case TransferStrategy::TRANSFER_ENGINE:
future =
submitTransferEngineOperation(handles, slices, op_code);
break;
default:
LOG(ERROR) << "Unknown transfer strategy: " << strategy;
return std::nullopt;
}
} else {
future = submitFileReadOperation(replica, slices, op_code);
}
if (future.has_value()) {
updateTransferMetrics(slices, op_code);
}
return future;
}
std::optional<TransferFuture> TransferSubmitter::submit_batch(
const std::vector<Replica::Descriptor>& replicas,
std::vector<std::vector<Slice>>& all_slices,
Transport::TransferRequest::OpCode op_code) {
std::optional<TransferFuture> future;
std::vector<Transport::TransferRequest> requests;
for (size_t i = 0; i < replicas.size(); ++i) {
auto& replica = replicas[i];
auto& slices = all_slices[i];
auto& mem_desc = replica.get_memory_descriptor();
if (!validateTransferParams(mem_desc.buffer_descriptors, slices,
true)) {
return std::nullopt;
}
auto handle = mem_desc.buffer_descriptors[0];
uint64_t offset = 0;
Transport::SegmentHandle seg =
engine_.openSegment(handle.transport_endpoint_);
if (seg == static_cast<uint64_t>(ERR_INVALID_ARGUMENT)) {
LOG(ERROR) << "Failed to open segment "
<< handle.transport_endpoint_;
return std::nullopt;
}
for (auto slice : slices) {
Transport::TransferRequest request;
request.opcode = op_code;
request.source = static_cast<char*>(slice.ptr);
request.target_id = seg;
request.target_offset = handle.buffer_address_ + offset;
request.length = slice.size;
requests.emplace_back(request);
offset += slice.size;
}
}
future = submitTransfer(requests);
if (future.has_value()) {
for (auto& slices : all_slices) {
updateTransferMetrics(slices, op_code);
}
}
return future;
}
std::optional<TransferFuture> TransferSubmitter::submitMemcpyOperation(
const std::vector<AllocatedBuffer::Descriptor>& handles,
std::vector<Slice>& slices, Transport::TransferRequest::OpCode op_code) {
auto state = std::make_shared<MemcpyOperationState>();
std::vector<MemcpyOperation> operations;
operations.reserve(handles.size());
for (size_t i = 0; i < handles.size(); ++i) {
const auto& handle = handles[i];
const auto& slice = slices[i];
if (slice.ptr == nullptr) continue;
void* dest;
const void* src;
if (op_code == Transport::TransferRequest::READ) {
dest = slice.ptr;
src = reinterpret_cast<const void*>(handle.buffer_address_);
} else {
dest = reinterpret_cast<void*>(handle.buffer_address_);
src = slice.ptr;
}
operations.emplace_back(dest, src, handle.size_);
}
MemcpyTask task(std::move(operations), state);
memcpy_pool_->submitTask(std::move(task));
VLOG(1) << "Memcpy transfer submitted to worker pool with "
<< handles.size() << " operations";
return TransferFuture(state);
}
std::optional<TransferFuture> TransferSubmitter::submitTransfer(
std::vector<Transport::TransferRequest>& requests) {
const size_t batch_size = requests.size();
BatchID batch_id = engine_.allocateBatchID(batch_size);
if (batch_id == Transport::INVALID_BATCH_ID) {
LOG(ERROR) << "Failed to allocate batch ID";
return std::nullopt;
}
Status s = engine_.submitTransfer(batch_id, requests);
if (!s.ok()) {
LOG(ERROR) << "Failed to submit all transfers, error code is "
<< s.code();
engine_.freeBatchID(batch_id);
return std::nullopt;
}
if (batch_id == Transport::INVALID_BATCH_ID) {
LOG(ERROR) << "Invalid batch ID for transfer engine operation";
return std::nullopt;
}
auto state = std::make_shared<TransferEngineOperationState>(
engine_, batch_id, batch_size);
return TransferFuture(state);
}
std::optional<TransferFuture> TransferSubmitter::submitTransferEngineOperation(
const std::vector<AllocatedBuffer::Descriptor>& handles,
std::vector<Slice>& slices, Transport::TransferRequest::OpCode op_code) {
std::vector<Transport::TransferRequest> requests;
requests.reserve(handles.size());
for (size_t i = 0; i < handles.size(); ++i) {
const auto& handle = handles[i];
const auto& slice = slices[i];
if (slice.ptr == nullptr) continue;
if (handle.transport_endpoint_.empty()) {
LOG(ERROR) << "Transport endpoint is empty for handle with address "
<< handle.buffer_address_;
return std::nullopt;
}
Transport::SegmentHandle seg =
engine_.openSegment(handle.transport_endpoint_);
if (seg == static_cast<uint64_t>(ERR_INVALID_ARGUMENT)) {
LOG(ERROR) << "Failed to open segment for endpoint='"
<< handle.transport_endpoint_ << "'";
return std::nullopt;
}
Transport::TransferRequest request;
request.opcode = op_code;
request.source = static_cast<char*>(slice.ptr);
request.target_id = seg;
request.target_offset = handle.buffer_address_;
request.length = handle.size_;
requests.emplace_back(request);
}
return submitTransfer(requests);
}
std::optional<TransferFuture> TransferSubmitter::submitFileReadOperation(
const Replica::Descriptor& replica, std::vector<Slice>& slices,
Transport::TransferRequest::OpCode op_code) {
auto state = std::make_shared<FilereadOperationState>();
auto disk_replica = replica.get_disk_descriptor();
std::string file_path = disk_replica.file_path;
size_t file_length = disk_replica.object_size;
FilereadTask task(file_path, file_length, slices, state);
fileread_pool_->submitTask(std::move(task));
VLOG(1) << "Fileread transfer submitted to worker pool with " << file_path;
return TransferFuture(state);
}
TransferStrategy TransferSubmitter::selectStrategy(
const std::vector<AllocatedBuffer::Descriptor>& handles,
const std::vector<Slice>& slices) const {
if (!memcpy_enabled_) {
VLOG(2) << "Memcpy operations disabled via MC_STORE_MEMCPY environment "
"variable";
return TransferStrategy::TRANSFER_ENGINE;
}
if (isLocalTransfer(handles)) {
return TransferStrategy::LOCAL_MEMCPY;
}
return TransferStrategy::TRANSFER_ENGINE;
}
namespace {
std::string extractIpAddress(const std::string& endpoint) {
if (endpoint.empty()) {
return "";
}
if (endpoint[0] == '[') {
size_t closing_bracket = endpoint.find(']');
if (closing_bracket != std::string::npos) {
return endpoint.substr(1, closing_bracket - 1);
}
}
size_t colon_pos = endpoint.rfind(':');
if (colon_pos != std::string::npos) {
return endpoint.substr(0, colon_pos);
}
return endpoint;
}
}
bool TransferSubmitter::isLocalTransfer(
const std::vector<AllocatedBuffer::Descriptor>& handles) const {
std::string local_ep = engine_.getLocalIpAndPort();
std::string local_ip = extractIpAddress(local_ep);
if (!local_ip.empty()) {
return std::all_of(handles.begin(), handles.end(),
[&local_ip](const auto& h) {
if (h.transport_endpoint_.empty()) {
return false;
}
std::string handle_ip = extractIpAddress(h.transport_endpoint_);
return !handle_ip.empty() && handle_ip == local_ip;
});
}
return false;
}
bool TransferSubmitter::validateTransferParams(
const std::vector<AllocatedBuffer::Descriptor>& handles,
const std::vector<Slice>& slices, bool is_multi_buffers) const {
if (handles.empty()) {
LOG(ERROR) << "handles is empty";
return false;
}
if (handles.size() > slices.size()) {
LOG(ERROR) << "invalid_partition_count handles_size=" << handles.size()
<< " slices_size=" << slices.size();
return false;
}
if (is_multi_buffers) {
uint64_t all_slice_len = 0;
for (auto slice : slices) {
all_slice_len += slice.size;
}
if (handles[0].size_ != all_slice_len) {
LOG(ERROR) << "handles len:" << handles[0].size_
<< ", all_slice_len:" << all_slice_len;
return false;
}
} else {
for (size_t i = 0; i < handles.size(); ++i) {
if (handles[i].size_ != slices[i].size) {
LOG(ERROR) << "Size of replica partition " << i << " ("
<< handles[i].size_
<< ") does not match provided buffer ("
<< slices[i].size << ")";
return false;
}
}
}
return true;
}
void TransferSubmitter::updateTransferMetrics(
const std::vector<Slice>& slices,
Transport::TransferRequest::OpCode op_code) {
size_t total_bytes = 0;
for (const auto& slice : slices) {
total_bytes += slice.size;
}
if (transfer_metric_ == nullptr) {
return;
}
if (op_code == Transport::TransferRequest::READ) {
transfer_metric_->total_read_bytes.inc(total_bytes);
} else if (op_code == Transport::TransferRequest::WRITE) {
transfer_metric_->total_write_bytes.inc(total_bytes);
}
}
}