* Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "trace_manager.h"
#include <array>
#include <functional>
#include <nlohmann/json.hpp>
#include <opentelemetry/exporters/ostream/common_utils.h>
#include <opentelemetry/sdk/common/attribute_utils.h>
#include "common/proto/pb/posix_pb.h"
#include "exporter/log_file_exporter_factory.h"
#include "opentelemetry/sdk/trace/processor.h"
#include "opentelemetry/sdk/trace/batch_span_processor.h"
#include "opentelemetry/sdk/trace/batch_span_processor_factory.h"
#include "opentelemetry/sdk/trace/batch_span_processor_options.h"
#include "opentelemetry/sdk/trace/tracer_provider.h"
#include "opentelemetry/sdk/resource/resource.h"
#include "opentelemetry/sdk/resource/semantic_conventions.h"
namespace functionsystem {
namespace trace {
using std::string;
using ProcessorList = std::vector<std::unique_ptr<opentelemetry::sdk::trace::SpanProcessor>>;
using GrpcExporterFactory = std::function<std::unique_ptr<opentelemetry::sdk::trace::SpanExporter>(
const OtelGrpcExporterConfig &)>;
using LogExporterFactory = std::function<std::unique_ptr<opentelemetry::sdk::trace::SpanExporter>()>;
constexpr uint32_t TRACE_ID_LENGTH = 32;
constexpr uint32_t SPAN_ID_LENGTH = 16;
constexpr uint32_t TRACE_ID_BUF_LENGTH = 16;
constexpr uint32_t SPAN_ID_BUF_LENGTH = 8;
constexpr uint32_t TRACE_PARENT_PARTS = 4;
constexpr uint32_t TRACE_PARENT_TRACE_ID_INDEX = 1;
constexpr uint32_t TRACE_PARENT_SPAN_ID_INDEX = 2;
constexpr uint32_t TRACE_PARENT_FLAGS_INDEX = 3;
constexpr uint32_t TRACE_PARENT_FLAGS_LENGTH = 2;
constexpr const char *TRACE_PARENT_VERSION = "00";
constexpr const char *TRACE_PARENT_DEFAULT_FLAGS = "01";
constexpr const char *TRACE_PARENT_OPTION_KEY = "traceparent";
constexpr const char *LEGACY_TRACE_PARENT_OPTION_KEY = "Traceparent";
static inline bool IsHexDecimal(const std::string &str)
{
return std::all_of(str.begin(), str.end(), ::isxdigit);
}
static bool TraceIdStrToArr(std::string traceID, uint8_t (&arr)[TRACE_ID_BUF_LENGTH])
{
(void)traceID.erase(std::remove(traceID.begin(), traceID.end(), '-'), traceID.end());
if (traceID.size() < TRACE_ID_LENGTH) {
(void)traceID.append(TRACE_ID_LENGTH - traceID.size(), '0');
}
traceID = traceID.substr(traceID.size() - TRACE_ID_LENGTH, TRACE_ID_LENGTH);
if (!IsHexDecimal(traceID)) {
YRLOG_WARN("invalid traceID format: {}", traceID);
return false;
}
if (traceID.length() != TRACE_ID_LENGTH && traceID.length() != (TRACE_ID_LENGTH - 1)) {
YRLOG_WARN("invalid length: {}, traceID: {}", traceID.length(), traceID);
return false;
}
YRLOG_DEBUG("load trace id: {} string to buffer array", traceID);
int pivot = 0;
for (size_t i = 0; i < traceID.length(); i += 2) {
std::string sub = traceID.substr(i, 2);
int value = std::stoi(sub, nullptr, 16);
arr[pivot++] = uint8_t(value);
}
return pivot == TRACE_ID_BUF_LENGTH;
}
static bool SpanIdStrToArr(const std::string &spanID, uint8_t (&arr)[SPAN_ID_BUF_LENGTH])
{
if (spanID.length() != SPAN_ID_LENGTH && spanID.length() != (SPAN_ID_LENGTH - 1)) {
YRLOG_WARN("invalid length: {}, spanID: {}", spanID.length(), spanID);
return false;
}
if (!IsHexDecimal(spanID)) {
YRLOG_WARN("invalid spanID format: {}", spanID);
return false;
}
int pivot = 0;
for (size_t i = 0; i < spanID.length(); i += 2) {
std::string sub = spanID.substr(i, 2);
int value = std::stoi(sub, nullptr, 16);
arr[pivot++] = uint8_t(value);
}
return pivot == SPAN_ID_BUF_LENGTH;
}
static bool TrySetParentFromTraceParent(opentelemetry::trace::StartSpanOptions &options, const std::string &traceParent)
{
if (traceParent.empty()) {
return false;
}
std::array<std::string, TRACE_PARENT_PARTS> parts;
size_t start = 0;
for (size_t i = 0; i < TRACE_PARENT_PARTS - 1; ++i) {
auto end = traceParent.find('-', start);
if (end == std::string::npos) {
YRLOG_WARN("invalid traceparent format: {}", traceParent);
return false;
}
parts[i] = traceParent.substr(start, end - start);
start = end + 1;
}
parts[TRACE_PARENT_PARTS - 1] = traceParent.substr(start);
if (parts[TRACE_PARENT_PARTS - 1].find('-') != std::string::npos) {
YRLOG_WARN("invalid traceparent format: {}", traceParent);
return false;
}
if (parts[TRACE_PARENT_FLAGS_INDEX].length() != TRACE_PARENT_FLAGS_LENGTH ||
!IsHexDecimal(parts[TRACE_PARENT_FLAGS_INDEX])) {
YRLOG_WARN("invalid traceparent trace flags: {}", traceParent);
return false;
}
uint8_t traceIdArr[TRACE_ID_BUF_LENGTH] = {};
uint8_t spanIdArr[SPAN_ID_BUF_LENGTH] = {};
if (!TraceIdStrToArr(parts[TRACE_PARENT_TRACE_ID_INDEX], traceIdArr) ||
!SpanIdStrToArr(parts[TRACE_PARENT_SPAN_ID_INDEX], spanIdArr)) {
YRLOG_WARN("failed to parse traceparent: {}", traceParent);
return false;
}
opentelemetry::trace::TraceId optlTraceId(traceIdArr);
opentelemetry::trace::SpanId optlSpanId(spanIdArr);
opentelemetry::trace::SpanContext spanContext(optlTraceId, optlSpanId, {}, true);
if (!spanContext.IsValid()) {
YRLOG_WARN("traceparent resolved to invalid span context: {}", traceParent);
return false;
}
options.parent = spanContext;
return true;
}
std::string TraceManager::GetTraceParentFromOptions(
const google::protobuf::Map<std::string, std::string> &options,
const google::protobuf::Map<std::string, std::string> *fallback)
{
auto traceParent = options.find(TRACE_PARENT_OPTION_KEY);
if (traceParent != options.end()) {
return traceParent->second;
}
auto legacyTraceParent = options.find(LEGACY_TRACE_PARENT_OPTION_KEY);
if (legacyTraceParent != options.end()) {
return legacyTraceParent->second;
}
if (fallback != nullptr) {
return GetTraceParentFromOptions(*fallback, nullptr);
}
return "";
}
void TraceManager::SetTraceParentToOptions(google::protobuf::Map<std::string, std::string> *options,
const std::string &traceParent)
{
if (options == nullptr || traceParent.empty()) {
return;
}
(*options)[TRACE_PARENT_OPTION_KEY] = traceParent;
(*options)[LEGACY_TRACE_PARENT_OPTION_KEY] = traceParent;
}
void TraceManager::PropagateSpanToOptions(const OtelSpan &span,
google::protobuf::Map<std::string, std::string> *options,
google::protobuf::Map<std::string, std::string> *fallback)
{
auto traceParent = GetTraceParentFromSpan(span);
if (traceParent.empty()) {
return;
}
SetTraceParentToOptions(options, traceParent);
if (fallback != nullptr) {
SetTraceParentToOptions(fallback, traceParent);
}
}
void AppendGrpcProcessor(const nlohmann::json &exporterConfig, ProcessorList *processors,
const GrpcExporterFactory &factory)
{
if (!exporterConfig.contains("enable") || !exporterConfig.at("enable").get<bool>()) {
YRLOG_INFO("Trace exporter {} is not enabled", OTLP_GRPC_EXPORTER);
return;
}
if (!exporterConfig.contains("endpoint") || exporterConfig.at("endpoint").get<std::string>().empty()) {
YRLOG_INFO("Trace exporter {} endpoint is not valid", OTLP_GRPC_EXPORTER);
return;
}
OtelGrpcExporterConfig config;
config.endpoint = exporterConfig.at("endpoint").get<std::string>();
opentelemetry::sdk::trace::BatchSpanProcessorOptions options;
YRLOG_INFO("OtelGrpcExporter is enable, endpoint is {}", config.endpoint);
processors->push_back(std::unique_ptr<opentelemetry::sdk::trace::SpanProcessor>(
opentelemetry::sdk::trace::BatchSpanProcessorFactory::Create(factory(config), options)));
}
void AppendLogFileProcessor(const nlohmann::json &exporterConfig, ProcessorList *processors,
const LogExporterFactory &factory)
{
if (!exporterConfig.contains("enable") || !exporterConfig.at("enable").get<bool>()) {
YRLOG_INFO("Trace exporter {} is not enabled", LOG_FILE_EXPORTER);
return;
}
opentelemetry::sdk::trace::BatchSpanProcessorOptions options;
YRLOG_INFO("logFileExporter is enable");
processors->push_back(std::unique_ptr<opentelemetry::sdk::trace::SpanProcessor>(
opentelemetry::sdk::trace::BatchSpanProcessorFactory::Create(factory(), options)));
}
void TraceManager::InitTrace(const std::string &serviceName, const std::string &hostID,
const bool &enableTrace, const std::string &traceConfig)
{
enableTrace_ = enableTrace;
YRLOG_INFO("init trace, enableTrace is {}, traceConfig is {}", enableTrace, traceConfig);
if (!enableTrace_) {
return;
}
hostID_ = hostID;
ProcessorList processors;
try {
auto confJson = nlohmann::json::parse(traceConfig);
for (auto &element : confJson.items()) {
if (element.key() == OTLP_GRPC_EXPORTER) {
AppendGrpcProcessor(element.value(), &processors,
[this](const OtelGrpcExporterConfig &config) {
return InitOtlpGrpcExporter(config);
});
} else if (element.key() == LOG_FILE_EXPORTER) {
AppendLogFileProcessor(element.value(), &processors, [this]() {
return InitLogFileExporter();
});
}
}
} catch (nlohmann::detail::parse_error &e) {
YRLOG_ERROR("Failed to parse trace config json, error: {}", e.what());
enableTrace_ = false;
return;
} catch (std::exception &e) {
YRLOG_ERROR("Failed to parse trace config json, error: {}", e.what());
enableTrace_ = false;
return;
}
if (processors.empty()) {
YRLOG_WARN("There is no supported exporter in config");
enableTrace_ = false;
return;
}
opentelemetry::sdk::resource::ResourceAttributes attributes = {
{ opentelemetry::sdk::resource::SemanticConventions::kTelemetrySdkLanguage, "" },
{ opentelemetry::sdk::resource::SemanticConventions::kTelemetrySdkName, "" },
{ opentelemetry::sdk::resource::SemanticConventions::kTelemetrySdkVersion, "" },
{ opentelemetry::sdk::resource::SemanticConventions::kServiceName, serviceName },
};
auto provider = std::shared_ptr<opentelemetry::trace::TracerProvider>(
std::make_shared<opentelemetry::sdk::trace::TracerProvider>(
std::move(processors), opentelemetry::sdk::resource::Resource::Create(attributes)));
opentelemetry::trace::Provider::SetTracerProvider(provider);
}
void TraceManager::ShutDown()
{
if (!enableTrace_) {
return;
}
YRLOG_INFO("enter TraceManager shutDown");
enableTrace_ = false;
auto provider = opentelemetry::trace::Provider::GetTracerProvider();
auto traceProvider = static_cast<opentelemetry::sdk::trace::TracerProvider*>(provider.get());
if (traceProvider != nullptr && !traceProvider->ForceFlush()) {
YRLOG_WARN("traceProvider shutDown failed");
}
opentelemetry::nostd::shared_ptr<opentelemetry::trace::TracerProvider> none;
opentelemetry::trace::Provider::SetTracerProvider(none);
}
void TraceManager::SetAttr(const std::string &attr, const std::string &value)
{
attribute_.insert_or_assign(attr, value);
}
TraceManager::OtelSpan TraceManager::CreateNoopSpan()
{
static auto noopTracer = std::make_shared<opentelemetry::trace::NoopTracer>();
return opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span>(
new opentelemetry::trace::NoopSpan(noopTracer));
}
TraceManager::OtelSpan TraceManager::StartSpan(const std::string &name,
const opentelemetry::common::KeyValueIterable &attributes,
const opentelemetry::trace::SpanContextKeyValueIterable &links,
const opentelemetry::trace::StartSpanOptions &options)
{
if (!enableTrace_) {
return CreateNoopSpan();
}
try {
auto tracer = GetTracer();
if (tracer != nullptr) {
return tracer->StartSpan(name, attributes, links, options);
}
} catch (const std::exception &e) {
YRLOG_ERROR("StartSpan exception: {}", e.what());
}
return CreateNoopSpan();
}
TraceManager::OtelSpan TraceManager::StartSpan(const std::string &name,
const opentelemetry::trace::StartSpanOptions &options)
{
return StartSpan(name, opentelemetry::common::NoopKeyValueIterable(),
opentelemetry::trace::NullSpanContext(), options);
}
TraceManager::OtelSpan TraceManager::StartSpan(
const std::string &name,
const std::string &traceID,
const std::string &spanID,
const std::string &traceParent,
AttributesVector &attrs)
{
YRLOG_DEBUG("start span with traceID and spanID, name: {}, traceID: {}, spanID: {}, traceParent: {}",
name, traceID, spanID, traceParent);
auto options = BuildOptWithParent(traceID, spanID, traceParent);
for (auto it : attribute_) {
attrs.emplace_back(it);
}
return StartSpan(name, opentelemetry::common::KeyValueIterableView(attrs),
opentelemetry::trace::NullSpanContext(), options);
}
TraceManager::OtelSpan TraceManager::StartSpanWithRecord(TraceManager::SpanParam &&spanParam)
{
std::string spanKey = spanParam.spanKey + "_" + spanParam.spanName;
YRLOG_DEBUG("(trace)start span, name: {}, traceID: {}, spanID: {}, traceParent: {}, function: {}, instanceID: {}",
spanParam.spanName, spanParam.traceID, spanParam.spanID, spanParam.traceParent,
spanParam.function, spanParam.instanceID);
AttributesVector attrs;
if (!spanParam.function.empty()) {
attrs.emplace_back("yr.function", spanParam.function);
}
if (!spanParam.instanceID.empty()) {
attrs.emplace_back("yr.instance_id", spanParam.instanceID);
}
if (!hostID_.empty()) {
attrs.emplace_back("host.id", hostID_);
}
auto span = StartSpan(spanParam.spanName, spanParam.traceID, spanParam.spanID, spanParam.traceParent, attrs);
if (span != nullptr) {
std::lock_guard<std::mutex> lock(spanMapMutex_);
spanMap_.emplace(spanKey, span);
}
return span;
}
void TraceManager::StopSpan(const std::string &spanName, const std::string &spanKey,
const AttributesVector &attrs,
const std::vector<std::string> &events)
{
std::string spanMapKey = spanKey + "_" + spanName;
YRLOG_DEBUG("stop span, key: {}", spanMapKey);
OtelSpan span;
{
std::lock_guard<std::mutex> lock(spanMapMutex_);
auto it = spanMap_.find(spanMapKey);
if (it == spanMap_.end()) {
YRLOG_WARN("no span: {} found with spanKey: {}", spanName, spanKey);
return;
}
span = it->second;
}
for (const auto &event : events) {
YRLOG_DEBUG("stopspan add event: {}", event);
span->AddEvent(event);
}
for (const auto &[key, value] : attrs) {
YRLOG_DEBUG("stopspan add attr: {}", key);
span->SetAttribute(key, value);
}
opentelemetry::trace::EndSpanOptions options;
options.end_steady_time = opentelemetry::common::SteadyTimestamp(std::chrono::steady_clock::now());
span->End(options);
{
std::lock_guard<std::mutex> lock(spanMapMutex_);
spanMap_.erase(spanMapKey);
YRLOG_DEBUG("stop current span, spanKey: {}, spanName: {}", spanKey, spanName);
}
}
std::string TraceManager::GetSpanIDFromStore(const std::string &spanKey, const std::string &spanName)
{
std::lock_guard<std::mutex> lock(spanMapMutex_);
auto spanMapKey = spanKey + "_" + spanName;
auto it = spanMap_.find(spanMapKey);
if (it == spanMap_.end()) {
YRLOG_WARN("cannot find span in spanMap_. spanKey: {}", spanMapKey);
return "";
}
auto spanID = it->second->GetContext().span_id();
return SpanIDToStr(spanID);
}
void TraceManager::Clear()
{
std::lock_guard<std::mutex> lock(spanMapMutex_);
spanMap_.clear();
}
std::string TraceManager::SpanIDToStr(const opentelemetry::trace::SpanId &spanId)
{
std::string spanIDStr;
for (auto it = spanId.Id().begin(); it != spanId.Id().end(); ++it) {
auto value = static_cast<int>(*it);
std::ostringstream ss;
ss << std::setfill('0') << std::setw(2) << std::hex << value;
std::string element = ss.str();
(void)spanIDStr.append(element);
}
return spanIDStr;
}
std::string TraceManager::TraceIDToStr(const opentelemetry::trace::TraceId &traceID)
{
std::string traceIDStr;
for (auto it = traceID.Id().begin(); it != traceID.Id().end(); ++it) {
auto value = static_cast<int>(*it);
std::ostringstream ss;
ss << std::setfill('0') << std::setw(2) << std::hex << value;
std::string element = ss.str();
(void)traceIDStr.append(element);
}
return traceIDStr;
}
std::string TraceManager::SpanContextToTraceParent(const opentelemetry::trace::SpanContext &spanContext)
{
if (!spanContext.IsValid()) {
return "";
}
return fmt::format("{}-{}-{}-{}",
TRACE_PARENT_VERSION,
TraceIDToStr(spanContext.trace_id()),
SpanIDToStr(spanContext.span_id()),
TRACE_PARENT_DEFAULT_FLAGS);
}
std::string TraceManager::GetTraceParentFromSpan(const OtelSpan &span)
{
if (span == nullptr) {
return "";
}
return SpanContextToTraceParent(span->GetContext());
}
opentelemetry::nostd::shared_ptr<opentelemetry::trace::Tracer> TraceManager::GetTracer(const std::string &name,
const std::string &version)
{
auto provider = opentelemetry::trace::Provider::GetTracerProvider();
return provider->GetTracer(name, version);
}
std::unique_ptr<opentelemetry::sdk::trace::SpanExporter> TraceManager::InitLogFileExporter()
{
return LogFileExporterFactory::Create();
}
std::unique_ptr<opentelemetry::sdk::trace::SpanExporter> TraceManager::InitOtlpGrpcExporter(
const OtelGrpcExporterConfig &conf)
{
if (conf.endpoint.empty()) {
return nullptr;
}
opentelemetry::exporter::otlp::OtlpGrpcExporterOptions options;
options.endpoint = conf.endpoint;
return opentelemetry::exporter::otlp::OtlpGrpcExporterFactory::Create(options);
}
opentelemetry::trace::StartSpanOptions TraceManager::BuildOptWithParent(const std::string &traceID,
const std::string &spanID,
const std::string &traceParent)
{
YRLOG_DEBUG("build options with parent, traceID: {}, spanID: {}, traceParent: {}",
traceID, spanID, traceParent);
opentelemetry::trace::StartSpanOptions options;
if (TrySetParentFromTraceParent(options, traceParent)) {
YRLOG_DEBUG("build parent context from traceparent");
} else if (!traceID.empty() && !spanID.empty()) {
uint8_t traceIdArr[TRACE_ID_BUF_LENGTH] = {};
uint8_t spanIdArr[SPAN_ID_BUF_LENGTH] = {};
if (!TraceIdStrToArr(traceID, traceIdArr) || !SpanIdStrToArr(spanID, spanIdArr)) {
YRLOG_WARN("failed to build parent from traceID/spanID, traceID: {}, spanID: {}", traceID, spanID);
options.start_steady_time = opentelemetry::common::SteadyTimestamp(std::chrono::steady_clock::now());
return options;
}
opentelemetry::trace::TraceId optlTraceId(traceIdArr);
opentelemetry::trace::SpanId optlSpanId(spanIdArr);
opentelemetry::trace::SpanContext spanContext(optlTraceId, optlSpanId, {}, false);
YRLOG_DEBUG("option is valid({})", spanContext.IsValid());
options.parent = spanContext;
} else if (!traceID.empty()) {
uint8_t traceIdArr[TRACE_ID_BUF_LENGTH] = {};
uint8_t spanIdArr[SPAN_ID_BUF_LENGTH] = {};
if (!TraceIdStrToArr(traceID, traceIdArr)) {
YRLOG_WARN("failed to build synthetic parent from traceID, traceID: {}", traceID);
options.start_steady_time = opentelemetry::common::SteadyTimestamp(std::chrono::steady_clock::now());
return options;
}
spanIdArr[SPAN_ID_BUF_LENGTH - 1] = 0x01;
opentelemetry::trace::TraceId optlTraceId(traceIdArr);
opentelemetry::trace::SpanId optlSpanId(spanIdArr);
opentelemetry::trace::SpanContext spanContext(optlTraceId, optlSpanId, {}, false);
YRLOG_DEBUG("traceID exists without parent span, fallback to synthetic parent");
options.parent = spanContext;
} else {
YRLOG_DEBUG("traceID is empty");
}
options.start_steady_time = opentelemetry::common::SteadyTimestamp(std::chrono::steady_clock::now());
return options;
}
}
}
