* Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
* libkperf licensed under the Mulan PSL v2.
* You can use this software according to the terms and conditions of the Mulan PSL v2.
* You may obtain a copy of Mulan PSL v2 at:
* http://license.coscl.org.cn/MulanPSL2
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
* PURPOSE.
* See the Mulan PSL v2 for more details.
* Author: Mr.Zhang
* Create: 2024-04-03
* Description: implementations for managing performance monitoring tasks, collecting data,
* and handling performance counters in the KUNPENG_PMU namespace
******************************************************************************/
#include <iostream>
#include <fstream>
#include <unistd.h>
#include <linux/perf_event.h>
#include <linux/version.h>
#include <cstring>
#include "common.h"
#include "pfm.h"
#include "pfm_event.h"
#include "pmu_event.h"
#include "pmu_list.h"
#include "linked_list.h"
#include "pcerr.h"
#include "safe_handler.h"
#include "pmu_metric.h"
#include "trace_point_parser.h"
#include "pmu.h"
#include "simple_pebs_backend.h"
using namespace pcerr;
using namespace KUNPENG_PMU;
using namespace std;
static unordered_map<unsigned, bool> runningStatus;
static SafeHandler<unsigned> pdMutex;
static pair<unsigned, const char**> uncoreEventPair;
static unordered_map<int, int> groupEvtCapacity = {{HIPA, 12}, {HIPB, 8}, {HIPC, 8},
{HIPF, 8}, {HIPE, 8}, {HIPG, 6}};
#define REQUEST_USER_ACCESS 0x2
#define HARD_WARE_METRIC 0x1
#define SAMPLING_RECORD_PERIOD 4000
struct PmuTaskAttr* AssignPmuTaskParam(PmuTaskType collectType, struct PmuAttr *attr);
static int PmuCollectStart(const int pd)
{
auto err = KUNPENG_PMU::PmuList::GetInstance()->Start(pd);
if (err != SUCCESS) {
New(err);
return -1;
}
return SUCCESS;
}
static int PmuCollectPause(const int pd)
{
auto err = KUNPENG_PMU::PmuList::GetInstance()->Pause(pd);
if (err != SUCCESS) {
New(err);
return -1;
}
return SUCCESS;
}
static int CheckCpuList(unsigned numCpu, int* cpuList)
{
const set<int>& onLineCpus = GetOnLineCpuIds();
if (numCpu > MAX_CPU_NUM) {
string errMsg = "Invalid numCpu: " + to_string(numCpu);
New(LIBPERF_ERR_INVALID_CPULIST, errMsg);
return LIBPERF_ERR_INVALID_CPULIST;
}
if (numCpu > 0 && cpuList == nullptr) {
New(LIBPERF_ERR_INVALID_CPULIST);
return LIBPERF_ERR_INVALID_CPULIST;
}
for (int i = 0; i < numCpu; i++) {
if (cpuList[i] < -1 || cpuList[i] >= MAX_CPU_NUM) {
string errMsg = "Invalid cpu id: " + to_string(cpuList[i]) + ", Please check cpu config parameter.";
New(LIBPERF_ERR_INVALID_CPULIST, errMsg);
return LIBPERF_ERR_INVALID_CPULIST;
}
if (cpuList[i] != -1 && !onLineCpus.count(cpuList[i])) {
string errMsg = "OffLine cpu id: " + to_string(cpuList[i]);
New(LIBPERF_ERR_INVALID_CPULIST, errMsg);
return LIBPERF_ERR_INVALID_CPULIST;
}
}
return SUCCESS;
}
static int CheckPidList(unsigned numPid, int* pidList)
{
if (numPid > 0 && pidList == nullptr) {
New(LIBPERF_ERR_INVALID_PIDLIST);
return LIBPERF_ERR_INVALID_PIDLIST;
}
for (int i = 0; i < numPid; i++) {
if (pidList[i] < 0) {
string errMsg = "Invalid pid: " + to_string(pidList[i]) + ", Please check pid config parameter.";
New(LIBPERF_ERR_INVALID_PIDLIST, errMsg);
return LIBPERF_ERR_INVALID_PIDLIST;
}
}
return SUCCESS;
}
static int CheckEvtList(unsigned numEvt, char** evtList)
{
if (numEvt > 0 && evtList == nullptr) {
New(LIBPERF_ERR_INVALID_EVTLIST, "Invalid event list: numEvt is greater than 0, but evtList is null.");
return LIBPERF_ERR_INVALID_EVTLIST;
}
return SUCCESS;
}
static int CheckGroupList(unsigned numEvtAttr, struct EvtAttr *evtAttr)
{
if (numEvtAttr > 0 && evtAttr == nullptr) {
New(LIBPERF_ERR_INVALID_EVTATTR, "Invalid evtAttr list: numEvtAttr is greater than 0, but evtAttr is null.");
return LIBPERF_ERR_INVALID_EVTATTR;
}
CHIP_TYPE chipType = GetCpuType();
int maxEvt;
auto findType = groupEvtCapacity.find(chipType);
if (findType == groupEvtCapacity.end()) {
return SUCCESS;
} else {
maxEvt = findType->second;
}
unordered_map<int, size_t> cntEvtMap;
for (size_t i = 0; i < numEvtAttr; i++) {
if (evtAttr[i].groupId != -1) {
cntEvtMap[evtAttr[i].groupId]++;
}
}
for (const auto& evt : cntEvtMap) {
if (evt.second > maxEvt) {
New(LIBPERF_ERR_INVALID_EVTATTR, "Invalid evtAttr list: the number of events in group " + to_string(evt.first)
+ " is: " + to_string(evt.second) + ", which exceeds the PMU collection limit: " + to_string(maxEvt));
return LIBPERF_ERR_INVALID_EVTATTR;
}
}
return SUCCESS;
}
static bool InvalidSampleRate(enum PmuTaskType collectType, struct PmuAttr *attr)
{
if (collectType != SAMPLING) {
return false;
}
if (!attr->useFreq) {
return false;
}
const string sysSampleRate = "/proc/sys/kernel/perf_event_max_sample_rate";
ifstream inSys(sysSampleRate);
if (!inSys.is_open()) {
return false;
}
unsigned long maxRate = 0;
inSys >> maxRate;
return attr->freq > maxRate;
}
static int CheckBranchSampleFilter(const unsigned long& branchSampleFilter, enum PmuTaskType collectType)
{
if (branchSampleFilter == KPERF_NO_BRANCH_SAMPLE) {
return SUCCESS;
}
if (collectType != SAMPLING) {
return LIBPERF_ERR_BRANCH_JUST_SUPPORT_SAMPLING;
}
unsigned long branchFilterTmp = 0;
for (int i = 0; i <= 16; i++) {
if (branchSampleFilter & (1U << i)) {
branchFilterTmp |= 1U << i;
}
}
if (branchSampleFilter != branchFilterTmp) {
return LIBPERF_ERR_INVALID_BRANCH_SAMPLE_FILTER;
}
if (branchSampleFilter <= (KPERF_SAMPLE_BRANCH_KERNEL | KPERF_SAMPLE_BRANCH_USER | KPERF_SAMPLE_BRANCH_HV)) {
pcerr::SetCustomErr(LIBPERF_ERR_INVALID_BRANCH_SAMPLE_FILTER,
"invalid value for branchSampleFilter, must set at least one or more "
"bits values greater than or equal to KPERF_SAMPLE_BRANCH_ANY.");
return LIBPERF_ERR_INVALID_BRANCH_SAMPLE_FILTER;
}
return SUCCESS;
}
static int CheckCgroupNameList(unsigned numCgroup, char** cgroupName)
{
if (numCgroup > 0 && cgroupName == nullptr) {
New(LIBPERF_ERR_INVALID_CGROUP_LIST, "Invalid cgroup name list: numCgroup is greater than 0, but cgroupNameList is null");
return LIBPERF_ERR_INVALID_CGROUP_LIST;
}
for (unsigned i = 0; i < numCgroup; i++) {
std::string cgroupPath = GetCgroupPath(cgroupName[i]);
if (cgroupPath.empty()) {
New(LIBPERF_ERR_INVALID_CGROUP_LIST, "The cgroup mount does not exist or has no permission to access");
return LIBPERF_ERR_INVALID_CGROUP_LIST;
}
int cgroupFd = open(cgroupPath.c_str(), O_RDONLY);
if (cgroupFd < 0) {
New(LIBPERF_ERR_OPEN_INVALID_FILE, "open " + cgroupPath + " failed.");
return LIBPERF_ERR_OPEN_INVALID_FILE;
}
close(cgroupFd);
}
return SUCCESS;
}
static bool CheckValidSpeEvtFilter(SpeEventFilter val)
{
switch (val) {
case SPE_EVENT_NONE:
case SPE_EVENT_RETIRED:
case SPE_EVENT_L1DMISS:
case SPE_EVENT_TLB_WALK:
case SPE_EVENT_MISPREDICTED:
return true;
default:
return false;
}
}
static bool CheckValidSpeFilter(SpeFilter val)
{
const unsigned long validMask =
SPE_FILTER_NONE |
TS_ENABLE |
PA_ENABLE |
PCT_ENABLE |
JITTER |
BRANCH_FILTER |
LOAD_FILTER |
STORE_FILTER;
if ((val & ~validMask) == 0) {
return true;
}
return false;
}
static int CheckCollectTypeConfig(enum PmuTaskType collectType, struct PmuAttr *attr)
{
if (collectType < 0 || collectType >= MAX_TASK_TYPE) {
New(LIBPERF_ERR_INVALID_TASK_TYPE);
return LIBPERF_ERR_INVALID_TASK_TYPE;
}
#ifdef IS_X86
if (collectType != COUNTING && collectType != SAMPLING) {
New(LIBPERF_ERR_INVALID_TASK_TYPE, "The x86 architecture supports only the COUNTING mode and SMAPLING mode");
return LIBPERF_ERR_INVALID_TASK_TYPE;
}
#endif
if ((collectType == COUNTING) && attr->evtList == nullptr) {
New(LIBPERF_ERR_INVALID_EVTLIST, "Counting mode requires a non-null event list.");
return LIBPERF_ERR_INVALID_EVTLIST;
}
if (collectType != SAMPLING && attr->blockedSample == 1) {
New(LIBPERF_ERR_INVALID_BLOCKED_SAMPLE, "blocked sample mode only support sampling mode!");
return LIBPERF_ERR_INVALID_BLOCKED_SAMPLE;
}
if (collectType == SAMPLING) {
if (attr->blockedSample == 0 && attr->evtList == nullptr) {
New(LIBPERF_ERR_INVALID_EVTLIST, "In sampling mode without blocked sample, the event list cannot be null.");
return LIBPERF_ERR_INVALID_EVTLIST;
} else if (attr->blockedSample == 1) {
if (attr->evtList != nullptr) {
New(LIBPERF_ERR_NOT_SUPPORT_CONFIG_EVENT, "Blocked sampling mode does not support configuring events to sample!");
return LIBPERF_ERR_NOT_SUPPORT_CONFIG_EVENT;
}
if (attr->evtAttr != nullptr) {
New(LIBPERF_ERR_NOT_SUPPORT_GROUP_EVENT, "Blocked sampling mode does not support grouped events!");
return LIBPERF_ERR_NOT_SUPPORT_GROUP_EVENT;
}
if (attr->pidList == nullptr) {
New(LIBPERF_ERR_NOT_SUPPORT_SYSTEM_SAMPLE, "Blocked sampling mode does not support sample system!");
return LIBPERF_ERR_NOT_SUPPORT_SYSTEM_SAMPLE;
}
}
}
if (attr->cgroupNameList != nullptr && attr->pidList != nullptr) {
New(LIBPERF_ERR_INVALID_CGROUP_LIST, "Cannot specify both cgroup and pid. Please use only one.");
return LIBPERF_ERR_INVALID_CGROUP_LIST;
}
if (collectType == SPE_SAMPLING) {
if( attr->evtAttr != nullptr) {
New(LIBPERF_ERR_INVALID_GROUP_SPE);
return LIBPERF_ERR_INVALID_GROUP_SPE;
}
if (attr->numCgroup > 1) {
New(LIBPERF_ERR_INVALID_CGROUP_LIST, "SPE mode only support one cgroup");
return LIBPERF_ERR_INVALID_CGROUP_LIST;
}
if (attr->minLatency > 4095) {
New(LIBPERF_ERR_INVALID_MIN_LATENCY, "Invalid min_latency: value must be between 0 and 4095");
return LIBPERF_ERR_INVALID_MIN_LATENCY;
}
if (!CheckValidSpeEvtFilter(attr->evFilter)) {
New(LIBPERF_ERR_INVALID_EVT_FILTER, "Invalid evt filter in SPE mode");
return LIBPERF_ERR_INVALID_EVT_FILTER;
}
if (!CheckValidSpeFilter(attr->dataFilter)) {
New(LIBPERF_ERR_INVALID_DATA_FILTER, "Invalid data filter in SPE mode");
return LIBPERF_ERR_INVALID_DATA_FILTER;
}
}
return SUCCESS;
}
static int CheckUserAccess(enum PmuTaskType collectType, struct PmuAttr *attr)
{
if (!attr->enableUserAccess) {
return SUCCESS;
}
if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0)) {
New(LIBPERF_ERR_CHECK_USER_ACCESS, "Pmuv3 is supported after linux-kernel-v6.6");
return LIBPERF_ERR_CHECK_USER_ACCESS;
}
if (attr->numPid != 1 || attr->pidList[0] != 0) {
New(LIBPERF_ERR_CHECK_USER_ACCESS, "The pidList is incorrectly set!");
return LIBPERF_ERR_CHECK_USER_ACCESS;
}
if (attr->numCpu != 1 || attr->cpuList[0] != -1) {
New(LIBPERF_ERR_CHECK_USER_ACCESS, "The cpuList is incorrectly set!");
return LIBPERF_ERR_CHECK_USER_ACCESS;
}
if (collectType != COUNTING) {
New(LIBPERF_ERR_CHECK_USER_ACCESS, "User Access only supports count!");
return LIBPERF_ERR_CHECK_USER_ACCESS;
}
if (attr->evtAttr != nullptr) {
New(LIBPERF_ERR_CHECK_USER_ACCESS, "User Access doesn't support event group!");
return LIBPERF_ERR_CHECK_USER_ACCESS;
}
std::ifstream ifs("/proc/sys/kernel/perf_user_access");
if (!ifs) {
return LIBPERF_ERR_CHECK_USER_ACCESS;
}
int val = 0;
if (ifs >> val && val != 1) {
New(LIBPERF_ERR_CHECK_USER_ACCESS, "Run 'sudo sysctl kernel/perf_user_access=1' to enable perf_user_access!");
return LIBPERF_ERR_CHECK_USER_ACCESS;
}
return SUCCESS;
}
static int CheckBpfMode(enum PmuTaskType collectType, struct PmuAttr *attr)
{
if (!attr->enableBpf) {
return SUCCESS;
}
#ifndef BPF_ENABLED
New(LIBPERF_ERR_INVALID_BPF_PARAM, "No compilation of 'bpf=true' to support bpf mode");
return LIBPERF_ERR_INVALID_BPF_PARAM;
#endif
if (collectType != COUNTING) {
New(LIBPERF_ERR_INVALID_BPF_PARAM, "Bpf mode only support counting");
return LIBPERF_ERR_INVALID_BPF_PARAM;
}
if (attr->cgroupNameList == nullptr && attr->pidList == nullptr) {
New(LIBPERF_ERR_INVALID_BPF_PARAM, "Bpf mode need collect pid or cgroup");
return LIBPERF_ERR_INVALID_BPF_PARAM;
}
if (attr->evtAttr != nullptr) {
New(LIBPERF_ERR_INVALID_BPF_PARAM, "Bpf mode doesn't support event group now");
return LIBPERF_ERR_INVALID_BPF_PARAM;
}
set<string> events;
for (int i = 0; i < attr->numEvt; i++) {
std::string evt(attr->evtList[i]);
if (!events.insert(evt).second) {
New(LIBPERF_ERR_INVALID_BPF_PARAM, "Bpf mode doesn't support duplicate event names");
return LIBPERF_ERR_INVALID_BPF_PARAM;
}
}
return SUCCESS;
}
static int CheckHwMetric(enum PmuTaskType collectType, struct PmuAttr* attr) {
if (!attr->enableHwMetric) {
return SUCCESS;
}
if (collectType != SAMPLING) {
New(LIBPERF_ERR_NOT_SUPPORT_HWMETRIC, "just sampling mode support hw metric");
return LIBPERF_ERR_NOT_SUPPORT_HWMETRIC;
}
if (!CheckCurKernelConfig("CONFIG_HISILICON_HW_METRIC=y")) {
New(LIBPERF_ERR_NOT_SUPPORT_HWMETRIC, "Current kernel can't support hw metric, Please upgrade the kernel version to 6.6.0-108");
return LIBPERF_ERR_NOT_SUPPORT_HWMETRIC;
}
return SUCCESS;
}
static int CheckEnableOnExec(struct PmuAttr* attr) {
if (!attr->enableOnExec) {
return SUCCESS;
}
if (!attr->numPid) {
New(LIBPERF_ERR_NOT_SUPPORT_EXEC_ON, "EnableExecOn can't be enabled without specifying a process");
return LIBPERF_ERR_NOT_SUPPORT_EXEC_ON;
}
return SUCCESS;
}
static int CheckPerThread(enum PmuTaskType collectType, struct PmuAttr* attr) {
if (!attr->perThread) {
return SUCCESS;
}
if (collectType != SAMPLING) {
New(LIBPERF_ERR_NOT_SUPPORT_PER_THREAD, "perThread just supports SAMPLING mode");
return LIBPERF_ERR_NOT_SUPPORT_PER_THREAD;
}
if (!attr->numPid) {
New(LIBPERF_ERR_NOT_SUPPORT_PER_THREAD, "perThread can't be enabled without specifying a process");
return LIBPERF_ERR_NOT_SUPPORT_PER_THREAD;
}
if (attr->numCpu) {
New(LIBPERF_ERR_NOT_SUPPORT_PER_THREAD, "perThread does not support CPU specification");
return LIBPERF_ERR_NOT_SUPPORT_PER_THREAD;
}
return SUCCESS;
}
int CheckAttr(enum PmuTaskType collectType, struct PmuAttr *attr)
{
auto err = CheckUserAccess(collectType, attr);
if (err != SUCCESS) {
return err;
}
err = CheckCpuList(attr->numCpu, attr->cpuList);
if (err != SUCCESS) {
return err;
}
err = CheckPidList(attr->numPid, attr->pidList);
if (err != SUCCESS) {
return err;
}
err = CheckEvtList(attr->numEvt, attr->evtList);
if (err != SUCCESS) {
return err;
}
err = CheckGroupList(attr->numEvtAttr, attr->evtAttr);
if (err != SUCCESS) {
return err;
}
err = CheckCollectTypeConfig(collectType, attr);
if (err != SUCCESS) {
return err;
}
if (InvalidSampleRate(collectType, attr)) {
New(LIBPERF_ERR_INVALID_SAMPLE_RATE);
return LIBPERF_ERR_INVALID_SAMPLE_RATE;
}
err = CheckBranchSampleFilter(attr->branchSampleFilter, collectType);
if (err != SUCCESS) {
New(err);
return err;
}
err = CheckCgroupNameList(attr->numCgroup, attr->cgroupNameList);
if (err != SUCCESS) {
return err;
}
err = CheckBpfMode(collectType, attr);
if (err != SUCCESS) {
return err;
}
err = CheckHwMetric(collectType, attr);
if (err != SUCCESS) {
return err;
}
err = CheckEnableOnExec(attr);
if (err != SUCCESS) {
return err;
}
err = CheckPerThread(collectType, attr);
if (err != SUCCESS) {
return err;
}
return SUCCESS;
}
static void CopyAttrData(PmuAttr* newAttr, PmuAttr* inputAttr, enum PmuTaskType collectType)
{
char **newEvtList = nullptr;
if ((inputAttr->blockedSample == 1)) {
newEvtList = new char* [2];
char* cycles = "cycles";
newEvtList[0] = new char[strlen(cycles) + 1];
strcpy(newEvtList[0], cycles);
char* cs = "context-switches";
newEvtList[1] = new char[strlen(cs) + 1];
strcpy(newEvtList[1], cs);
inputAttr->numEvt = 2;
} else if (inputAttr->numEvt > 0) {
newEvtList = new char *[inputAttr->numEvt];
for (int i = 0; i < inputAttr->numEvt; ++i) {
newEvtList[i] = new char[strlen(inputAttr->evtList[i]) + 1];
strcpy(newEvtList[i], inputAttr->evtList[i]);
}
}
newAttr->evtList = newEvtList;
newAttr->numEvt = inputAttr->numEvt;
}
static void FreeEvtList(unsigned evtNum, char** evtList)
{
if (!evtList) {
return;
}
for (int i = 0; i < evtNum; i++) {
if (evtList[i]) {
delete[] evtList[i];
evtList[i] = nullptr;
}
}
delete[] evtList;
evtList = nullptr;
}
static bool AppendChildEvents(char* evt, unordered_map<string, char*>& eventSplitMap)
{
string strName(evt);
auto findSlash = strName.find('/');
string devName = strName.substr(0, findSlash);
string evtName = strName.substr(devName.size() + 1, strName.size() - 1 - (devName.size() + 1));
auto numEvt = uncoreEventPair.first;
auto uncoreEventList = uncoreEventPair.second;
if (uncoreEventList == nullptr) {
return false;
}
bool invalidFlag = true;
for (int i = 0; i < numEvt; ++i) {
string uncoreEvent = uncoreEventList[i];
auto findUncoreSlash = uncoreEvent.find('/');
string uncoreDevName = uncoreEvent.substr(0, findUncoreSlash);
string uncoreEvtName = uncoreEvent.substr(
uncoreDevName.size() + 1, uncoreEvent.size() - 1 - (uncoreDevName.size() + 1));
if (strncmp(uncoreEvent.c_str(), devName.c_str(), devName.length()) == 0 && evtName == uncoreEvtName) {
invalidFlag = false;
eventSplitMap.emplace(uncoreEvent, evt);
}
}
if (invalidFlag) {
return false;
}
return true;
}
static bool SplitUncoreEvent(struct PmuAttr *attr, unordered_map<string, char*> &eventSplitMap)
{
char** evtList = attr->evtList;
unsigned size = attr->numEvt;
int newSize = 0;
unsigned numEvt;
auto eventList = PmuEventList(UNCORE_EVENT, &numEvt);
uncoreEventPair = make_pair(numEvt, eventList);
for (int i = 0; i < size; ++i) {
char* evt = evtList[i];
char* slashPos = std::strchr(evt, '/');
if (slashPos != nullptr && slashPos != evt) {
char* prevChar = slashPos - 1;
if (!std::isdigit(*prevChar)) {
if (!AppendChildEvents(evt, eventSplitMap)) {
eventSplitMap.emplace(evt, evt);
newSize++;
continue;
}
continue;
}
}
eventSplitMap.emplace(evt, evt);
newSize++;
}
return true;
}
static unsigned GenerateSplitList(unordered_map<string, char*>& eventSplitMap, vector<char*> &newEvtlist, const struct PmuAttr *attr, vector<struct EvtAttr> &newEvtAttrList)
{
for (int i = 0; i < attr->numEvt; ++i) {
auto evt = attr->evtList[i];
EvtAttr evtAttr = {-1, 0, false, false};
if (attr->evtAttr != nullptr && i < attr->numEvtAttr) {
auto inputAttr = attr->evtAttr[i];
evtAttr = {inputAttr.groupId, inputAttr.period, inputAttr.excludeUser, inputAttr.excludeKernel};
}
if (eventSplitMap.find(evt) != eventSplitMap.end()) {
newEvtlist.push_back(evt);
newEvtAttrList.push_back(evtAttr);
} else {
for (auto &aggregtaChildEvt : eventSplitMap) {
if (strcmp(evt, aggregtaChildEvt.second) == 0) {
newEvtlist.push_back(const_cast<char*>(aggregtaChildEvt.first.c_str()));
newEvtAttrList.push_back(evtAttr);
}
}
}
}
return newEvtlist.size();
}
static bool PdValid(const int &pd)
{
return PmuList::GetInstance()->IsPdAlive(pd);
}
void PmuTaskAttrFree(PmuTaskAttr *taskAttr)
{
auto node = taskAttr;
while (node) {
auto current = node;
node = node->next;
current->pmuEvt = nullptr;
delete current;
}
}
int PmuOpen(enum PmuTaskType collectType, struct PmuAttr *attr)
{
SetWarn(SUCCESS);
New(SUCCESS);
if (attr == nullptr) {
New(LIBPERF_ERR_NULL_POINTER, "PmuAttr cannot be null");
return -1;
}
PmuAttr copiedAttr = *attr;
pair<unsigned, char**> previousEventList = {0, nullptr};
try {
auto err = CheckAttr(collectType, attr);
if (err != SUCCESS) {
return -1;
}
CopyAttrData(&copiedAttr, attr, collectType);
previousEventList = make_pair(copiedAttr.numEvt, copiedAttr.evtList);
int pd = -1;
unordered_map<string, char*> eventSplitMap;
do {
if (!SplitUncoreEvent(&copiedAttr, eventSplitMap)) {
break;
}
vector<char *> newEvtlist;
vector<struct EvtAttr> newEvtAttrList;
auto numEvt = GenerateSplitList(eventSplitMap, newEvtlist, &copiedAttr, newEvtAttrList);
copiedAttr.numEvt = numEvt;
copiedAttr.evtList = newEvtlist.data();
copiedAttr.evtAttr = newEvtAttrList.data();
auto pTaskAttr = AssignPmuTaskParam(collectType, &copiedAttr);
if (pTaskAttr == nullptr) {
break;
}
unique_ptr<PmuTaskAttr, void (*)(PmuTaskAttr *)> taskAttr(pTaskAttr, PmuTaskAttrFree);
pd = PmuList::GetInstance()->NewPd();
if (pd == -1) {
New(LIBPERF_ERR_NO_AVAIL_PD);
break;
}
PmuList::GetInstance()->FillPidList(pd, copiedAttr.numPid, copiedAttr.pidList);
PmuList::GetInstance()->SetSymbolMode(pd, attr->symbolMode);
PmuList::GetInstance()->SetBranchSampleFilter(pd, attr->branchSampleFilter);
PmuList::GetInstance()->SetAnalysisStatus(pd, GOING_RESOLVE);
err = PmuList::GetInstance()->Register(pd, taskAttr.get());
if (err != SUCCESS) {
PmuList::GetInstance()->Close(pd);
pd = -1;
}
New(err);
} while(false);
if (pd == -1) {
FreeEvtList(previousEventList.first, previousEventList.second);
#ifdef IS_X86
if (attr->branchSampleFilter != 0) {
int driverPd = PebsDriverManager::Instance().Open(attr);
if (driverPd >= 0) {
return driverPd;
} else {
New(LIBPERF_ERR_LBR_DRIVER_INVALID, "The pebs driver is not loaded or unavailable to collect lbr.");
return -1;
}
}
#endif
return -1;
}
PmuList::GetInstance()->StoreSplitData(pd, previousEventList, eventSplitMap);
return pd;
} catch (std::bad_alloc&) {
FreeEvtList(previousEventList.first, previousEventList.second);
New(COMMON_ERR_NOMEM);
return -1;
} catch (exception& ex) {
FreeEvtList(previousEventList.first, previousEventList.second);
New(UNKNOWN_ERROR, ex.what());
return -1;
}
}
int PmuEnable(int pd)
{
SetWarn(SUCCESS);
#ifdef IS_X86
if (PebsDriverManager::IsDriverPd(pd)) {
return PebsDriverManager::Instance().Enable(pd);
}
#endif
return PmuCollectStart(pd);
}
int PmuDisable(int pd)
{
SetWarn(SUCCESS);
#ifdef IS_X86
if (PebsDriverManager::IsDriverPd(pd)) {
return PebsDriverManager::Instance().Disable(pd);
}
#endif
return PmuCollectPause(pd);
}
int PmuAppendData(struct PmuData *fromData, struct PmuData **toData)
{
SetWarn(SUCCESS);
int toLen = 0;
PmuList::GetInstance()->AppendData(fromData, toData, toLen);
return toLen;
}
static int DoCollectCounting(int pd, int milliseconds, unsigned collectInterval)
{
constexpr int usecPerMilli = 1000;
int remained = milliseconds;
bool unlimited = milliseconds == -1;
PmuCollectStart(pd);
while (remained > 0 || unlimited) {
int interval = collectInterval;
if (!unlimited && remained < collectInterval) {
interval = remained;
}
usleep(usecPerMilli * interval);
if (PmuList::GetInstance()->IsAllPidExit(pd)) {
break;
}
pdMutex.tryLock(pd);
if (!runningStatus[pd]) {
pdMutex.releaseLock(pd);
break;
}
pdMutex.releaseLock(pd);
remained -= interval;
}
PmuCollectPause(pd);
auto err = PmuList::GetInstance()->ReadDataToBuffer(pd);
if (err != SUCCESS) {
New(err);
return err;
}
return SUCCESS;
}
static int DoCollectNonCounting(int pd, int milliseconds, unsigned collectInterval)
{
constexpr int usecPerMilli = 1000;
int remained = milliseconds;
bool unlimited = milliseconds == -1;
while (remained > 0 || unlimited) {
int interval = collectInterval;
if (!unlimited && remained < collectInterval) {
interval = remained;
}
PmuCollectStart(pd);
usleep(usecPerMilli * interval);
PmuCollectPause(pd);
auto err = PmuList::GetInstance()->ReadDataToBuffer(pd);
if (err != SUCCESS) {
New(err);
return err;
}
if (PmuList::GetInstance()->AllPmuDead(pd)) {
break;
}
pdMutex.tryLock(pd);
if (!runningStatus[pd]) {
pdMutex.releaseLock(pd);
break;
}
pdMutex.releaseLock(pd);
remained -= interval;
}
return SUCCESS;
}
static int DoCollect(int pd, int milliseconds, unsigned interval)
{
if (PmuList::GetInstance()->GetTaskType(pd) == COUNTING) {
return DoCollectCounting(pd, milliseconds, interval);
}
return DoCollectNonCounting(pd, milliseconds, interval);
}
int PmuCollect(int pd, int milliseconds, unsigned interval)
{
SetWarn(SUCCESS);
int err = SUCCESS;
string errMsg = "";
try {
if (!PdValid(pd)) {
New(LIBPERF_ERR_INVALID_PD);
return -1;
}
if (milliseconds != -1 && milliseconds < 0) {
New(LIBPERF_ERR_INVALID_TIME);
return -1;
}
if (interval < 100) {
New(LIBPERF_ERR_INVALID_TIME);
return -1;
}
pdMutex.tryLock(pd);
runningStatus[pd] = true;
pdMutex.releaseLock(pd);
err = DoCollect(pd, milliseconds, interval);
} catch (std::bad_alloc&) {
err = COMMON_ERR_NOMEM;
} catch (exception& ex) {
err = UNKNOWN_ERROR;
errMsg = ex.what();
}
pdMutex.tryLock(pd);
runningStatus[pd] = false;
pdMutex.releaseLock(pd);
if (!errMsg.empty()) {
New(err, errMsg);
} else {
New(err);
}
if (err != SUCCESS) {
return -1;
}
return err;
}
static int InnerCollect(int *pds, unsigned len, size_t collectTime, bool &stop)
{
for (unsigned i = 0; i < len; ++i) {
PmuCollectStart(pds[i]);
}
usleep(collectTime);
for (unsigned i = 0; i < len; ++i) {
PmuCollectPause(pds[i]);
}
for (unsigned i = 0; i < len; ++i) {
auto err = PmuList::GetInstance()->ReadDataToBuffer(pds[i]);
if (err != SUCCESS) {
return err;
}
}
bool allDead = true;
for (unsigned i = 0; i < len; ++i) {
auto taskType = PmuList::GetInstance()->GetTaskType(pds[i]);
if (taskType == COUNTING) {
allDead = false;
break;
}
if (!PmuList::GetInstance()->AllPmuDead(pds[i])) {
allDead = false;
break;
}
}
if (allDead) {
stop = true;
return SUCCESS;
}
bool allStopped = true;
for (unsigned i = 0; i < len; ++i) {
pdMutex.tryLock(pds[i]);
if (runningStatus[pds[i]]) {
allStopped = false;
pdMutex.releaseLock(pds[i]);
break;
}
pdMutex.releaseLock(pds[i]);
}
if (allStopped) {
stop = true;
}
return SUCCESS;
}
int PmuCollectV(int *pds, unsigned len, int milliseconds)
{
SetWarn(SUCCESS);
constexpr int collectInterval = 100;
constexpr int usecPerMilli = 1000;
int remained = milliseconds;
bool unlimited = milliseconds == -1;
for (int i = 0; i < len; ++i) {
pdMutex.tryLock(pds[i]);
runningStatus[pds[i]] = true;
pdMutex.releaseLock(pds[i]);
}
while (remained > 0 || unlimited) {
int interval = collectInterval;
if (!unlimited && remained < collectInterval) {
interval = remained;
}
bool stop = false;
auto err = InnerCollect(pds, len, static_cast<size_t>(usecPerMilli * interval), stop);
if (err != SUCCESS) {
New(err);
return err;
}
if (stop) {
break;
}
remained -= interval;
}
return SUCCESS;
}
void PmuStop(int pd)
{
SetWarn(SUCCESS);
if (!PdValid(pd)) {
New(LIBPERF_ERR_INVALID_PD);
return;
}
pdMutex.tryLock(pd);
runningStatus[pd] = false;
pdMutex.releaseLock(pd);
New(SUCCESS);
}
int PmuRead(int pd, struct PmuData** pmuData)
{
SetWarn(SUCCESS);
#ifdef IS_X86
if (PebsDriverManager::IsDriverPd(pd)) {
return PebsDriverManager::Instance().Read(pd, pmuData);
}
#endif
try {
if (!PdValid(pd)) {
New(LIBPERF_ERR_INVALID_PD);
return -1;
}
New(SUCCESS);
auto& retData = KUNPENG_PMU::PmuList::GetInstance()->Read(pd);
if (!retData.empty()) {
*pmuData = retData.data();
return retData.size();
} else {
*pmuData = nullptr;
return 0;
}
} catch (std::bad_alloc&) {
New(COMMON_ERR_NOMEM);
return -1;
} catch (exception& ex) {
New(UNKNOWN_ERROR, ex.what());
return -1;
}
}
int ResolvePmuDataSymbol(struct PmuData* pmuData)
{
return PmuList::GetInstance()->ResolvePmuDataSymbol(pmuData);
}
void PmuClose(int pd)
{
SetWarn(SUCCESS);
#ifdef IS_X86
if (PebsDriverManager::IsDriverPd(pd)) {
return PebsDriverManager::Instance().Close(pd);
}
#endif
if (!PdValid(pd)) {
New(LIBPERF_ERR_INVALID_PD);
return;
}
try {
KUNPENG_PMU::PmuList::GetInstance()->Close(pd);
PmuDeviceBdfListFree();
New(SUCCESS);
} catch (std::bad_alloc&) {
New(COMMON_ERR_NOMEM);
} catch (exception& ex) {
New(UNKNOWN_ERROR, ex.what());
}
}
void PmuExit(int pd)
{
return PmuList::GetInstance()->SetAnalysisStatus(pd, STOP_RESOLVE);
}
static struct PmuEvt* GetPmuEvent(const char* pmuName, int collectType)
{
return PfmGetPmuEvent(pmuName, collectType);
}
static void PrepareCpuList(PmuAttr *attr, PmuTaskAttr *taskParam, PmuEvt* pmuEvt)
{
if (!pmuEvt->cpuMaskList.empty()) {
for (int i : pmuEvt->cpuMaskList) {
taskParam->cpuList.push_back(i);
}
taskParam->pidList.clear();
taskParam->pidList.push_back(-1);
} else if (attr->cpuList == nullptr && attr->pidList != nullptr && pmuEvt->collectType == COUNTING) {
if(attr->enableBpf) {
for(int i = 0; i < MAX_CPU_NUM; i++) {
taskParam->cpuList.push_back(i);
}
} else {
taskParam->cpuList.push_back(-1);
}
} else if (attr->cpuList == nullptr) {
if (attr->perThread) {
taskParam->cpuList.emplace_back(-1);
return;
}
const set<int> &onLineCpus = GetOnLineCpuIds();
for (const auto &cpuId : onLineCpus) {
taskParam->cpuList.push_back(cpuId);
}
} else {
for (int i = 0; i < attr->numCpu; i++) {
taskParam->cpuList.push_back(attr->cpuList[i]);
}
}
}
int GetCgroupFd(std::string& cgroupName) {
std::string cgroupPath = GetCgroupPath(cgroupName);
int cgroupFd = open(cgroupPath.c_str(), O_RDONLY);
if (cgroupFd < 0) {
New(LIBPERF_ERR_OPEN_INVALID_FILE, "open " + cgroupPath + " failed.");
return -1;
}
return cgroupFd;
}
static inline int BuildScopedGroupId(int groupId, int cgroupFd)
{
if (groupId < 0 || cgroupFd < 0) {
return groupId;
}
return ((cgroupFd & 0xffff) << 16) | (groupId & 0xffff);
}
static EvtAttr ResolveEvtAttrParams(struct PmuAttr *attr, int index) {
int groupId = index >= attr->numEvtAttr? -1 : attr->evtAttr[index].groupId;
bool excludeKernel = index >= attr->numEvtAttr ? false : attr->evtAttr[index].excludeKernel;
excludeKernel = attr->excludeKernel ? true : excludeKernel;
bool excludeUser = index >= attr->numEvtAttr ? false : attr->evtAttr[index].excludeUser;
excludeUser = attr->excludeUser ? true : excludeUser;
unsigned period = index >= attr->numEvtAttr ? SAMPLING_RECORD_PERIOD : attr->evtAttr[index].period;
period = attr->period ? attr->period : period;
if (!period) {
period = SAMPLING_RECORD_PERIOD;
}
EvtAttr evtAttrDto = {groupId, period, excludeUser, excludeKernel};
return evtAttrDto;
}
static struct PmuTaskAttr* AssignTaskParam(PmuTaskType collectType, PmuAttr *attr, const char* evtName, EvtAttr &evtAttrDto, const char* cgroupName, int cgroupFd)
{
unique_ptr<PmuTaskAttr, void (*)(PmuTaskAttr*)> taskParam(CreateNode<struct PmuTaskAttr>(), PmuTaskAttrFree);
* Assign pids to collect
*/
for (int i = 0; i < attr->numPid; i++) {
taskParam->pidList.push_back(attr->pidList[i]);
}
PmuEvt* pmuEvt = nullptr;
if (collectType == SPE_SAMPLING) {
pmuEvt = PfmGetSpeEvent(attr->dataFilter, attr->evFilter, attr->minLatency, collectType);
if (pmuEvt == nullptr) {
New(LIBPERF_ERR_SPE_UNAVAIL);
return nullptr;
}
} else {
pmuEvt = GetPmuEvent(evtName, collectType);
if (pmuEvt == nullptr) {
if (Perrorno() != SUCCESS) {
return nullptr;
}
#ifdef IS_X86
New(LIBPERF_ERR_INVALID_EVENT, "Invalid event: " + string(evtName) + ";x86 just supports core event and raw event");
#else
New(LIBPERF_ERR_INVALID_EVENT, "Invalid event: " + string(evtName));
#endif
return nullptr;
}
}
* Assign cpus to collect
*/
PrepareCpuList(attr, taskParam.get(), pmuEvt);
taskParam->groupId = BuildScopedGroupId(evtAttrDto.groupId, cgroupFd);
taskParam->pmuEvt = shared_ptr<PmuEvt>(pmuEvt, PmuEvtFree);
taskParam->pmuEvt->useFreq = attr->useFreq;
taskParam->pmuEvt->period = evtAttrDto.period;
taskParam->pmuEvt->excludeKernel = evtAttrDto.excludeKernel;
taskParam->pmuEvt->excludeUser = evtAttrDto.excludeUser;
taskParam->pmuEvt->callStack = attr->callStack;
taskParam->pmuEvt->blockedSample = attr->blockedSample;
taskParam->pmuEvt->includeNewFork = attr->includeNewFork;
taskParam->pmuEvt->cgroupFd = cgroupFd;
if (cgroupName != nullptr) {
taskParam->pmuEvt->cgroupName = cgroupName;
}
taskParam->pmuEvt->enableUserAccess = attr->enableUserAccess;
if (attr->enableUserAccess) {
taskParam->pmuEvt->config1 = REQUEST_USER_ACCESS;
}
taskParam->pmuEvt->enableHwMetric = attr->enableHwMetric;
if (attr->enableHwMetric) {
taskParam->pmuEvt->config2 = HARD_WARE_METRIC;
}
taskParam->pmuEvt->numEvent = attr->numEvt;
taskParam->pmuEvt->enableBpf = attr->enableBpf;
taskParam->pmuEvt->enableOnExec = attr->enableOnExec;
taskParam->pmuEvt->perThread = attr->perThread;
return taskParam.release();
}
bool InitCgroupFds(struct PmuAttr *attr, std::unordered_map<std::string, int>& cgroupFds) {
for (int i = 0; i < attr->numCgroup; ++i) {
std::string cgroupName = attr->cgroupNameList[i];
int fd = GetCgroupFd(cgroupName);
if (fd == -1) {
for (auto iter = cgroupFds.begin(); iter != cgroupFds.end(); iter++) {
close(iter->second);
}
return false;
}
cgroupFds[cgroupName] = fd;
}
return true;
}
struct PmuTaskAttr* AssignPmuTaskParam(enum PmuTaskType collectType, struct PmuAttr *attr)
{
struct PmuTaskAttr* taskParam = nullptr;
if (attr->numCgroup > 0) {
std::unordered_map<std::string, int> cgroupFds;
if (!InitCgroupFds(attr, cgroupFds)) {
return nullptr;
}
if (collectType == SPE_SAMPLING) {
std::string cgroupName(attr->cgroupNameList[0]);
EvtAttr evtAttrDto = {0, attr->period, attr->excludeUser, attr->excludeKernel};
taskParam = AssignTaskParam(collectType, attr, nullptr, evtAttrDto, cgroupName.c_str(), cgroupFds[cgroupName]);
return taskParam;
}
for (int i = 0; i < attr->numCgroup; ++i) {
std::string cgroupName(attr->cgroupNameList[i]);
for (int j = 0; j < attr->numEvt; ++j) {
EvtAttr evtAttrDto = ResolveEvtAttrParams(attr, j);
struct PmuTaskAttr* current = AssignTaskParam(collectType, attr, attr->evtList[j], evtAttrDto, cgroupName.c_str(), cgroupFds[cgroupName]);
if (current == nullptr) {
PmuTaskAttrFree(taskParam);
return nullptr;
}
AddTail(&taskParam, ¤t);
}
}
} else {
if (collectType == SPE_SAMPLING) {
EvtAttr evtAttrDto = {-1, attr->period, attr->excludeUser, attr->excludeKernel};
taskParam = AssignTaskParam(collectType, attr, nullptr, evtAttrDto, nullptr, -1);
return taskParam;
}
for (int i = 0; i < attr->numEvt; ++i) {
EvtAttr evtAttrDto = ResolveEvtAttrParams(attr, i);
struct PmuTaskAttr* current = AssignTaskParam(collectType, attr, attr->evtList[i], evtAttrDto, nullptr, -1);
if (current == nullptr) {
PmuTaskAttrFree(taskParam);
return nullptr;
}
AddTail(&taskParam, ¤t);
}
}
return taskParam;
}
void PmuDataFree(struct PmuData* pmuData)
{
SetWarn(SUCCESS);
PmuList::GetInstance()->FreeData(pmuData);
New(SUCCESS);
}
static void DumpStack(ofstream &out, Stack *stack, int dumpDwf)
{
if (stack->next) {
out << endl;
}
while (stack) {
if (stack->symbol) {
auto symbol = stack->symbol;
out << std::hex << symbol->addr << std::dec << " "
<< symbol->symbolName << " 0x" << std::hex
<< symbol->offset << std::dec << " "
<< symbol->module << " ";
if (dumpDwf) {
out << symbol->fileName << ":" << symbol->lineNum;
}
out << endl;
}
stack = stack->next;
}
}
int PmuDumpData(struct PmuData *pmuData, unsigned len, char *filepath, int dumpDwf)
{
SetWarn(SUCCESS);
if (filepath == nullptr) {
New(LIBPERF_ERR_NULL_POINTER, "filepath cannot be null");
return -1;
}
ofstream out(filepath, ios_base::app);
if (!out.is_open()) {
New(LIBPERF_ERR_PATH_INACCESSIBLE, "cannot access: " + string(filepath));
return -1;
}
if (pmuData == nullptr) {
New(LIBPERF_ERR_NULL_POINTER, "PmuData cannot be null");
return -1;
}
for (unsigned i = 0; i < len; ++i) {
auto &data = pmuData[i];
if (data.comm) {
out << data.comm << " ";
} else {
out << "NULL ";
}
out << data.pid << " "
<< data.tid << " "
<< data.cpu << " "
<< data.period << " ";
if (data.evt) {
out << data.evt << " ";
} else {
out << "NULL ";
}
out << data.count << " ";
if (data.ext) {
out << std::hex << data.ext->va << " "
<< data.ext->pa << " "
<< data.ext->event << " " << std::dec;
}
if (data.stack) {
DumpStack(out, data.stack, dumpDwf);
}
out << endl;
}
New(SUCCESS);
return 0;
}
int PmuGetField(struct SampleRawData *rawData, const char *fieldName, void *value, uint32_t vSize) {
#ifdef IS_X86
New(LIBPERF_ERR_INTERFACE_NOT_SUPPORT_X86);
return LIBPERF_ERR_INTERFACE_NOT_SUPPORT_X86;
#else
if (rawData == nullptr) {
New(LIBPERF_ERR_INVALID_FIELD_ARGS, "rawData cannot be nullptr.");
return LIBPERF_ERR_INVALID_FIELD_ARGS;
}
return TraceParser::ParseTraceData(rawData->data, fieldName, value, vSize);
#endif
}
struct SampleRawField *PmuGetFieldExp(struct SampleRawData *rawData, const char *fieldName) {
#ifdef IS_X86
New(LIBPERF_ERR_INTERFACE_NOT_SUPPORT_X86);
return nullptr;
#else
if (rawData == nullptr) {
New(LIBPERF_ERR_INVALID_FIELD_ARGS, "rawData cannot be nullptr.");
return nullptr;
}
SampleRawField *rt = TraceParser::GetSampleRawField(rawData->data, fieldName);
if (rt) {
New(SUCCESS);
}
return rt;
#endif
}