#include "gwp_asan/guarded_pool_allocator.h"
#include "gwp_asan/crash_handler.h"
#include "gwp_asan/options.h"
#include "gwp_asan/utilities.h"
#include "gwp_asan/stack_trace_compressor.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_stacktrace_printer.h"
#include "sanitizer_common/sanitizer_symbolizer.h"
#include "sanitizer_common/sanitizer_symbolizer_internal.h"
#include <assert.h>
#include <stddef.h>
using AllocationMetadata = gwp_asan::AllocationMetadata;
using Error = gwp_asan::Error;
namespace gwp_asan {
namespace {
GuardedPoolAllocator *SingletonPtr = nullptr;
size_t roundUpTo(size_t Size, size_t Boundary) {
return (Size + Boundary - 1) & ~(Boundary - 1);
}
uintptr_t getPageAddr(uintptr_t Ptr, uintptr_t PageSize) {
return Ptr & ~(PageSize - 1);
}
bool isPowerOfTwo(uintptr_t X) { return (X & (X - 1)) == 0; }
}
struct BufferHeader {
size_t max_slots;
size_t sample_rate;
};
struct MetadataSnapshot {
uintptr_t addr;
size_t size;
uint64_t allocation_time;
uint8_t compressed_trace[AllocationMetadata::kStackFrameStorageBytes];
size_t trace_size;
};
#if defined (__OHOS__)
uint64_t getEndTime() {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (uint64_t)(ts.tv_sec * 1000 + ts.tv_nsec / 1000000);
}
#endif
size_t collectMetadataSnapshots(AllocationMetadata *Metadata,
size_t MaxSimultaneousAllocations,
MetadataSnapshot *Snapshots,
size_t MaxCount,
uint64_t StartTime) {
size_t Count = 0;
for (size_t i = 0;
i < MaxSimultaneousAllocations && Count < MaxCount; ++i) {
const AllocationMetadata &Meta = Metadata[i];
if (Meta.Addr && !Meta.IsDeallocated && Meta.AllocationTime < StartTime) {
MetadataSnapshot &Snapshot = Snapshots[Count];
Snapshot.addr = Meta.Addr;
Snapshot.size = Meta.RequestedSize;
Snapshot.allocation_time = Meta.AllocationTime;
Snapshot.trace_size = Meta.AllocationTrace.TraceSize;
__sanitizer::internal_memcpy(
Snapshot.compressed_trace,
Meta.AllocationTrace.CompressedTrace,
Meta.AllocationTrace.TraceSize);
++Count;
}
}
return Count;
}
void writeAllocationToBuffer(const MetadataSnapshot &Snapshot,
char *EntryBaseBytes,
size_t EntrySize,
size_t Depth,
uint64_t CurrentTime) {
*reinterpret_cast<uintptr_t *>(EntryBaseBytes) = Snapshot.addr;
*reinterpret_cast<size_t *>(EntryBaseBytes + sizeof(uintptr_t)) =
Snapshot.size;
uint64_t Lifetime = CurrentTime - Snapshot.allocation_time;
*reinterpret_cast<uint64_t *>(
EntryBaseBytes + sizeof(uintptr_t) + sizeof(size_t)) = Lifetime;
uintptr_t *StackArray = reinterpret_cast<uintptr_t *>(
EntryBaseBytes + sizeof(uintptr_t) + sizeof(size_t) +
sizeof(uint64_t));
for (size_t j = 0; j < Depth; ++j) {
StackArray[j] = 0;
}
if (Snapshot.trace_size > 0) {
uintptr_t UnpackedBuffer[AllocationMetadata::kMaxTraceLengthToCollect];
__sanitizer::internal_memset(UnpackedBuffer, 0, sizeof(UnpackedBuffer));
size_t UnpackedLength = compression::unpack(
Snapshot.compressed_trace, Snapshot.trace_size, UnpackedBuffer,
AllocationMetadata::kMaxTraceLengthToCollect);
constexpr uintptr_t kInvalidPC1 = static_cast<uintptr_t>(-1);
constexpr uintptr_t kInvalidPC2 = static_cast<uintptr_t>(-2);
if (UnpackedLength > 0) {
size_t FramesToCopy =
(UnpackedLength < Depth) ? UnpackedLength : Depth;
for (size_t j = 0; j < FramesToCopy; ++j) {
if (UnpackedBuffer[j] != 0 && UnpackedBuffer[j] != kInvalidPC1 &&
UnpackedBuffer[j] != kInvalidPC2) {
StackArray[j] = UnpackedBuffer[j];
}
}
}
}
}
GuardedPoolAllocator *GuardedPoolAllocator::getSingleton() {
return SingletonPtr;
}
void GuardedPoolAllocator::init(const options::Options &Opts) {
if (!Opts.Enabled || Opts.SampleRate == 0 ||
Opts.MaxSimultaneousAllocations == 0)
return;
Check(Opts.SampleRate >= 0, "GWP-ASan Error: SampleRate is < 0.");
Check(Opts.SampleRate < (1 << 30), "GWP-ASan Error: SampleRate is >= 2^30.");
Check(Opts.MaxSimultaneousAllocations >= 0,
"GWP-ASan Error: MaxSimultaneousAllocations is < 0.");
SingletonPtr = this;
Backtrace = Opts.Backtrace;
MinSampleSize = Opts.MinSampleSize;
WhiteListPath = Opts.WhiteListPath;
State.VersionMagic = {{AllocatorVersionMagic::kAllocatorVersionMagic[0],
AllocatorVersionMagic::kAllocatorVersionMagic[1],
AllocatorVersionMagic::kAllocatorVersionMagic[2],
AllocatorVersionMagic::kAllocatorVersionMagic[3]},
AllocatorVersionMagic::kAllocatorVersion,
0};
State.MaxSimultaneousAllocations = Opts.MaxSimultaneousAllocations;
#if defined (__OHOS__)
MUSL_LOG("[gwp_asan]: SlotLength %{public}d, SampleRate %{public}d",
Opts.MaxSimultaneousAllocations, Opts.SampleRate);
#endif
const size_t PageSize = getPlatformPageSize();
assert((PageSize & (PageSize - 1)) == 0);
State.PageSize = PageSize;
size_t PoolBytesRequired =
PageSize * (2 + State.MaxSimultaneousAllocations) +
State.MaxSimultaneousAllocations * State.maximumAllocationSize();
assert(PoolBytesRequired % PageSize == 0);
void *GuardedPoolMemory = reserveGuardedPool(PoolBytesRequired);
size_t BytesRequired =
roundUpTo(State.MaxSimultaneousAllocations * sizeof(*Metadata), PageSize);
Metadata = reinterpret_cast<AllocationMetadata *>(
map(BytesRequired, kGwpAsanMetadataName));
BytesRequired = roundUpTo(
State.MaxSimultaneousAllocations * sizeof(*FreeSlots), PageSize);
FreeSlots =
reinterpret_cast<size_t *>(map(BytesRequired, kGwpAsanFreeSlotsName));
if (Opts.SampleRate != 1)
AdjustedSampleRatePlusOne = static_cast<uint32_t>(Opts.SampleRate) * 2 + 1;
else
AdjustedSampleRatePlusOne = 2;
initPRNG();
getThreadLocals()->NextSampleCounter =
((getRandomUnsigned32() % (AdjustedSampleRatePlusOne - 1)) + 1) &
ThreadLocalPackedVariables::NextSampleCounterMask;
State.GuardedPagePool = reinterpret_cast<uintptr_t>(GuardedPoolMemory);
State.GuardedPagePoolEnd =
reinterpret_cast<uintptr_t>(GuardedPoolMemory) + PoolBytesRequired;
if (Opts.InstallForkHandlers)
installAtFork();
#if defined (__OHOS__)
if (WhiteListPath && __sanitizer::internal_strlen(WhiteListPath) != 0) {
Symbolizer = __sanitizer::Symbolizer::GetOrInit();
Symbolizer->RefreshModules();
parseWhiteList();
findmodule();
}
#endif
}
void GuardedPoolAllocator::disable() {
PoolMutex.lock();
BacktraceMutex.lock();
}
void GuardedPoolAllocator::enable() {
PoolMutex.unlock();
BacktraceMutex.unlock();
}
void GuardedPoolAllocator::iterate(void *Base, size_t Size, iterate_callback Cb,
void *Arg) {
uintptr_t Start = reinterpret_cast<uintptr_t>(Base);
for (size_t i = 0; i < State.MaxSimultaneousAllocations; ++i) {
const AllocationMetadata &Meta = Metadata[i];
if (Meta.Addr && !Meta.IsDeallocated && Meta.Addr >= Start &&
Meta.Addr < Start + Size)
Cb(Meta.Addr, Meta.RequestedSize, Arg);
}
}
bool GuardedPoolAllocator::hasFreeMem()
{
if (NumSampledAllocations < State.MaxSimultaneousAllocations) {
return true;
}
if (FreeSlotsLength > 0) {
return true;
}
return false;
}
void GuardedPoolAllocator::uninitTestOnly() {
if (State.GuardedPagePool) {
unreserveGuardedPool();
State.GuardedPagePool = 0;
State.GuardedPagePoolEnd = 0;
}
if (Metadata) {
unmap(Metadata,
roundUpTo(State.MaxSimultaneousAllocations * sizeof(*Metadata),
State.PageSize));
Metadata = nullptr;
}
if (FreeSlots) {
unmap(FreeSlots,
roundUpTo(State.MaxSimultaneousAllocations * sizeof(*FreeSlots),
State.PageSize));
FreeSlots = nullptr;
}
*getThreadLocals() = ThreadLocalPackedVariables();
}
size_t GuardedPoolAllocator::getRequiredBackingSize(size_t Size,
size_t Alignment,
size_t PageSize) {
assert(isPowerOfTwo(Alignment) && "Alignment must be a power of two!");
assert(Alignment != 0 && "Alignment should be non-zero");
assert(Size != 0 && "Size should be non-zero");
if (Alignment <= PageSize)
return Size;
return Size + Alignment - PageSize;
}
uintptr_t GuardedPoolAllocator::alignUp(uintptr_t Ptr, size_t Alignment) {
assert(isPowerOfTwo(Alignment) && "Alignment must be a power of two!");
assert(Alignment != 0 && "Alignment should be non-zero");
if ((Ptr & (Alignment - 1)) == 0)
return Ptr;
Ptr += Alignment - (Ptr & (Alignment - 1));
return Ptr;
}
uintptr_t GuardedPoolAllocator::alignDown(uintptr_t Ptr, size_t Alignment) {
assert(isPowerOfTwo(Alignment) && "Alignment must be a power of two!");
assert(Alignment != 0 && "Alignment should be non-zero");
if ((Ptr & (Alignment - 1)) == 0)
return Ptr;
Ptr -= Ptr & (Alignment - 1);
return Ptr;
}
void *GuardedPoolAllocator::allocate(size_t Size, size_t Alignment) {
if (State.GuardedPagePoolEnd == 0) {
getThreadLocals()->NextSampleCounter =
(AdjustedSampleRatePlusOne - 1) &
ThreadLocalPackedVariables::NextSampleCounterMask;
return nullptr;
}
if (Size == 0)
Size = 1;
if (Alignment == 0)
Alignment = alignof(max_align_t);
if (!isPowerOfTwo(Alignment) || Alignment > State.maximumAllocationSize() ||
Size > State.maximumAllocationSize() || Size < MinSampleSize)
return nullptr;
size_t BackingSize = getRequiredBackingSize(Size, Alignment, State.PageSize);
if (BackingSize > State.maximumAllocationSize())
return nullptr;
if (getThreadLocals()->RecursiveGuard)
return nullptr;
ScopedRecursiveGuard SRG;
size_t Index;
{
ScopedLock L(PoolMutex);
Index = reserveSlot();
}
if (Index == kInvalidSlotID)
return nullptr;
uintptr_t SlotStart = State.slotToAddr(Index);
AllocationMetadata *Meta = addrToMetadata(SlotStart);
uintptr_t SlotEnd = State.slotToAddr(Index) + State.maximumAllocationSize();
uintptr_t UserPtr;
if (getRandomUnsigned32() % 2 == 0)
UserPtr = alignUp(SlotStart, Alignment);
else
UserPtr = alignDown(SlotEnd - Size, Alignment);
assert(UserPtr >= SlotStart);
assert(UserPtr + Size <= SlotEnd);
const size_t PageSize = State.PageSize;
allocateInGuardedPool(
reinterpret_cast<void *>(getPageAddr(UserPtr, PageSize)),
roundUpTo(Size, PageSize));
Meta->RecordAllocation(UserPtr, Size);
{
ScopedLock UL(BacktraceMutex);
Meta->AllocationTrace.RecordBacktrace(Backtrace);
}
return reinterpret_cast<void *>(UserPtr);
}
void GuardedPoolAllocator::raiseInternallyDetectedError(uintptr_t Address,
Error E) {
disable();
State.FailureType = E;
State.FailureAddress = Address;
volatile char *p =
reinterpret_cast<char *>(State.internallyDetectedErrorFaultAddress());
*p = 0;
assert(State.FailureType == Error::UNKNOWN);
assert(State.FailureAddress == 0u);
deallocateInGuardedPool(
reinterpret_cast<void *>(getPageAddr(
State.internallyDetectedErrorFaultAddress(), State.PageSize)),
State.PageSize);
enable();
}
void GuardedPoolAllocator::deallocate(void *Ptr) {
assert(pointerIsMine(Ptr) && "Pointer is not mine!");
uintptr_t UPtr = reinterpret_cast<uintptr_t>(Ptr);
size_t Slot = State.getNearestSlot(UPtr);
uintptr_t SlotStart = State.slotToAddr(Slot);
AllocationMetadata *Meta = addrToMetadata(UPtr);
if (Meta->HasCrashed)
return;
if (Meta->Addr != UPtr) {
raiseInternallyDetectedError(UPtr, Error::INVALID_FREE);
return;
}
if (Meta->IsDeallocated) {
raiseInternallyDetectedError(UPtr, Error::DOUBLE_FREE);
return;
}
{
ScopedLock L(PoolMutex);
Meta->RecordDeallocation();
if (!getThreadLocals()->RecursiveGuard) {
ScopedRecursiveGuard SRG;
ScopedLock UL(BacktraceMutex);
Meta->DeallocationTrace.RecordBacktrace(Backtrace);
}
}
deallocateInGuardedPool(reinterpret_cast<void *>(SlotStart),
State.maximumAllocationSize());
ScopedLock L(PoolMutex);
freeSlot(Slot);
}
static bool PreviousRecursiveGuard;
void GuardedPoolAllocator::preCrashReport(void *Ptr) {
assert(pointerIsMine(Ptr) && "Pointer is not mine!");
uintptr_t InternalCrashAddr = __gwp_asan_get_internal_crash_address(
&State, reinterpret_cast<uintptr_t>(Ptr));
if (!InternalCrashAddr)
disable();
PreviousRecursiveGuard = getThreadLocals()->RecursiveGuard;
getThreadLocals()->RecursiveGuard = true;
}
void GuardedPoolAllocator::postCrashReportRecoverableOnly(void *SignalPtr) {
uintptr_t SignalUPtr = reinterpret_cast<uintptr_t>(SignalPtr);
uintptr_t InternalCrashAddr =
__gwp_asan_get_internal_crash_address(&State, SignalUPtr);
uintptr_t ErrorUptr = InternalCrashAddr ?: SignalUPtr;
AllocationMetadata *Metadata = addrToMetadata(ErrorUptr);
Metadata->HasCrashed = true;
allocateInGuardedPool(
reinterpret_cast<void *>(getPageAddr(SignalUPtr, State.PageSize)),
State.PageSize);
if (InternalCrashAddr) {
State.FailureType = Error::UNKNOWN;
State.FailureAddress = 0;
}
size_t Slot = State.getNearestSlot(ErrorUptr);
for (size_t i = 0; i < FreeSlotsLength; ++i) {
if (FreeSlots[i] == Slot) {
FreeSlots[i] = FreeSlots[FreeSlotsLength - 1];
FreeSlotsLength -= 1;
break;
}
}
getThreadLocals()->RecursiveGuard = PreviousRecursiveGuard;
if (!InternalCrashAddr)
enable();
}
size_t GuardedPoolAllocator::getSize(const void *Ptr) {
assert(pointerIsMine(Ptr));
ScopedLock L(PoolMutex);
AllocationMetadata *Meta = addrToMetadata(reinterpret_cast<uintptr_t>(Ptr));
assert(Meta->Addr == reinterpret_cast<uintptr_t>(Ptr));
return Meta->RequestedSize;
}
AllocationMetadata *GuardedPoolAllocator::addrToMetadata(uintptr_t Ptr) const {
return &Metadata[State.getNearestSlot(Ptr)];
}
size_t GuardedPoolAllocator::reserveSlot() {
#if defined (__OHOS__)
accumulatePersistInterval(NumSampledAllocations - FreeSlotsLength);
#endif
if (NumSampledAllocations < State.MaxSimultaneousAllocations) {
#if defined (__OHOS__)
++ReserveCounter;
#endif
return NumSampledAllocations++;
}
if (FreeSlotsLength == 0)
return kInvalidSlotID;
size_t ReservedIndex = getRandomUnsigned32() % FreeSlotsLength;
size_t SlotIndex = FreeSlots[ReservedIndex];
#if defined (__OHOS__)
++ReserveCounter;
#endif
FreeSlots[ReservedIndex] = FreeSlots[--FreeSlotsLength];
return SlotIndex;
}
void GuardedPoolAllocator::freeSlot(size_t SlotIndex) {
#if defined (__OHOS__)
accumulatePersistInterval(NumSampledAllocations - FreeSlotsLength);
#endif
assert(FreeSlotsLength < State.MaxSimultaneousAllocations);
FreeSlots[FreeSlotsLength++] = SlotIndex;
}
uint32_t GuardedPoolAllocator::getRandomUnsigned32() {
uint32_t RandomState = getThreadLocals()->RandomState;
RandomState ^= RandomState << 13;
RandomState ^= RandomState >> 17;
RandomState ^= RandomState << 5;
getThreadLocals()->RandomState = RandomState;
return RandomState;
}
void GuardedPoolAllocator::accumulatePersistInterval(size_t reservedSlotsLength) {
assert(reservedSlotsLength >= 0);
size_t curTime = __sanitizer::NanoTime() / 1000;
if (NumSampledAllocations == 0) {
PreTime = curTime;
return;
}
PersistInterval += (curTime - PreTime) * reservedSlotsLength;
PreTime = curTime;
};
size_t GuardedPoolAllocator::collectAllocationsByTimeRange(
uint64_t Timespan, uintptr_t *Buffer, size_t MaxCount, size_t Depth) {
if (!Buffer || MaxCount == 0)
return 0;
#if defined (__OHOS__)
uint64_t EndTime = getEndTime();
#else
uint64_t EndTime = getCoarseTimeMs();
#endif
if (Depth == 0)
Depth = 1;
size_t HeaderSize = sizeof(BufferHeader);
size_t EntrySize = sizeof(uintptr_t) + sizeof(size_t) + sizeof(uint64_t) +
Depth * sizeof(uintptr_t);
BufferHeader *Header = reinterpret_cast<BufferHeader *>(Buffer);
Header->max_slots = State.MaxSimultaneousAllocations;
if (AdjustedSampleRatePlusOne == 0) {
Header->sample_rate = 0;
} else if (AdjustedSampleRatePlusOne == 2) {
Header->sample_rate = 1;
} else {
Header->sample_rate = (AdjustedSampleRatePlusOne - 1) / 2;
}
uint64_t TimespanMilliseconds = Timespan * 1000;
uint64_t StartTime = (EndTime > TimespanMilliseconds)
? (EndTime - TimespanMilliseconds)
: 0;
size_t SnapshotsSize = MaxCount * sizeof(MetadataSnapshot);
MetadataSnapshot *Snapshots = static_cast<MetadataSnapshot *>(
__sanitizer::MmapOrDie(SnapshotsSize, "MetadataSnapshot"));
if (!Snapshots)
return 0;
size_t Count = collectMetadataSnapshots(
Metadata, State.MaxSimultaneousAllocations, Snapshots,
MaxCount, StartTime);
char *DataBase = reinterpret_cast<char *>(Buffer) + HeaderSize;
for (size_t i = 0; i < Count; ++i) {
char *EntryBaseBytes = DataBase + (i * EntrySize);
writeAllocationToBuffer(Snapshots[i], EntryBaseBytes, EntrySize, Depth,
EndTime);
}
__sanitizer::UnmapOrDie(Snapshots, SnapshotsSize);
return Count;
}
#if defined (__OHOS__)
bool GuardedPoolAllocator::checkLib() {
if (LibraryPathLength == 0 && ModuleLength == 0)
return false;
static constexpr unsigned kMaximumStackFramesForCrashTrace = 512;
uintptr_t Trace[kMaximumStackFramesForCrashTrace];
size_t TraceLength = Backtrace(Trace, kMaximumStackFramesForCrashTrace);
uintptr_t pc;
for (int i = 0; i < TraceLength; ++i) {
pc = Trace[i];
for (int j = 0; j < ModuleLength; ++j) {
if (Modules[j]->containsAddress(pc))
return true;
}
if (LibraryPathLength == 0)
continue;
if (Symbolizer->GetModulesFresh())
continue;
{
ScopedLock l(FindModMutex);
Symbolizer->RefreshModules();
findmodule();
}
for (int j = 0; j < ModuleLength; ++j) {
if (Modules[j]->containsAddress(pc))
return true;
}
}
return false;
}
void GuardedPoolAllocator::parseWhiteList() {
int Num_colons = 1;
for (const char* p = WhiteListPath; *p != '\0'; ++p) {
if (*p == ':') ++Num_colons;
}
size_t BytesRequired = roundUpTo(
Num_colons * sizeof(*LibraryPath), State.PageSize);
LibraryPath =
reinterpret_cast<char **>(map(BytesRequired, "GWP-ASan Checked LibraryPath"));
BytesRequired = roundUpTo(
Num_colons * sizeof(*Modules), State.PageSize);
Modules =
reinterpret_cast<const __sanitizer::LoadedModule **>(map(BytesRequired, "GWP-ASan Checked Module"));
const char *Start = WhiteListPath;
for (int i = 0; i < Num_colons; ++i) {
const char *End = Start;
while(*End != ':' && *End != '\0') ++End;
int Len = End - Start;
BytesRequired = roundUpTo((Len + 1)*sizeof(char), State.PageSize);
LibraryPath[i] = reinterpret_cast<char *>(map(BytesRequired, "GWP-ASan Checked Library"));
__sanitizer::internal_strncpy(LibraryPath[i], Start, Len);
LibraryPath[i][Len] = '\0';
Start = End + 1;
}
LibraryPathLength = Num_colons;
}
void GuardedPoolAllocator::findmodule() {
int Index = 0;
while (Index < LibraryPathLength) {
if (auto *Module = Symbolizer->FindLibraryByName(LibraryPath[Index])) {
Modules[ModuleLength++] = Module;
LibraryPath[Index] = LibraryPath[--LibraryPathLength];
} else {
Index++;
}
}
}
#endif
}