已开启
fix: synchronize the demangle code across the compiler/runtime/tools repositories #1273
fix: synchronize the demangle code across the compiler/runtime/tools repositories #1273
已开启
余敲鱼创建于 20 天前
共 4 个文件变更+177-117
@@ -31,7 +31,7 @@ const std::unordered_map<char, size_t> base62Chars = {{'0', 0}, {'1', 1}, {'2',
31 {'u', 56}, {'v', 57}, {'w', 58}, {'x', 59}, {'y', 60}, {'z', 61}};31 {'u', 56}, {'v', 57}, {'w', 58}, {'x', 59}, {'y', 60}, {'z', 61}};
32 32 
33const char PRIMITIVE_PREFIX_SET[] = "nucbfdasilqhtjmrDv";33const char PRIMITIVE_PREFIX_SET[] = "nucbfdasilqhtjmrDv";
34-constexpr size_t PRIMITIVE_LEN = 18;34+constexpr size_t PRIMITIVE_LEN = (sizeof(PRIMITIVE_PREFIX_SET) / sizeof(PRIMITIVE_PREFIX_SET[0])) - 1;
35} // namespace35} // namespace
36 36 
37namespace Cangjie {37namespace Cangjie {
@@ -51,7 +51,7 @@ inline size_t StripCangjiePrefix(T& identifier, size_t idx)
51{51{
52 if (idx + MANGLE_CHAR_LEN < identifier.Length() && identifier[idx] == MANGLE_UNDERSCORE_PREFIX &&52 if (idx + MANGLE_CHAR_LEN < identifier.Length() && identifier[idx] == MANGLE_UNDERSCORE_PREFIX &&
53 identifier[idx + MANGLE_CHAR_LEN] == 'C') {53 identifier[idx + MANGLE_CHAR_LEN] == 'C') {
54- if(idx + MANGLE_CHAR_LEN * 2 < identifier.Length() && identifier[idx + MANGLE_CHAR_LEN * 2] == 'V') {54+ if (idx + MANGLE_CHAR_LEN * 2 < identifier.Length() && identifier[idx + MANGLE_CHAR_LEN * 2] == 'V') {
55 return idx + PREFIX_LEN + MANGLE_CHAR_LEN;55 return idx + PREFIX_LEN + MANGLE_CHAR_LEN;
56 } else {56 } else {
57 return idx + PREFIX_LEN;57 return idx + PREFIX_LEN;
@@ -70,8 +70,8 @@ inline size_t GetNumber(T base62)
70 return 0;70 return 0;
71 }71 }
72 T newBase62 = base62.SubStr(0, base62.Length() - 1);72 T newBase62 = base62.SubStr(0, base62.Length() - 1);
73- long long decimal = 0;73+ size_t decimal = 0;
74- long long power = 1;74+ size_t power = 1;
75 for (size_t i = newBase62.Length() - 1; i != NPOS; i--) {75 for (size_t i = newBase62.Length() - 1; i != NPOS; i--) {
76 decimal += base62Chars.at(newBase62[i]) * power;76 decimal += base62Chars.at(newBase62[i]) * power;
77 power *= n;77 power *= n;
@@ -94,7 +94,11 @@ bool DeCompression<T>::IsGlobalEncode(T& mangled)
94template<typename T>94template<typename T>
95bool DeCompression<T>::IsSamePrefix(T& first, T second, size_t idx)95bool DeCompression<T>::IsSamePrefix(T& first, T second, size_t idx)
96{96{
97- for (size_t i = idx; i < idx + PREFIX_LEN; i++) {97+ size_t endIdx = idx + PREFIX_LEN;
98+ if (endIdx > first.Length() || endIdx > second.Length()) {
wangyinqiang
wangyinqiangwangyinqiang12 天前

[minor] IsSamePrefix 用同一个 idx 同时索引 first 和 second

问题:second 恒为长度 2 的字面量("_C"/"GV"/"GP"/"GF"/"PI"),但循环 for (i = idx; i < endIdx; i++) 用同一个 i 去索引 first 和 second。正确语义应是 first[idx + i] 对 second[i]。

影响:调用点 CJMangledDeCompression 传入的 idx 是 StripCangjieAt 的返回值,'@' 开头时为 1,此时本次新增的 endIdx > second.Length() 会直接短路成 false。改动前是 first[1] 对 second[1]('_' vs 'C')同样得 false,所以本次没有引入回归,但也没有修正错位——只是把失败原因从「错位比较」换成了「长度短路」。

建议:改为 first[idx + i] != second[i],长度校验对 first 用 idx + PREFIX_LEN、对 second 用 PREFIX_LEN。另需说明:即使这样修好,公开 API Cangjie::Demangle() 上 '@' 开头的名字仍然出不来结果——Demangler 入口的 SkipString(MANGLE_CANGJIE_PREFIX) 要求 "_C" 落在 currentIndex = 0,'@' 从未被剥离。所以验证只能在 DeCompression::CJMangledDeCompression 这一层加白盒用例;若要端到端生效,需另行在 Demangler 入口剥 '@'。

likedislike
99+ return false;
100+ }
101+ for (size_t i = idx; i < endIdx; i++) {
98 if (first[i] != second[i]) {102 if (first[i] != second[i]) {
99 return false;103 return false;
100 }104 }
@@ -163,6 +167,9 @@ bool DeCompression<T>::IsVarDeclEncode(T& mangled)
163{167{
164 this->isRecord = false;168 this->isRecord = false;
165 const size_t n = mangled.Length();169 const size_t n = mangled.Length();
170+ if (n == 0) {
171+ return false;
172+ }
166 size_t idx = 0;173 size_t idx = 0;
167 if (mangled[0] == END) {174 if (mangled[0] == END) {
168 return true;175 return true;
@@ -258,6 +265,9 @@ bool DeCompression<T>::HasDuplicates(T& mangled, size_t mid)
258{265{
259 // The subscript indexes of the treeIdMap are as follows:266 // The subscript indexes of the treeIdMap are as follows:
260 // { 0: index, 1: pos }267 // { 0: index, 1: pos }
268+ if (mid >= this->treeIdMap.size()) {
269+ return false;
270+ }
261 T curStr = mangled.SubStr(std::get<0>(treeIdMap[mid]), std::get<1>(treeIdMap[mid]));271 T curStr = mangled.SubStr(std::get<0>(treeIdMap[mid]), std::get<1>(treeIdMap[mid]));
262 size_t tid = 0;272 size_t tid = 0;
263 for (auto it = treeIdMap.begin(); it != treeIdMap.end(); ++it, ++tid) {273 for (auto it = treeIdMap.begin(); it != treeIdMap.end(); ++it, ++tid) {
@@ -295,7 +305,7 @@ inline size_t ForwardFileNameNumber(T& mangled, size_t idx)
295 curIdx++;305 curIdx++;
296 }306 }
297 if (isValid) {307 if (isValid) {
298- for (int i = idx; i <= static_cast<int>(curIdx); i++) {308+ for (size_t i = idx; i <= curIdx; i++) {
299 if (!isalnum(mangled[i])) {309 if (!isalnum(mangled[i])) {
300 return idx;310 return idx;
301 }311 }
@@ -337,7 +347,7 @@ size_t DeCompression<T>::ForwardGenericTypes(T& mangled, size_t& cnt, size_t idx
337{347{
338 size_t curCnt = cnt;348 size_t curCnt = cnt;
339 size_t curIdx = ForwardTypes(mangled, cnt, idx + MANGLE_CHAR_LEN);349 size_t curIdx = ForwardTypes(mangled, cnt, idx + MANGLE_CHAR_LEN);
340- if (curIdx != idx && mangled[curIdx] == END) {350+ if (curIdx < mangled.Length() && mangled[curIdx] == END) {
341 return curIdx + MANGLE_CHAR_LEN;351 return curIdx + MANGLE_CHAR_LEN;
342 }352 }
343 if (this->isRecord) {353 if (this->isRecord) {
@@ -356,8 +366,8 @@ size_t DeCompression<T>::ForwardClassType(T& mangled, size_t& cnt, size_t idx)
356 size_t curIdx = idx;366 size_t curIdx = idx;
357 size_t curCnt = cnt;367 size_t curCnt = cnt;
358 // Skip 'CN' 'RN' 'NN'368 // Skip 'CN' 'RN' 'NN'
359- if (idx + PREFIX_LEN < mangled.Length() && (mangled[idx] == 'C' || mangled[idx] == 'R' || mangled[idx] == 'N')369+ if (idx + PREFIX_LEN < mangled.Length() && (mangled[idx] == 'C' || mangled[idx] == 'R' || mangled[idx] == 'N') &&
360- && mangled[idx + MANGLE_CHAR_LEN] == MANGLE_NESTED_PREFIX[0]) {370+ mangled[idx + MANGLE_CHAR_LEN] == MANGLE_NESTED_PREFIX[0]) {
361 curIdx += PREFIX_LEN;371 curIdx += PREFIX_LEN;
362 } else {372 } else {
363 return idx;373 return idx;
@@ -397,7 +407,7 @@ size_t DeCompression<T>::ForwardFunctionType(T& mangled, size_t& cnt, size_t idx
397 size_t curCnt = cnt;407 size_t curCnt = cnt;
398 size_t curIdx = ForwardType(mangled, cnt, idx + PREFIX_LEN);408 size_t curIdx = ForwardType(mangled, cnt, idx + PREFIX_LEN);
399 curIdx = ForwardTypes(mangled, cnt, curIdx);409 curIdx = ForwardTypes(mangled, cnt, curIdx);
400- if (curIdx != idx && mangled[curIdx] == END) {410+ if (curIdx < mangled.Length() && mangled[curIdx] == END) {
401 return curIdx + MANGLE_CHAR_LEN;411 return curIdx + MANGLE_CHAR_LEN;
402 }412 }
403 if (this->isRecord) {413 if (this->isRecord) {
@@ -438,15 +448,8 @@ size_t DeCompression<T>::ForwardTupleType(T& mangled, size_t& cnt, size_t idx)
438template<typename T>448template<typename T>
439size_t DeCompression<T>::ForwardCPointer(T& mangled, size_t& cnt, size_t idx)449size_t DeCompression<T>::ForwardCPointer(T& mangled, size_t& cnt, size_t idx)
440{450{
441- size_t curCnt = cnt;
442 size_t curIdx = ForwardType(mangled, cnt, idx + MANGLE_CHAR_LEN);451 size_t curIdx = ForwardType(mangled, cnt, idx + MANGLE_CHAR_LEN);
443- if (curIdx != idx) {452+ return curIdx;
444- return curIdx;
445- }
446- if (this->isRecord) {
447- TreeIdMapPop(cnt, curCnt);
448- }
449- return idx;
450}453}
451 454 
452/**455/**
@@ -476,7 +479,7 @@ size_t DeCompression<T>::ForwardArrayType(T& mangled, size_t& cnt, size_t idx)
476 479 
477// Forward generic type, G<identifier-length><identifier>.480// Forward generic type, G<identifier-length><identifier>.
478template<typename T>481template<typename T>
479-size_t DeCompression<T>::ForwardGenericType(T& mangled, size_t& cnt, size_t idx)482+size_t DeCompression<T>::ForwardGenericType(T& mangled, [[maybe_unused]] size_t& cnt, size_t idx)
480{483{
481 size_t nextIdx = ForwardNumber(mangled, idx + MANGLE_CHAR_LEN);484 size_t nextIdx = ForwardNumber(mangled, idx + MANGLE_CHAR_LEN);
482 if (nextIdx != idx + MANGLE_CHAR_LEN) {485 if (nextIdx != idx + MANGLE_CHAR_LEN) {
@@ -518,10 +521,13 @@ size_t DeCompression<T>::ForwardName(T& mangled, size_t idx)
518 }521 }
519 if (mangled[idx] == MANGLE_ANONYMOUS_PREFIX) { return idx + MANGLE_CHAR_LEN; }522 if (mangled[idx] == MANGLE_ANONYMOUS_PREFIX) { return idx + MANGLE_CHAR_LEN; }
520 size_t numberLen = 0;523 size_t numberLen = 0;
521- while (idx < mangled.Length() && isdigit(mangled[idx + numberLen])) {524+ while (idx + numberLen < mangled.Length() && isdigit(mangled[idx + numberLen])) {
522 numberLen++;525 numberLen++;
523 }526 }
524 size_t number = atoi(mangled.SubStr(idx, numberLen).Str());527 size_t number = atoi(mangled.SubStr(idx, numberLen).Str());
wangyinqiangyanjs
wangyinqiangwangyinqiang12 天前

[major] atoi 没跟上 compiler 的溢出防护

问题:本行的 numberLen 由上一行的 while 循环得到,长度无上限;超出 int 范围时 atoi 是未定义行为。compiler 侧同一函数已改用 std::stoll + uint32_t 范围检查;Demangler.cpp 的 DemangleLength 也是同样情况(compiler 已改,本仓仍是 return atoi(numStr.Str());)。

影响:紧接着新增的 idx + numberLen + number > mangled.Length() 校验正是建立在这个 atoi 结果上——在最需要它的畸形输入上反而不可靠。这也是本次「三仓一致」目标上最实质的一处未达成。

建议:本仓若未禁用异常,可直接采用 compiler 的 std::stoll + 范围检查方案;否则用无异常的等价实现(手工逐位累加并检查溢出,或 std::from_chars)。注意 runtime 侧是 -fno-exceptions,三仓在这一处未必能写成同一份代码,若决定保留差异请在 PR 描述里写明理由。

likedislike
yanjsyanjs11 天前

Bug
⚠️ Warning

atoi has no overflow protection for unbounded digit run

numberLen comes from a digit run whose length is unbounded, and atoi(mangled.SubStr(idx, numberLen).Str()) is undefined behavior when the value exceeds int range. The new bounds check on the next line (idx + numberLen + number > mangled.Length()) is built on this result, so it is unreliable exactly on the malformed inputs it is meant to guard. Note the companion compiler PR converted the equivalent function to std::stoll with a uint32_t range check; DemangleLength in Demangler.cpp has the same atoi pattern. If this repo is built with -fno-exceptions (as the runtime is), use the exception-free equivalent — manual digit accumulation with an overflow check, or std::from_chars — and document the intentional difference from the compiler side.


Prompt for AI Agents
In third_party/demangler/DeCompression.cpp in `ForwardName`, replace the `atoi` call for `number` with an overflow-safe conversion: either use `std::stoll` with an explicit range check against `uint32_t` max (guarded by `__cpp_exceptions`), or the exception-free equivalent using `std::from_chars` / manual digit accumulation with an overflow check, returning `idx` when the value does not fit. Apply the same fix to the `atoi` usage in `DemangleLength` in Demangler.cpp so the three repos agree.

Not useful? Reply @yanjs reject to dismiss this suggestion.

likedislike
528+ if (idx + numberLen + number > mangled.Length()) {
529+ return idx;
530+ }
525 return idx + numberLen + number;531 return idx + numberLen + number;
526}532}
527 533 
@@ -564,6 +570,9 @@ void DeCompression<T>::TreeIdMapAssign(T& mangled, T& mangledCopy, size_t mapId,
564 // { 0: idx, 1: nextIdx }570 // { 0: idx, 1: nextIdx }
565 size_t idx = std::get<0>(eleInfo);571 size_t idx = std::get<0>(eleInfo);
566 size_t nextIdx = std::get<1>(eleInfo);572 size_t nextIdx = std::get<1>(eleInfo);
573+ if (mapId >= this->treeIdMap.size()) {
574+ return;
575+ }
567 std::get<1>(treeIdMap[mapId]) = nextIdx - idx;576 std::get<1>(treeIdMap[mapId]) = nextIdx - idx;
568 if (HasDuplicates(mangled, mapId)) {577 if (HasDuplicates(mangled, mapId)) {
569 TreeIdMapErase(mangledCopy, cnt, mapId, idx);578 TreeIdMapErase(mangledCopy, cnt, mapId, idx);
@@ -571,13 +580,73 @@ void DeCompression<T>::TreeIdMapAssign(T& mangled, T& mangledCopy, size_t mapId,
571 }580 }
572}581}
573 582 
583+// Forward compound type encodes: class ('CN'/'RN'/'NN'), function (F0/FC),
584+// tuple ('T'), C pointer ('P'), array ('A'/'V') and generic ('G').
585+// Returns idx when the encode is unknown.
586+template<typename T>
587+size_t DeCompression<T>::ForwardCompoundType(T& mangled, size_t& cnt, size_t idx)
588+{
589+ switch (mangled[idx]) {
590+ case 'C':
591+ case 'R':
592+ case 'N':
593+ return ForwardClassType(mangled, cnt, idx);
594+ case 'F': {
595+ // The function type prefix is F0/FC.
596+ if (idx + MANGLE_CHAR_LEN < mangled.Length() &&
597+ (mangled[idx + MANGLE_CHAR_LEN] == '0' || mangled[idx + MANGLE_CHAR_LEN] == 'C')) {
598+ return ForwardFunctionType(mangled, cnt, idx);
599+ }
600+ return idx;
601+ }
602+ case 'T':
603+ return ForwardTupleType(mangled, cnt, idx);
604+ case 'P':
605+ return ForwardCPointer(mangled, cnt, idx);
606+ case 'A':
607+ case 'V':
608+ return ForwardArrayType(mangled, cnt, idx);
609+ case 'G':
610+ return ForwardGenericType(mangled, cnt, idx);
611+ default:
612+ return idx;
613+ }
614+}
615+ 
616+// Forward compressed type reference, Y<number>. The reference is replaced by the span
617+// recorded in treeIdMap, and no span is recorded for the reference itself.
618+template<typename T>
619+size_t DeCompression<T>::ForwardCompressedType(T& mangled, size_t& cnt, size_t curCnt, size_t idx)
620+{
621+ size_t nextIdx = ForwardNumber(mangled, idx + MANGLE_CHAR_LEN);
622+ std::tuple<size_t, size_t> updateRes{0, nextIdx};
623+ if (nextIdx != idx) {
624+ updateRes = UpdateCompressedName(mangled, idx);
625+ }
626+ if (this->isRecord) {
627+ TreeIdMapPop(cnt, curCnt);
628+ }
629+ return std::get<0>(updateRes) == 0 ? idx : std::get<1>(updateRes);
630+}
631+ 
632+// Pop the placeholder record pushed for the current type and return the forwarding
633+// result. Used by the encodes that record no span, e.g. 'k' and primitive types.
634+template<typename T>
635+size_t DeCompression<T>::PopRecordedType(size_t& cnt, size_t curCnt, size_t nextIdx)
636+{
637+ if (this->isRecord) {
638+ TreeIdMapPop(cnt, curCnt);
639+ }
640+ return nextIdx;
641+}
642+ 
574template<typename T>643template<typename T>
575size_t DeCompression<T>::ForwardType(T& mangled, size_t& cnt, size_t idx)644size_t DeCompression<T>::ForwardType(T& mangled, size_t& cnt, size_t idx)
576{645{
577- char ch = mangled[idx];646+ if (idx >= mangled.Length()) {
578- size_t nextIdx = idx;647+ return idx;
648+ }
579 size_t curCnt = cnt;649 size_t curCnt = cnt;
580- T mangledCopy = mangled;
581 // Record current treeIdMap's index after treeIdMap pushed type650 // Record current treeIdMap's index after treeIdMap pushed type
582 size_t mapId = this->treeIdMap.size();651 size_t mapId = this->treeIdMap.size();
583 if (this->isRecord) {652 if (this->isRecord) {
@@ -585,66 +654,20 @@ size_t DeCompression<T>::ForwardType(T& mangled, size_t& cnt, size_t idx)
585 bool isPush = TreeIdMapPushBack(mangled, pos);654 bool isPush = TreeIdMapPushBack(mangled, pos);
586 cnt = isPush ? cnt + 1 : cnt;655 cnt = isPush ? cnt + 1 : cnt;
587 }656 }
588- switch (ch) {657+ char ch = mangled[idx];
589- case 'C':658+ // The CString ('k'), compressed ('Y') and primitive type encodes record no span:
590- case 'R':659+ // they pop the placeholder record and return directly.
591- case 'N':660+ if (ch == 'k') {
592- nextIdx = ForwardClassType(mangled, cnt, idx);661+ return PopRecordedType(cnt, curCnt, idx + MANGLE_CHAR_LEN);
593- break;
594- case 'F': {
595- // The function type prefix is F0/FC.
596- if (idx + MANGLE_CHAR_LEN < mangled.Length() &&
597- (mangled[idx + MANGLE_CHAR_LEN] == '0' || mangled[idx + MANGLE_CHAR_LEN] == 'C')) {
598- nextIdx = ForwardFunctionType(mangled, cnt, idx);
599- } else {
600- nextIdx = idx;
601- }
602- break;
603- }
604- case 'T':
605- nextIdx = ForwardTupleType(mangled, cnt, idx);
606- break;
607- case 'P':
608- nextIdx = ForwardCPointer(mangled, cnt, idx);
609- break;
610- case 'A':
611- case 'V':
612- nextIdx = ForwardArrayType(mangled, cnt, idx);
613- break;
614- case 'k': {
615- nextIdx = idx + MANGLE_CHAR_LEN;
616- if (this->isRecord) {
617- TreeIdMapPop(cnt, curCnt);
618- }
619- return nextIdx;
620- }
621- case 'G':
622- nextIdx = ForwardGenericType(mangled, cnt, idx);
623- break;
624- case 'Y': {
625- nextIdx = ForwardNumber(mangled, idx + MANGLE_CHAR_LEN);
626- std::tuple<size_t, size_t> updateRes{0, nextIdx};
627- if (nextIdx != idx) {
628- updateRes = UpdateCompressedName(mangled, idx);
629- }
630- if (this->isRecord) {
631- TreeIdMapPop(cnt, curCnt);
632- }
633- return std::get<0>(updateRes) == 0 ? idx : std::get<1>(updateRes);
634- }
635- default: {
636- if (IsPrimitiveType(ch)) {
637- nextIdx = ForwardPrimitiveType(mangled, idx);
638- if (this->isRecord) {
639- TreeIdMapPop(cnt, curCnt);
640- }
641- return nextIdx;
642- } else {
643- nextIdx = idx;
644- }
645- break;
646- }
647 }662 }
663+ if (ch == MANGLE_COMPRESS_PREFIX) {
664+ return ForwardCompressedType(mangled, cnt, curCnt, idx);
665+ }
666+ if (IsPrimitiveType(ch)) {
667+ return PopRecordedType(cnt, curCnt, ForwardPrimitiveType(mangled, idx));
668+ }
669+ T mangledCopy = mangled;
670+ size_t nextIdx = ForwardCompoundType(mangled, cnt, idx);
648 if (this->isRecord && nextIdx != idx) {671 if (this->isRecord && nextIdx != idx) {
649 std::tuple<size_t, size_t> eleInfo(idx, nextIdx);672 std::tuple<size_t, size_t> eleInfo(idx, nextIdx);
650 TreeIdMapAssign(mangled, mangledCopy, mapId, cnt, eleInfo);673 TreeIdMapAssign(mangled, mangledCopy, mapId, cnt, eleInfo);
@@ -655,12 +678,9 @@ size_t DeCompression<T>::ForwardType(T& mangled, size_t& cnt, size_t idx)
655}678}
656 679 
657template<typename T>680template<typename T>
658-size_t DeCompression<T>::ForwardTypes(T& mangled, size_t& cnt, size_t startId)681+size_t DeCompression<T>::ForwardTypes(T& mangled, size_t& cnt, size_t startIdx)
659{682{
660- if (mangled.Length() - startId <= 0) {683+ size_t idx = startIdx;
661- return startId;
662- }
663- size_t idx = startId;
664 while (idx < mangled.Length() && mangled[idx] != END) {684 while (idx < mangled.Length() && mangled[idx] != END) {
665 size_t curIdx = ForwardType(mangled, cnt, idx);685 size_t curIdx = ForwardType(mangled, cnt, idx);
666 if (curIdx == idx) {686 if (curIdx == idx) {
@@ -678,7 +698,7 @@ bool DeCompression<T>::IsCompressed(T& mangled)
678 return false;698 return false;
679 }699 }
680 size_t pos = 0;700 size_t pos = 0;
681- while ((pos = mangled.Find(MANGLE_COMPRESS_PREFIX, pos)) != NPOS) {701+ while ((pos = static_cast<size_t>(mangled.Find(MANGLE_COMPRESS_PREFIX, pos))) != NPOS) {
682 size_t nextIdx = ForwardNumber(mangled, pos + MANGLE_CHAR_LEN);702 size_t nextIdx = ForwardNumber(mangled, pos + MANGLE_CHAR_LEN);
683 if (nextIdx != pos + MANGLE_CHAR_LEN) {703 if (nextIdx != pos + MANGLE_CHAR_LEN) {
684 return true;704 return true;
@@ -842,6 +862,9 @@ size_t DeCompression<T>::UpdateCompressedName(T& compressed, size_t sid, size_t
842{862{
843 size_t idx = sid;863 size_t idx = sid;
844 while (idx < eid) {864 while (idx < eid) {
865+ if (idx >= compressed.Length()) {
866+ break;
867+ }
845 if (compressed[idx] == MANGLE_COMPRESS_PREFIX && !this->treeIdMap.empty() &&868 if (compressed[idx] == MANGLE_COMPRESS_PREFIX && !this->treeIdMap.empty() &&
846 idx + MANGLE_CHAR_LEN < compressed.Length()) {869 idx + MANGLE_CHAR_LEN < compressed.Length()) {
847 size_t curIdx = ForwardNumber(compressed, idx + MANGLE_CHAR_LEN);870 size_t curIdx = ForwardNumber(compressed, idx + MANGLE_CHAR_LEN);
@@ -870,8 +893,9 @@ size_t DeCompression<T>::UpdateCompressedName(T& compressed, size_t sid, size_t
870 return 0;893 return 0;
871}894}
872 895 
873-template<typename T>896+template <typename T>
874-size_t DeCompression<T>::TryExtendPath(T& mangled, size_t& count, size_t idx, size_t entityId, T& curMangled)897+size_t DeCompression<T>::TryExtendPath(
898+ T& mangled, size_t& count, size_t idx, size_t entityId, [[maybe_unused]] T& curMangled)
875{899{
876 bool isPush = false;900 bool isPush = false;
877 T mangledCopy = mangled;901 T mangledCopy = mangled;
@@ -923,9 +947,9 @@ size_t DeCompression<T>::TryLambdaPath(T& mangled, size_t& count, size_t idx, si
923 return nextIdx;947 return nextIdx;
924}948}
925 949 
926-template<typename T>950+template <typename T>
927-size_t DeCompression<T>::TryGenericPrefixPath(T& mangled, size_t& count, T& curMangled,951+size_t DeCompression<T>::TryGenericPrefixPath(
928- std::tuple<size_t, size_t, size_t> rParams)952+ T& mangled, size_t& count, [[maybe_unused]] T& curMangled, std::tuple<size_t, size_t, size_t> rParams)
929{953{
930 T mangledCopy = mangled;954 T mangledCopy = mangled;
931 size_t idx = std::get<0>(rParams);955 size_t idx = std::get<0>(rParams);
@@ -989,8 +1013,8 @@ size_t DeCompression<T>::TryNamePrefixPath(T& mangled, size_t& count, T& curMang
989 }1013 }
990 nextIdx = nextIdx == nameIdx + 1 ? nameIdx : nextIdx;1014 nextIdx = nextIdx == nameIdx + 1 ? nameIdx : nextIdx;
991 } else if (mangled[nextIdx] == MANGLE_GENERIC_PREFIX) {1015 } else if (mangled[nextIdx] == MANGLE_GENERIC_PREFIX) {
992- std::tuple<size_t, size_t, size_t> rParams{idx, entityId, nextIdx};1016+ std::tuple<size_t, size_t, size_t> genericParams{idx, entityId, nextIdx};
993- nextIdx = TryGenericPrefixPath(mangled, count, curMangled, rParams);1017+ nextIdx = TryGenericPrefixPath(mangled, count, curMangled, genericParams);
994 } else if (mangled[nextIdx] == MANGLE_FUNCTION_PREFIX) {1018 } else if (mangled[nextIdx] == MANGLE_FUNCTION_PREFIX) {
995 nextIdx = ForwardTypes(mangled, count, nameIdx + MANGLE_CHAR_LEN);1019 nextIdx = ForwardTypes(mangled, count, nameIdx + MANGLE_CHAR_LEN);
996 if (nextIdx < mangled.Length() && mangled[nextIdx] == END) {1020 if (nextIdx < mangled.Length() && mangled[nextIdx] == END) {
@@ -132,10 +132,10 @@ public:
132 *132 *
133 * @param mangled The name needs to decompress.133 * @param mangled The name needs to decompress.
134 * @param cnt Record the number of new elements added to the treeIdMap.134 * @param cnt Record the number of new elements added to the treeIdMap.
135- * @param idx The start index of the demangled name.135+ * @param startIdx The start index of the demangled name.
136 * @return size_t The end index of the demangled name.136 * @return size_t The end index of the demangled name.
137 */137 */
138- size_t ForwardTypes(T& mangled, size_t& cnt, size_t idx = 0);138+ size_t ForwardTypes(T& mangled, size_t& cnt, size_t startIdx = 0);
139 139 
140 /**140 /**
141 * @brief Get the index at the end of the number.141 * @brief Get the index at the end of the number.
@@ -339,6 +339,40 @@ public:
339 std::tuple<size_t, size_t>& eleInfo);339 std::tuple<size_t, size_t>& eleInfo);
340 340 
341private:341private:
342+ /**
343+ * @brief Forward the compound type encodes: class ('CN'/'RN'/'NN'),
344+ * function (F0/FC), tuple ('T'), C pointer ('P'), array ('A'/'V') and generic ('G').
345+ *
346+ * @param mangled The name needs to decompress.
347+ * @param cnt Record the number of new elements added to the treeIdMap.
348+ * @param idx The start index of the demangled name.
349+ * @return size_t The end index of the demangled name, or idx when the encode is unknown.
350+ */
351+ size_t ForwardCompoundType(T& mangled, size_t& cnt, size_t idx);
352+ 
353+ /**
354+ * @brief Forward the compressed type reference, Y<number>. The reference is replaced
355+ * by the span recorded in treeIdMap, and no span is recorded for the reference itself.
356+ *
357+ * @param mangled The name needs to decompress.
358+ * @param cnt Record the number of new elements added to the treeIdMap.
359+ * @param curCnt The treeIdMap count captured before the type was recorded.
360+ * @param idx The start index of the demangled name.
361+ * @return size_t The end index of the demangled name.
362+ */
363+ size_t ForwardCompressedType(T& mangled, size_t& cnt, size_t curCnt, size_t idx);
364+ 
365+ /**
366+ * @brief Pop the placeholder record pushed for the current type and return the
367+ * forwarding result. Used by the encodes that record no span, e.g. 'k' and primitives.
368+ *
369+ * @param cnt Record the number of new elements added to the treeIdMap.
370+ * @param curCnt The treeIdMap count captured before the type was recorded.
371+ * @param nextIdx The end index of the demangled name.
372+ * @return size_t The end index passed in nextIdx.
373+ */
374+ size_t PopRecordedType(size_t& cnt, size_t curCnt, size_t nextIdx);
375+ 
342 size_t pid{};376 size_t pid{};
343 bool isRecord;377 bool isRecord;
344 T mangledName;378 T mangledName;
@@ -4,7 +4,7 @@
4//4//
5// See https://cangjie-lang.cn/pages/LICENSE for license information.5// See https://cangjie-lang.cn/pages/LICENSE for license information.
6 6 
7- 7+#include "Demangler.h"
8#ifdef BUILD_LIB_CANGJIE_DEMANGLE // to reuse the code to compile CangjieDemangle.cpp8#ifdef BUILD_LIB_CANGJIE_DEMANGLE // to reuse the code to compile CangjieDemangle.cpp
9#include <cstring>9#include <cstring>
10#include <string>10#include <string>
@@ -15,7 +15,6 @@
15#endif15#endif
16#include "DeCompression.h"16#include "DeCompression.h"
17#include "Utils.h"17#include "Utils.h"
18-#include "Demangler.h"
19 18 
20namespace {19namespace {
21constexpr char MANGLE_TUPLE_PREFIX = 'T';20constexpr char MANGLE_TUPLE_PREFIX = 'T';
@@ -183,9 +182,9 @@ T ReplaceString(T str, const char* pattern, const char* replacement)
183 auto pos = str.Find(pattern);182 auto pos = str.Find(pattern);
184 auto n = str.Length();183 auto n = str.Length();
185 auto pLen = strlen(pattern);184 auto pLen = strlen(pattern);
186- while (pos > -1 && n - pos - pLen > 0) {185+ while (pos > -1 && n - static_cast<size_t>(pos) - pLen > 0) {
187 n = str.Length();186 n = str.Length();
188- str = str.SubStr(0, pos) + replacement + str.SubStr(pos + pLen, n - pos - pLen);187+ str = str.SubStr(0, pos) + replacement + str.SubStr(pos + pLen, n - static_cast<size_t>(pos) - pLen);
189 pos = str.Find(pattern);188 pos = str.Find(pattern);
190 }189 }
191 return str;190 return str;
@@ -318,7 +317,7 @@ bool Demangler<T>::IsCFunctionWrapper() const
318}317}
319 318 
320template<typename T>319template<typename T>
321-bool Demangler<T>::IsWrappedFunction() const 320+bool Demangler<T>::IsWrappedFunction() const
322{321{
323 return mangledName.Find(MANGLE_WRAPPED_FUNCTION_PREFIX, 0) == 0;322 return mangledName.Find(MANGLE_WRAPPED_FUNCTION_PREFIX, 0) == 0;
324}323}
@@ -355,7 +354,7 @@ bool Demangler<T>::IsDecl() const
355}354}
356 355 
357template<typename T>356template<typename T>
358-void Demangler<T>::ErrorLog(const char* msg) const357+void Demangler<T>::ErrorLog([[maybe_unused]] const char* msg) const
359{358{
360#if defined(MRT_DEBUG) && (MRT_DEBUG == 1) && !defined(BUILD_LIB_CANGJIE_DEMANGLE)359#if defined(MRT_DEBUG) && (MRT_DEBUG == 1) && !defined(BUILD_LIB_CANGJIE_DEMANGLE)
361 PRINT_ERROR("'%s' is not a valid mangling name.\n ", mangledName.Str());360 PRINT_ERROR("'%s' is not a valid mangling name.\n ", mangledName.Str());
@@ -368,7 +367,7 @@ void Demangler<T>::ErrorLog(const char* msg) const
368}367}
369 368 
370template<typename T>369template<typename T>
371-bool Demangler<T>::MatchForward(const char pattern[], uint32_t len) const370+bool Demangler<T>::MatchForward(const char pattern[], size_t len) const
372{371{
373 auto maxLen = mangledName.Length() - currentIndex;372 auto maxLen = mangledName.Length() - currentIndex;
374 len = len > maxLen ? maxLen : len;373 len = len > maxLen ? maxLen : len;
@@ -554,7 +553,8 @@ T DemangleInfo<T>::GetArgTypesName(const uint32_t argsNum) const
554template<typename T>553template<typename T>
555bool DemangleInfo<T>::IsFunctionLike() const554bool DemangleInfo<T>::IsFunctionLike() const
556{555{
557- return type == TypeKind::FUNCTION_DECL || type == TypeKind::LAMBDA_FUNCTION || type == TypeKind::FUNCTION || type == TypeKind::WRAPPED_FUNCTION;556+ return type == TypeKind::FUNCTION_DECL || type == TypeKind::LAMBDA_FUNCTION ||
557+ type == TypeKind::FUNCTION || type == TypeKind::WRAPPED_FUNCTION;
558}558}
559 559 
560template<typename T>560template<typename T>
@@ -682,7 +682,7 @@ DemangleInfo<T> Demangler<T>::Demangle(bool isType)
682 TypeKind::NAME, isValid };682 TypeKind::NAME, isValid };
683 }683 }
684 // Wrapped func like ""_CVN7default1S4testHv$N7default1SE$CN7default1IE""684 // Wrapped func like ""_CVN7default1S4testHv$N7default1SE$CN7default1IE""
685- if(IsWrappedFunction()) {685+ if (IsWrappedFunction()) {
686 return DemangleWrappedFunction();686 return DemangleWrappedFunction();
687 }687 }
688 return DemangleDecl();688 return DemangleDecl();
@@ -716,8 +716,8 @@ DemangleInfo<T> Demangler<T>::DemanglePackageName()
716 if (pkg.IsEmpty()) {716 if (pkg.IsEmpty()) {
717 pkg = DemangleStringName();717 pkg = DemangleStringName();
718 auto pos = pkg.Find(':');718 auto pos = pkg.Find(':');
719- if (pos > -1 && pkg.Length() - pos > 0) {719+ if (pos > -1 && pkg.Length() - static_cast<size_t>(pos) > 1) {
wangyinqiang
wangyinqiangwangyinqiang12 天前

[major] 同目录下 Utils.h 未同步,MANGLE_STDPKG_MAP 存在真实的 demangle 结果分歧

(本条与本行代码无关,因 Utils.h 不在本次 diff 内,借 DemanglePackageName 内这处改动提出)

问题:issue #1123 的复现步骤指向的是三个 demangler 目录,本次只同步了其中 4 个文件。我把三仓 head 上另外 5 个共享文件也比了一遍,全部未同步:CangjieDemangle.cpp / CangjieDemangle.h / Utils.h / StdString.h / Cjfilt.cpp。其中 Utils.h 的 MANGLE_STDPKG_MAP 条目数为 compiler 65、runtime 68、本仓 65,runtime 独有 {"aa", "std.ad"}、{"bp", "std.net.native.cjvm"}、{"br", "std.ffi.java"} 三条。

影响:这张表在 DemanglePackageName 里直接决定包名还原,同一个 mangled 名在 runtime 与 cjfilt 下会 demangle 成不同结果——正是 #1123 要消灭的那类不一致。另外 StdPkgHash 的 + static_cast<uint8_t>(cPkg[id++])(compiler)与 + cPkg[id++](runtime/tools)也未对齐,因键都是 2 字符 ASCII 暂不触发差异。

建议:本轮补上 Utils.h 的这三条映射;其余文件若不在本轮处理,请在 PR 描述里写明「本次只同步 4 个文件」以及剩余分歧与 #1123 的关系,否则 issue 关掉后这部分不一致就无人认领。

likedislike
720- pkg = pkg.SubStr(0, pos) + T{':'} + pkg.SubStr(pos, pkg.Length() - pos);720+ pkg = pkg.SubStr(0, pos) + T{':'} + pkg.SubStr(pos, pkg.Length() - static_cast<size_t>(pos));
721 }721 }
722 }722 }
723 if (IsFileName()) {723 if (IsFileName()) {
@@ -1060,7 +1060,8 @@ DemangleInfo<T> Demangler<T>::DemangleNestedDecls(bool isClass, bool isParamInit
1060 }1060 }
1061 lastElement = curDi;1061 lastElement = curDi;
1062 }1062 }
1063- result += GetArgTypesFullName(argsArr, i % (MAX_ARGS_SIZE + 1), delimiter);1063+ uint32_t remainingLength = i == 0 ? 0 : ((i - 1) % MAX_ARGS_SIZE) + 1;
wangyinqiang
wangyinqiangwangyinqiang12 天前

[test] 这一行是真实的行为修复,但三个 PR 都没有新增用例

问题:i % (MAX_ARGS_SIZE + 1) 是以 17 为模,i=17 得 0(应为 1)、i=18 得 1(应为 2),即嵌套声明 ≥17 层时输出的名字会丢段/错位;本次改成 i == 0 ? 0 : ((i - 1) % MAX_ARGS_SIZE) + 1 修掉了它。同一批改动里还有 DemanglePackageName 的 > 0 → > 1、ForwardName 的 while 条件(原来检查的是 idx、递增的却是 numberLen,是真越界读)以及新增的长度校验,都是行为变更。

影响:PR 自检勾的是「测试用例:不涉及(同步代码,不涉及新功能)」,与实际不符。本仓 third_party/demangler 目录下没有任何测试载体,这批行为变更在本仓完全没有回归网。

建议:至少在 compiler 的 unittests/Demangle 或 runtime 的 CheckDemangle.cpp 里补上对应用例,并在 PR 描述里说明本仓依赖哪一侧的用例做回归。

likedislike
1064+ result += GetArgTypesFullName(argsArr, remainingLength, delimiter);
1064 DemangleInfo<T> resDi = { result, typeKind, isValid };1065 DemangleInfo<T> resDi = { result, typeKind, isValid };
1065 resDi.functionParameterTypes = lastElement.functionParameterTypes;1066 resDi.functionParameterTypes = lastElement.functionParameterTypes;
1066 return resDi;1067 return resDi;
@@ -1131,7 +1132,7 @@ DemangleInfo<T> Demangler<T>::DemangleVArray()
1131{1132{
1132 DemangleInfo<T> di{ MANGLE_VARRAY_STR, TypeKind::TUPLE };1133 DemangleInfo<T> di{ MANGLE_VARRAY_STR, TypeKind::TUPLE };
1133 SkipChar(MANGLE_VARRAY_PREFIX);1134 SkipChar(MANGLE_VARRAY_PREFIX);
1134- uint32_t num = DemangleManglingNumber();1135+ uint32_t num = static_cast<uint32_t>(DemangleManglingNumber());
1135 di.genericTypes = DemangleGenericTypes().GetFullName(scopeResolution, num);1136 di.genericTypes = DemangleGenericTypes().GetFullName(scopeResolution, num);
1136 return di;1137 return di;
1137}1138}
@@ -1163,7 +1164,7 @@ DemangleInfo<T> Demangler<T>::DemangleCommonDecl(bool isClass)
1163 auto pkgDi = DemanglePackageName();1164 auto pkgDi = DemanglePackageName();
1164 di.pkgName = pkgDi.pkgName;1165 di.pkgName = pkgDi.pkgName;
1165 di.isPrivateDeclaration = pkgDi.isPrivateDeclaration;1166 di.isPrivateDeclaration = pkgDi.isPrivateDeclaration;
1166- 1167+ 
1167 // Take the part before E as identifier and concatenate with scope resolution.1168 // Take the part before E as identifier and concatenate with scope resolution.
1168 DemangleInfo<T> nestedDi = DemangleNestedDecls(isClass);1169 DemangleInfo<T> nestedDi = DemangleNestedDecls(isClass);
1169 di.demangled = nestedDi.demangled;1170 di.demangled = nestedDi.demangled;
@@ -1373,14 +1374,14 @@ DemangleInfo<T> Demangler<T>::DemangleInnerFunction()
1373}1374}
1374 1375 
1375template<typename T>1376template<typename T>
1376-DemangleInfo<T> Demangler<T>::DemangleWrappedFunction() 1377+DemangleInfo<T> Demangler<T>::DemangleWrappedFunction()
1377{1378{
1378 auto endIndex = mangledName.Find(MANGLE_END, currentIndex);1379 auto endIndex = mangledName.Find(MANGLE_END, currentIndex);
1379 auto prefixIndex = currentIndex + strlen(MANGLE_WRAPPED_FUNCTION_PREFIX);1380 auto prefixIndex = currentIndex + strlen(MANGLE_WRAPPED_FUNCTION_PREFIX);
1380 mangledName = T(MANGLE_CANGJIE_PREFIX) + mangledName.SubStr(prefixIndex, endIndex - prefixIndex);1381 mangledName = T(MANGLE_CANGJIE_PREFIX) + mangledName.SubStr(prefixIndex, endIndex - prefixIndex);
1381 auto di = DemangleDecl();1382 auto di = DemangleDecl();
1382 di.demangled += WRAPPED_FUNCTION_SUFFIX_INFO;1383 di.demangled += WRAPPED_FUNCTION_SUFFIX_INFO;
1383- di.type = TypeKind::WRAPPED_FUNCTION; 1384+ di.type = TypeKind::WRAPPED_FUNCTION;
1384 return di;1385 return di;
1385}1386}
1386 1387 
@@ -8,16 +8,17 @@
8#ifndef CANGJIE_DEMANGLER_H8#ifndef CANGJIE_DEMANGLER_H
9#define CANGJIE_DEMANGLER_H9#define CANGJIE_DEMANGLER_H
10 10 
11-#include <array>
12#ifndef BUILD_LIB_CANGJIE_DEMANGLE11#ifndef BUILD_LIB_CANGJIE_DEMANGLE
13-#include <vector>
14#include "Base/CString.h"12#include "Base/CString.h"
13+#else
14+#include <string>
15+#include <vector>
余敲鱼wangyinqiang
余敲鱼余敲鱼19 天前

#include <vector> 挪到 Line:15~16

likedislike
wangyinqiangwangyinqiang12 天前

[nit] 被挪进了不使用它的分支

改动后 #ifndef BUILD_LIB_CANGJIE_DEMANGLE 分支只剩 "Base/CString.h", 和 都进了 #else,但本文件里 std::vector<std::string> genericVec 是无条件的。本仓内没有 Base/CString.h,那条分支在本仓不可构建,所以这处移动在本仓无实际影响;但 runtime 仓走的正是这条路径,那边 现在只剩传递包含。既然三仓这个文件要逐字一致,建议统一把 与 提到条件块外无条件包含。

likedislike
15#endif16#endif
17+#include <array>
16#include <functional>18#include <functional>
17#include <cassert>19#include <cassert>
18#include <cstdint>20#include <cstdint>
19 21 
20- 
21namespace Cangjie {22namespace Cangjie {
22 23 
23enum class TypeKind {24enum class TypeKind {
@@ -281,7 +282,7 @@ private:
281 bool IsQualifiedType() const;282 bool IsQualifiedType() const;
282 bool IsDecl() const;283 bool IsDecl() const;
283 bool IsNotEndOfMangledName() const;284 bool IsNotEndOfMangledName() const;
284- bool MatchForward(const char pattern[], uint32_t len) const;285+ bool MatchForward(const char pattern[], size_t len) const;
285 286 
286 /**287 /**
287 * @brief Get the current character without incrementing `currentIndex`.288 * @brief Get the current character without incrementing `currentIndex`.