* Copyright (c) 2021 Huawei Device Co., Ltd.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "common_components/base/config.h"
#include "common_components/base/utf_helper.h"
#include "common_components/log/log.h"
#include "libpandabase/utils/span.h"
static constexpr int32_t U16_SURROGATE_OFFSET = (0xd800 << 10UL) + 0xdc00 - 0x10000;
#define U16_GET_SUPPLEMENTARY(lead, trail) \
((static_cast<int32_t>(lead) << 10UL) + static_cast<int32_t>(trail) - U16_SURROGATE_OFFSET)
namespace common::utf_helper {
uint32_t UTF16Decode(uint16_t lead, uint16_t trail)
{
DCHECK_CC((lead >= DECODE_LEAD_LOW && lead <= DECODE_LEAD_HIGH) &&
(trail >= DECODE_TRAIL_LOW && trail <= DECODE_TRAIL_HIGH));
uint32_t cp = (lead - DECODE_LEAD_LOW) * DECODE_FIRST_FACTOR + (trail - DECODE_TRAIL_LOW) + DECODE_SECOND_FACTOR;
return cp;
}
uint32_t DecodeUTF16(uint16_t const *utf16, size_t len, size_t *index, bool cesu8)
{
uint16_t high = utf16[*index];
if ((high & SURROGATE_MASK) != DECODE_LEAD_LOW || !IsUTF16HighSurrogate(high) || *index == len - 1) {
return high;
}
uint16_t low = utf16[*index + 1];
if (!IsUTF16LowSurrogate(low) || cesu8) {
return high;
}
(*index)++;
return ((high - DECODE_LEAD_LOW) << UTF16_OFFSET) + (low - DECODE_TRAIL_LOW) + DECODE_SECOND_FACTOR;
}
uint32_t HandleAndDecodeInvalidUTF16(uint16_t const *utf16, size_t len, size_t *index)
{
uint16_t first = utf16[*index];
if (IsUTF16LowSurrogate(first)) {
return UTF16_REPLACEMENT_CHARACTER;
}
if (IsUTF16HighSurrogate(first) || (first & SURROGATE_MASK) == DECODE_LEAD_LOW) {
if (*index == len - 1) {
return UTF16_REPLACEMENT_CHARACTER;
}
uint16_t second = utf16[*index + 1];
if (!IsUTF16LowSurrogate(second)) {
return UTF16_REPLACEMENT_CHARACTER;
}
(*index)++;
return ((first - DECODE_LEAD_LOW) << UTF16_OFFSET) + (second - DECODE_TRAIL_LOW) + DECODE_SECOND_FACTOR;
}
return first;
}
size_t EncodeUTF8(uint32_t codepoint, uint8_t* utf8, size_t index, size_t size)
{
for (size_t j = size - 1; j > 0; j--) {
uint8_t cont = ((codepoint | byteMark) & byteMask);
utf8[index + j] = cont;
codepoint >>= UTF8_OFFSET;
}
utf8[index] = codepoint | firstByteMark[size];
return size;
}
bool IsValidUTF8(const std::vector<uint8_t> &data)
{
uint32_t length = data.size();
switch (length) {
case UtfLength::ONE:
if (data.at(0) >= BIT_MASK_1) {
return false;
}
break;
case UtfLength::TWO:
if ((data.at(0) & BIT_MASK_3) != BIT_MASK_2) {
return false;
}
if (data.at(0) < UTF8_2B_FIRST_MIN) {
return false;
}
break;
case UtfLength::THREE:
if ((data.at(0) & BIT_MASK_4) != BIT_MASK_3) {
return false;
}
if (data.at(0) == UTF8_3B_FIRST && data.at(1) < UTF8_3B_SECOND_MIN) {
return false;
}
if (data.at(0) == UTF8_3B_RESERVED_FIRST && data.at(1) >= UTF8_3B_RESERVED_SECOND_MIN &&
data.at(1) <= UTF8_3B_RESERVED_SECOND_MAX) {
return false;
}
break;
case UtfLength::FOUR:
if ((data.at(0) & BIT_MASK_5) != BIT_MASK_4) {
return false;
}
if (data.at(0) == UTF8_4B_FIRST && data.at(1) < UTF8_4B_SECOND_MIN) {
return false;
}
if (data.at(0) > UTF8_4B_FIRST_MAX ||
(data.at(0) == UTF8_4B_FIRST_MAX && data.at(1) > UTF8_4B_SECOND_MAX)) {
return false;
}
break;
default:
LOG_COMMON(FATAL) << "this branch is unreachable";
UNREACHABLE_CC();
break;
}
for (uint32_t i = 1; i < length; i++) {
if ((data.at(i) & BIT_MASK_2) != BIT_MASK_1) {
return false;
}
}
return true;
}
Utf8Char ConvertUtf16ToUtf8(uint16_t d0, uint16_t d1, bool modify, bool isWriteBuffer)
{
if (d1 == 0 && d0 >= HI_SURROGATE_MIN && d0 <= LO_SURROGATE_MAX) {
auto ch0 = static_cast<uint8_t>(UTF8_3B_FIRST | static_cast<uint8_t>(d0 >> UtfOffset::TWELVE));
auto ch1 = static_cast<uint8_t>(UTF8_3B_SECOND | (static_cast<uint8_t>(d0 >> UtfOffset::SIX) & MASK_6BIT));
auto ch2 = static_cast<uint8_t>(UTF8_3B_THIRD | (d0 & MASK_6BIT));
return {UtfLength::THREE, {ch0, ch1, ch2}};
}
if (d0 == 0) {
if (isWriteBuffer) {
return {1, {0x00U}};
}
if (modify) {
return {UtfLength::TWO, {UTF8_2B_FIRST, UTF8_2B_SECOND}};
}
return {0, {0x00U}};
}
if (d0 <= UTF8_1B_MAX) {
return {UtfLength::ONE, {static_cast<uint8_t>(d0)}};
}
if (d0 <= UTF8_2B_MAX) {
auto ch0 = static_cast<uint8_t>(UTF8_2B_FIRST | static_cast<uint8_t>(d0 >> UtfOffset::SIX));
auto ch1 = static_cast<uint8_t>(UTF8_2B_SECOND | (d0 & MASK_6BIT));
return {UtfLength::TWO, {ch0, ch1}};
}
if (d0 < HI_SURROGATE_MIN || d0 > HI_SURROGATE_MAX) {
auto ch0 = static_cast<uint8_t>(UTF8_3B_FIRST | static_cast<uint8_t>(d0 >> UtfOffset::TWELVE));
auto ch1 = static_cast<uint8_t>(UTF8_3B_SECOND | (static_cast<uint8_t>(d0 >> UtfOffset::SIX) & MASK_6BIT));
auto ch2 = static_cast<uint8_t>(UTF8_3B_THIRD | (d0 & MASK_6BIT));
return {UtfLength::THREE, {ch0, ch1, ch2}};
}
if (d1 < LO_SURROGATE_MIN || d1 > LO_SURROGATE_MAX) {
LOG_COMMON(FATAL) << "this branch is unreachable";
UNREACHABLE_CC();
}
uint32_t codePoint = CombineTwoU16(d0, d1);
auto ch0 = static_cast<uint8_t>((codePoint >> UtfOffset::EIGHTEEN) | UTF8_4B_FIRST);
auto ch1 = static_cast<uint8_t>(((codePoint >> UtfOffset::TWELVE) & MASK_6BIT) | MASK1);
auto ch2 = static_cast<uint8_t>(((codePoint >> UtfOffset::SIX) & MASK_6BIT) | MASK1);
auto ch3 = static_cast<uint8_t>((codePoint & MASK_6BIT) | MASK1);
return {UtfLength::FOUR, {ch0, ch1, ch2, ch3}};
}
size_t Utf16ToUtf8Size(const uint16_t *utf16, uint32_t length, bool modify, bool isGetBufferSize, bool cesu8)
{
size_t res = 1;
if (length == 1 && utf16[0] >= HI_SURROGATE_MIN &&
utf16[0] <= LO_SURROGATE_MAX) {
res += UtfLength::THREE;
return res;
}
for (uint32_t i = 0; i < length; ++i) {
if (utf16[i] == 0) {
if (isGetBufferSize) {
res += UtfLength::ONE;
} else if (modify) {
res += UtfLength::TWO;
}
} else if (utf16[i] <= UTF8_1B_MAX) {
res += 1;
} else if (utf16[i] <= UTF8_2B_MAX) {
res += UtfLength::TWO;
} else if (utf16[i] < HI_SURROGATE_MIN || utf16[i] > HI_SURROGATE_MAX) {
res += UtfLength::THREE;
} else {
if (!cesu8 && i < length - 1 &&
utf16[i + 1] >= LO_SURROGATE_MIN &&
utf16[i + 1] <= LO_SURROGATE_MAX) {
res += UtfLength::FOUR;
++i;
} else {
res += UtfLength::THREE;
}
}
}
return res;
}
size_t ConvertRegionUtf16ToUtf8(const uint16_t *utf16In, uint8_t *utf8Out, size_t utf16Len, size_t utf8Len,
size_t start, bool modify, bool isWriteBuffer, bool cesu8)
{
#if ENABLE_LATEST_OPTIMIZATION && defined(ENABLE_HISPEED_PLUGIN)
auto hispeedConvertRegionUtf16ToUtf8 = HispeedLibSingleton::GetInstance().GetHispeedConvertRegionUtf16ToUtf8();
if (hispeedConvertRegionUtf16ToUtf8 != nullptr) {
return hispeedConvertRegionUtf16ToUtf8(utf16In, utf8Out, utf16Len, utf8Len,
start, modify, isWriteBuffer, cesu8);
}
#endif
if (utf16In == nullptr || utf8Out == nullptr || utf8Len == 0) {
return 0;
}
size_t utf8Pos = 0;
size_t end = start + utf16Len;
for (size_t i = start; i < end; ++i) {
uint32_t codepoint = DecodeUTF16(utf16In, end, &i, cesu8);
if (codepoint == 0) {
if (isWriteBuffer) {
utf8Out[utf8Pos++] = 0x00U;
continue;
}
if (modify) {
utf8Out[utf8Pos++] = UTF8_2B_FIRST;
utf8Out[utf8Pos++] = UTF8_2B_SECOND;
}
continue;
}
size_t size = UTF8Length(codepoint);
if (utf8Pos + size > utf8Len) {
break;
}
utf8Pos += EncodeUTF8(codepoint, utf8Out, utf8Pos, size);
}
return utf8Pos;
}
size_t DebuggerConvertRegionUtf16ToUtf8(const uint16_t *utf16In, uint8_t *utf8Out, size_t utf16Len, size_t utf8Len,
size_t start, bool modify, bool isWriteBuffer)
{
if (utf16In == nullptr || utf8Out == nullptr || utf8Len == 0) {
return 0;
}
size_t utf8Pos = 0;
size_t end = start + utf16Len;
for (size_t i = start; i < end; ++i) {
uint32_t codepoint = HandleAndDecodeInvalidUTF16(utf16In, end, &i);
if (codepoint == 0) {
if (isWriteBuffer) {
utf8Out[utf8Pos++] = 0x00U;
continue;
}
if (modify) {
utf8Out[utf8Pos++] = UTF8_2B_FIRST;
utf8Out[utf8Pos++] = UTF8_2B_SECOND;
}
continue;
}
size_t size = UTF8Length(codepoint);
if (utf8Pos + size > utf8Len) {
break;
}
utf8Pos += EncodeUTF8(codepoint, utf8Out, utf8Pos, size);
}
return utf8Pos;
}
std::pair<uint32_t, size_t> ConvertUtf8ToUtf16Pair(const uint8_t *data, bool combine)
{
uint8_t d0 = data[0];
if ((d0 & MASK1) == 0) {
return {d0, 1};
}
uint8_t d1 = data[1];
if ((d0 & MASK2) == 0) {
return {((d0 & MASK_5BIT) << DATA_WIDTH) | (d1 & MASK_6BIT), UtfLength::TWO};
}
uint8_t d2 = data[UtfLength::TWO];
if ((d0 & MASK3) == 0) {
return {((d0 & MASK_4BIT) << UtfOffset::TWELVE) | ((d1 & MASK_6BIT) << DATA_WIDTH) |
(d2 & MASK_6BIT),
UtfLength::THREE};
}
uint8_t d3 = data[UtfLength::THREE];
uint32_t codePoint = ((d0 & MASK_4BIT) << UtfOffset::EIGHTEEN) | ((d1 & MASK_6BIT) << UtfOffset::TWELVE) |
((d2 & MASK_6BIT) << DATA_WIDTH) | (d3 & MASK_6BIT);
uint32_t pair = 0;
if (combine) {
uint32_t lead = ((codePoint >> (PAIR_ELEMENT_WIDTH - DATA_WIDTH)) + U16_LEAD);
uint32_t tail = ((codePoint & MASK_10BIT) + U16_TAIL) & MASK_16BIT;
pair = static_cast<uint32_t>(U16_GET_SUPPLEMENTARY(lead, tail));
} else {
pair |= ((codePoint >> (PAIR_ELEMENT_WIDTH - DATA_WIDTH)) + U16_LEAD) << PAIR_ELEMENT_WIDTH;
pair |= ((codePoint & MASK_10BIT) + U16_TAIL) & MASK_16BIT;
}
return {pair, UtfLength::FOUR};
}
static inline size_t FixUtf8Len(const uint8_t* utf8, size_t utf8Len)
{
size_t trimSize = 0;
if (utf8Len >= 1 && utf8[utf8Len - 1] >= 0xC0) {
trimSize = 1;
}
if (utf8Len >= CONST_2 && utf8[utf8Len - CONST_2] >= 0xE0) {
trimSize = CONST_2;
}
if (utf8Len >= CONST_3 && utf8[utf8Len - CONST_3] >= 0xF0) {
trimSize = CONST_3;
}
return utf8Len - trimSize;
}
size_t Utf8ToUtf16Size(const uint8_t *utf8, size_t utf8Len)
{
#if ENABLE_LATEST_OPTIMIZATION && defined(ENABLE_HISPEED_PLUGIN)
auto hispeedUtf8ToUtf16Size = HispeedLibSingleton::GetInstance().GetHispeedUtf8ToUtf16Size();
if (hispeedUtf8ToUtf16Size != nullptr) {
return hispeedUtf8ToUtf16Size(utf8, utf8Len);
}
#endif
size_t safeUtf8Len = FixUtf8Len(utf8, utf8Len);
size_t in_pos = 0;
size_t res = 0;
while (in_pos < safeUtf8Len) {
uint8_t src = utf8[in_pos];
switch (src & 0xF0) {
case 0xF0: {
const uint8_t c2 = utf8[++in_pos];
const uint8_t c3 = utf8[++in_pos];
const uint8_t c4 = utf8[++in_pos];
uint32_t codePoint = ((src & LOW_3BITS) << OFFSET_18POS) | ((c2 & LOW_6BITS) << OFFSET_12POS) |
((c3 & LOW_6BITS) << OFFSET_6POS) | (c4 & LOW_6BITS);
if (codePoint >= SURROGATE_RAIR_START) {
res += CONST_2;
} else {
res++;
}
in_pos++;
break;
}
case 0xE0: {
in_pos += CONST_3;
res++;
break;
}
case 0xD0:
case 0xC0: {
in_pos += CONST_2;
res++;
break;
}
default:
do {
in_pos++;
res++;
} while (in_pos < safeUtf8Len && utf8[in_pos] < 0x80);
break;
}
}
res += utf8Len - in_pos;
return res;
}
size_t ConvertRegionUtf8ToUtf16(const uint8_t *utf8In, uint16_t *utf16Out, size_t utf8Len, size_t utf16Len)
{
#if ENABLE_LATEST_OPTIMIZATION && defined(ENABLE_HISPEED_PLUGIN)
auto hispeedConvertRegionUtf8ToUtf16 = HispeedLibSingleton::GetInstance().GetHispeedConvertRegionUtf8ToUtf16();
if (hispeedConvertRegionUtf8ToUtf16 != nullptr) {
return hispeedConvertRegionUtf8ToUtf16(utf8In, utf16Out, utf8Len, utf16Len);
}
#endif
size_t safeUtf8Len = FixUtf8Len(utf8In, utf8Len);
size_t in_pos = 0;
size_t out_pos = 0;
while (in_pos < safeUtf8Len && out_pos < utf16Len) {
uint8_t src = utf8In[in_pos];
switch (src & 0xF0) {
case 0xF0: {
const uint8_t c2 = utf8In[++in_pos];
const uint8_t c3 = utf8In[++in_pos];
const uint8_t c4 = utf8In[++in_pos];
uint32_t codePoint = ((src & LOW_3BITS) << OFFSET_18POS) | ((c2 & LOW_6BITS) << OFFSET_12POS) |
((c3 & LOW_6BITS) << OFFSET_6POS) | (c4 & LOW_6BITS);
if (codePoint >= SURROGATE_RAIR_START) {
DCHECK_CC(utf16Len >= 1);
if (out_pos >= utf16Len - 1) {
return out_pos;
}
codePoint -= SURROGATE_RAIR_START;
utf16Out[out_pos++] = static_cast<uint16_t>((codePoint >> OFFSET_10POS) | H_SURROGATE_START);
utf16Out[out_pos++] = static_cast<uint16_t>((codePoint & 0x3FF) | L_SURROGATE_START);
} else {
utf16Out[out_pos++] = static_cast<uint16_t>(codePoint);
}
in_pos++;
break;
}
case 0xE0: {
const uint8_t c2 = utf8In[++in_pos];
const uint8_t c3 = utf8In[++in_pos];
utf16Out[out_pos++] = static_cast<uint16_t>(((src & LOW_4BITS) << OFFSET_12POS) |
((c2 & LOW_6BITS) << OFFSET_6POS) | (c3 & LOW_6BITS));
in_pos++;
break;
}
case 0xD0:
case 0xC0: {
const uint8_t c2 = utf8In[++in_pos];
utf16Out[out_pos++] = static_cast<uint16_t>(((src & LOW_5BITS) << OFFSET_6POS) | (c2 & LOW_6BITS));
in_pos++;
break;
}
default:
do {
utf16Out[out_pos++] = static_cast<uint16_t>(utf8In[in_pos++]);
} while (in_pos < safeUtf8Len && out_pos < utf16Len && utf8In[in_pos] < 0x80);
break;
}
}
while (in_pos < utf8Len && out_pos < utf16Len) {
utf16Out[out_pos++] = static_cast<uint16_t>(utf8In[in_pos++]);
}
return out_pos;
}
size_t ConvertRegionUtf16ToLatin1(const uint16_t *utf16In, uint8_t *latin1Out, size_t utf16Len, size_t latin1Len)
{
if (utf16In == nullptr || latin1Out == nullptr || latin1Len == 0) {
return 0;
}
size_t latin1Pos = 0;
size_t end = utf16Len;
for (size_t i = 0; i < end; ++i) {
if (latin1Pos == latin1Len) {
break;
}
uint32_t codepoint = DecodeUTF16(utf16In, end, &i);
uint8_t latin1Code = static_cast<uint8_t>(codepoint & latin1Limit);
latin1Out[latin1Pos++] = latin1Code;
}
return latin1Pos;
}
std::pair<int32_t, size_t> ConvertUtf8ToUnicodeChar(const uint8_t *utf8, size_t maxLen)
{
if (maxLen == 0) {
return {INVALID_UTF8, 0};
}
panda::Span<const uint8_t> sp(utf8, maxLen);
uint8_t d0 = sp[0];
if ((d0 & BIT_MASK_1) == 0) {
return {d0, UtfLength::ONE};
}
if (maxLen < UtfLength::TWO) {
return {INVALID_UTF8, 0};
}
uint8_t d1 = sp[UtfLength::ONE];
if ((d0 & BIT_MASK_3) == BIT_MASK_2) {
if ((d1 & BIT_MASK_2) == BIT_MASK_1) {
return {((d0 & MASK_5BIT) << DATA_WIDTH) | (d1 & MASK_6BIT), UtfLength::TWO};
} else {
return {INVALID_UTF8, 0};
}
}
if (maxLen < UtfLength::THREE) {
return {INVALID_UTF8, 0};
}
uint8_t d2 = sp[UtfLength::TWO];
if ((d0 & BIT_MASK_4) == BIT_MASK_3) {
if (((d1 & BIT_MASK_2) == BIT_MASK_1) && ((d2 & BIT_MASK_2) == BIT_MASK_1)) {
return {((d0 & MASK_4BIT) << UtfOffset::TWELVE) |
((d1 & MASK_6BIT) << DATA_WIDTH) | (d2 & MASK_6BIT), UtfLength::THREE};
} else {
return {INVALID_UTF8, 0};
}
}
if (maxLen < UtfLength::FOUR) {
return {INVALID_UTF8, 0};
}
uint8_t d3 = sp[UtfLength::THREE];
if ((d0 & BIT_MASK_5) == BIT_MASK_4) {
if (((d1 & BIT_MASK_2) == BIT_MASK_1) &&
((d2 & BIT_MASK_2) == BIT_MASK_1) && ((d3 & BIT_MASK_2) == BIT_MASK_1)) {
return {((d0 & MASK_4BIT) << UtfOffset::EIGHTEEN) | ((d1 & MASK_6BIT) << UtfOffset::TWELVE) |
((d2 & MASK_6BIT) << DATA_WIDTH) | (d3 & MASK_6BIT), UtfLength::FOUR};
} else {
return {INVALID_UTF8, 0};
}
}
return {INVALID_UTF8, 0};
}
#ifdef ENABLE_HISPEED_PLUGIN
HispeedLibSingleton& HispeedLibSingleton::GetInstance()
{
static HispeedLibSingleton instance;
return instance;
}
#endif
}