#ifndef V8_STRINGS_UNICODE_INL_H_
#define V8_STRINGS_UNICODE_INL_H_
#include "src/strings/unicode.h"
#include "src/base/logging.h"
#include "src/utils/utils.h"
#include "third_party/simdutf/simdutf.h"
namespace unibrow {
#ifndef V8_INTL_SUPPORT
template <class T, int s>
bool Predicate<T, s>::get(uchar code_point) {
CacheEntry entry = entries_[code_point & kMask];
if (entry.code_point() == code_point) return entry.value();
return CalculateValue(code_point);
}
template <class T, int s>
bool Predicate<T, s>::CalculateValue(uchar code_point) {
bool result = T::Is(code_point);
entries_[code_point & kMask] = CacheEntry(code_point, result);
return result;
}
template <class T, int s>
int Mapping<T, s>::get(uchar c, uchar n, uchar* result) {
CacheEntry entry = entries_[c & kMask];
if (entry.code_point_ == c) {
if (entry.offset_ == 0) {
return 0;
} else {
result[0] = c + entry.offset_;
return 1;
}
} else {
return CalculateValue(c, n, result);
}
}
template <class T, int s>
int Mapping<T, s>::CalculateValue(uchar c, uchar n, uchar* result) {
bool allow_caching = true;
int length = T::Convert(c, n, result, &allow_caching);
if (allow_caching) {
if (length == 1) {
entries_[c & kMask] = CacheEntry(c, result[0] - c);
return 1;
} else {
entries_[c & kMask] = CacheEntry(c, 0);
return 0;
}
} else {
return length;
}
}
#endif
bool Utf16::HasUnpairedSurrogate(const uint16_t* code_units, size_t length) {
return !simdutf::validate_utf16(reinterpret_cast<const char16_t*>(code_units),
length);
}
uchar Utf8::ValueOfIncremental(const uint8_t** cursor, State* state,
Utf8IncrementalBuffer* buffer) {
DCHECK_NOT_NULL(buffer);
State old_state = *state;
uint8_t next = **cursor;
*cursor += 1;
if (V8_LIKELY(next <= kMaxOneByteChar && old_state == State::kAccept)) {
DCHECK_EQ(0u, *buffer);
return static_cast<uchar>(next);
}
Utf8DfaDecoder::Decode(next, state, buffer);
switch (*state) {
case State::kAccept: {
uchar t = *buffer;
*buffer = 0;
return t;
}
case State::kReject:
*state = State::kAccept;
*buffer = 0;
if (old_state != State::kAccept) {
*cursor -= 1;
}
return kBadChar;
default:
return kIncomplete;
}
}
unsigned Utf8::EncodeOneByte(char* str, uint8_t c) {
static const int kMask = ~(1 << 6);
if (c <= kMaxOneByteChar) {
str[0] = c;
return 1;
} else {
str[0] = 0xC0 | (c >> 6);
str[1] = 0x80 | (c & kMask);
return 2;
}
}
unsigned Utf8::Encode(char* str, uchar c, int previous, bool replace_invalid) {
static const int kMask = ~(1 << 6);
if (c <= kMaxOneByteChar) {
str[0] = c;
return 1;
} else if (c <= kMaxTwoByteChar) {
str[0] = 0xC0 | (c >> 6);
str[1] = 0x80 | (c & kMask);
return 2;
} else if (c <= kMaxThreeByteChar) {
DCHECK(!Utf16::IsLeadSurrogate(Utf16::kNoPreviousCharacter));
if (Utf16::IsSurrogatePair(previous, c)) {
const int kUnmatchedSize = kSizeOfUnmatchedSurrogate;
return Encode(str - kUnmatchedSize,
Utf16::CombineSurrogatePair(previous, c),
Utf16::kNoPreviousCharacter, replace_invalid) -
kUnmatchedSize;
} else if (replace_invalid &&
(Utf16::IsLeadSurrogate(c) || Utf16::IsTrailSurrogate(c))) {
c = kBadChar;
}
str[0] = 0xE0 | (c >> 12);
str[1] = 0x80 | ((c >> 6) & kMask);
str[2] = 0x80 | (c & kMask);
return 3;
} else {
str[0] = 0xF0 | (c >> 18);
str[1] = 0x80 | ((c >> 12) & kMask);
str[2] = 0x80 | ((c >> 6) & kMask);
str[3] = 0x80 | (c & kMask);
return 4;
}
}
uchar Utf8::ValueOf(const uint8_t* bytes, size_t length, size_t* cursor) {
if (length == 0) return kBadChar;
uint8_t first = bytes[0];
if (V8_LIKELY(first <= kMaxOneByteChar)) {
*cursor += 1;
return first;
}
return CalculateValue(bytes, length, cursor);
}
unsigned Utf8::LengthOneByte(uint8_t c) {
if (c <= kMaxOneByteChar) {
return 1;
} else {
return 2;
}
}
unsigned Utf8::Length(uchar c, int previous) {
if (c <= kMaxOneByteChar) {
return 1;
} else if (c <= kMaxTwoByteChar) {
return 2;
} else if (c <= kMaxThreeByteChar) {
DCHECK(!Utf16::IsLeadSurrogate(Utf16::kNoPreviousCharacter));
if (Utf16::IsSurrogatePair(previous, c)) {
return kSizeOfUnmatchedSurrogate - kBytesSavedByCombiningSurrogates;
}
return 3;
} else {
return 4;
}
}
bool Utf8::IsValidCharacter(uchar c) {
return c < 0xD800u || (c >= 0xE000u && c < 0xFDD0u) ||
(c > 0xFDEFu && c <= 0x10FFFFu && (c & 0xFFFEu) != 0xFFFEu &&
c != kBadChar);
}
template <typename Char>
Utf8::EncodingResult Utf8::Encode(v8::base::Vector<const Char> string,
char* buffer, size_t capacity,
bool write_null, bool replace_invalid_utf8) {
constexpr bool kSourceIsOneByte = sizeof(Char) == 1;
if constexpr (kSourceIsOneByte) {
replace_invalid_utf8 = false;
}
size_t write_index = 0;
const Char* characters = string.begin();
size_t content_capacity = capacity - write_null;
CHECK_LE(content_capacity, capacity);
size_t read_index = 0;
if (kSourceIsOneByte) {
size_t writeable = std::min(string.size(), content_capacity);
size_t ascii_length =
Utf8::WriteLeadingAscii(characters, buffer, writeable);
read_index = ascii_length;
write_index = ascii_length;
}
uint16_t last = Utf16::kNoPreviousCharacter;
for (; read_index < string.size(); read_index++) {
Char character = characters[read_index];
size_t required_capacity;
if constexpr (kSourceIsOneByte) {
required_capacity = Utf8::LengthOneByte(character);
} else {
required_capacity = Utf8::Length(character, last);
}
size_t remaining_capacity = content_capacity - write_index;
if (remaining_capacity < required_capacity) {
if (Utf16::IsSurrogatePair(last, character)) {
DCHECK_GE(write_index, Utf8::kSizeOfUnmatchedSurrogate);
write_index -= Utf8::kSizeOfUnmatchedSurrogate;
DCHECK_NE(read_index, 0);
--read_index;
}
break;
}
if constexpr (kSourceIsOneByte) {
write_index += Utf8::EncodeOneByte(buffer + write_index, character);
} else {
if ((read_index + 1 < string.size()) &&
Utf16::IsSurrogatePair(character, characters[read_index + 1])) {
write_index += Utf8::kSizeOfUnmatchedSurrogate;
} else {
write_index += Utf8::Encode(buffer + write_index, character, last,
replace_invalid_utf8);
}
}
last = character;
}
DCHECK_LE(write_index, capacity);
if (write_null) {
DCHECK_LT(write_index, capacity);
buffer[write_index++] = '\0';
}
size_t bytes_written = write_index;
size_t characters_processed = read_index;
return {bytes_written, characters_processed};
}
}
#endif